This is an archive of the discontinued LLVM Phabricator instance.

Differential D115684

[benchmark] Pull latest; disable -Werror
ClosedPublic

Authored by mtrofin on Dec 13 2021, 4:11 PM.

Details

Reviewers
aeubanks
Meinersbur
ldionne
lebedev.ri
jdoerfert
thakis
Commits
rGa290770fa848: [benchmark] Pull latest; disable -Werror
Summary

Pulled the latest (b000672), which adds an option to disable -Werror,
which caused issues previously (see for instance
https://reviews.llvm.org/D112012)

Applied https://github.com/google/benchmark/pull/1305 to unblock, as
this issue is causing downstream buildbot failures.

The only change is in llvm/CMakeLists.txt, where we set
BENCHMARK_ENABLE_WERROR to have the same value as LLVM_ENABLE_WERROR.

This patch also introduces a convenience script to pull benchmark from
upstream.

Diff Detail

Unit TestsFailed

TimeTest
90 msx64 debian > LLVM.Bindings/Go::go.test

Event Timeline

mtrofin created this revision.Dec 13 2021, 4:11 PM
Herald added subscribers: luke957, s.egerton, arphaman and 3 others. · View Herald TranscriptDec 13 2021, 4:11 PM
mtrofin requested review of this revision.Dec 13 2021, 4:11 PM
Herald added a reviewer: jdoerfert. · View Herald TranscriptDec 13 2021, 4:11 PM
Herald added a project: Restricted Project. · View Herald Transcript
Herald added subscribers: llvm-commits, sstefan1, aheejin. · View Herald Transcript
mtrofin updated this revision to Diff 394082.Dec 13 2021, 4:26 PM

spurious change in cmakelists

Herald added a subscriber: ormris. · View Herald TranscriptDec 13 2021, 4:26 PM
thakis accepted this revision.Dec 13 2021, 4:31 PM
thakis added a subscriber: thakis.

stamp

This revision is now accepted and ready to land.Dec 13 2021, 4:31 PM
This revision was landed with ongoing or failed builds.Dec 13 2021, 5:13 PM
Closed by commit rGa290770fa848: [benchmark] Pull latest; disable -Werror (authored by mtrofin). · Explain Why
This revision was automatically updated to reflect the committed changes.
mtrofin added a commit: rGa290770fa848: [benchmark] Pull latest; disable -Werror.
Harbormaster completed remote builds in B139102: Diff 394082.Dec 13 2021, 5:41 PM

Revision Contents

PathSize
llvm/
2 lines
third-party/
benchmark/
5 lines
7 lines
208 lines
115 lines
2 lines
12 lines
45 lines
2 lines
1186 lines
15 lines
bindings/
python/
google_benchmark/
6 lines
2 lines
cmake/
6 lines
15 lines
5 lines
Modules/
16 lines
16 lines
15 lines
26 lines
docs/
2 lines
10 lines
48 lines
19 lines
docs/
11 lines
user_guide.md
README.md
440 lines
include/
benchmark/
247 lines
1 line
3 lines
src/
87 lines
109 lines
8 lines
18 lines
22 lines
87 lines
20 lines
42 lines
39 lines
8 lines
20 lines
5 lines
18 lines
53 lines
18 lines
4 lines
104 lines
2 lines
2 lines
2 lines
2 lines
2 lines
12 lines
5 lines
1 line
39 lines
12 lines
23 lines
73 lines
1 line
8 lines
42 lines
test/
1 line
6 lines
8 lines
56 lines
17 lines
157 lines
28 lines
1 line
18 lines
7 lines
6 lines
10 lines
8 lines
13 lines
2 lines
12 lines
1 line
4 lines
8 lines
7 lines
14 lines
85 lines
8 lines
4 lines
8 lines
10 lines
10 lines
171 lines
19 lines
95 lines
7 lines
149 lines
4 lines
61 lines
2 lines
3 lines
tools/
gbench/
Inputs/
21 lines
21 lines
213 lines
14 lines

Diff 394082

llvm/CMakeLists.txt

Show First 20 Lines • Show All 1,203 Lines • ▼ Show 20 Lines
if (LLVM_INCLUDE_BENCHMARKS) if (LLVM_INCLUDE_BENCHMARKS)
# Override benchmark defaults so that when the library itself is updated these # Override benchmark defaults so that when the library itself is updated these
# modifications are not lost. # modifications are not lost.
set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing" FORCE) set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing" FORCE)
set(BENCHMARK_ENABLE_EXCEPTIONS OFF CACHE BOOL "Disable benchmark exceptions" FORCE) set(BENCHMARK_ENABLE_EXCEPTIONS OFF CACHE BOOL "Disable benchmark exceptions" FORCE)
set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Don't install benchmark" FORCE) set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Don't install benchmark" FORCE)
set(BENCHMARK_DOWNLOAD_DEPENDENCIES OFF CACHE BOOL "Don't download dependencies" FORCE) set(BENCHMARK_DOWNLOAD_DEPENDENCIES OFF CACHE BOOL "Don't download dependencies" FORCE)
set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "Disable Google Test in benchmark" FORCE) set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "Disable Google Test in benchmark" FORCE)
set(BENCHMARK_ENABLE_WERROR ${LLVM_ENABLE_WERROR} CACHE BOOL
"Handle -Werror for Google Benchmark based on LLVM_ENABLE_WERROR" FORCE)
# Since LLVM requires C++11 it is safe to assume that std::regex is available. # Since LLVM requires C++11 it is safe to assume that std::regex is available.
set(HAVE_STD_REGEX ON CACHE BOOL "OK" FORCE) set(HAVE_STD_REGEX ON CACHE BOOL "OK" FORCE)
add_subdirectory(${LLVM_THIRD_PARTY_DIR}/benchmark add_subdirectory(${LLVM_THIRD_PARTY_DIR}/benchmark
${CMAKE_CURRENT_BINARY_DIR}/third-party/benchmark) ${CMAKE_CURRENT_BINARY_DIR}/third-party/benchmark)
add_subdirectory(benchmarks) add_subdirectory(benchmarks)
endif() endif()
if (LLVM_INCLUDE_UTILS AND LLVM_INCLUDE_TOOLS) if (LLVM_INCLUDE_UTILS AND LLVM_INCLUDE_TOOLS)
add_subdirectory(utils/llvm-locstats) add_subdirectory(utils/llvm-locstats)
endif() endif()

third-party/benchmark/.clang-format

  • This file was added.
---
Language: Cpp
BasedOnStyle: Google
PointerAlignment: Left
...

third-party/benchmark/.clang-tidy

  • This file was added.
---
Checks: 'clang-analyzer-*,readability-redundant-*,performance-*'
WarningsAsErrors: 'clang-analyzer-*,readability-redundant-*,performance-*'
HeaderFilterRegex: '.*'
AnalyzeTemporaryDtors: false
FormatStyle: none
User: user

third-party/benchmark/.travis.yml

  • This file was added.
sudo: required
dist: trusty
language: cpp
matrix:
include:
- compiler: gcc
addons:
apt:
packages:
- lcov
env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Coverage
- compiler: gcc
addons:
apt:
packages:
- g++-multilib
- libc6:i386
env:
- COMPILER=g++
- C_COMPILER=gcc
- BUILD_TYPE=Debug
- BUILD_32_BITS=ON
- EXTRA_FLAGS="-m32"
- compiler: gcc
addons:
apt:
packages:
- g++-multilib
- libc6:i386
env:
- COMPILER=g++
- C_COMPILER=gcc
- BUILD_TYPE=Release
- BUILD_32_BITS=ON
- EXTRA_FLAGS="-m32"
- compiler: gcc
env:
- INSTALL_GCC6_FROM_PPA=1
- COMPILER=g++-6 C_COMPILER=gcc-6 BUILD_TYPE=Debug
- ENABLE_SANITIZER=1
- EXTRA_FLAGS="-fno-omit-frame-pointer -g -O2 -fsanitize=undefined,address -fuse-ld=gold"
# Clang w/ libc++
- compiler: clang
dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug
- LIBCXX_BUILD=1
- EXTRA_CXX_FLAGS="-stdlib=libc++"
- compiler: clang
dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release
- LIBCXX_BUILD=1
- EXTRA_CXX_FLAGS="-stdlib=libc++"
# Clang w/ 32bit libc++
- compiler: clang
dist: xenial
addons:
apt:
packages:
- clang-3.8
- g++-multilib
- libc6:i386
env:
- INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug
- LIBCXX_BUILD=1
- BUILD_32_BITS=ON
- EXTRA_FLAGS="-m32"
- EXTRA_CXX_FLAGS="-stdlib=libc++"
# Clang w/ 32bit libc++
- compiler: clang
dist: xenial
addons:
apt:
packages:
- clang-3.8
- g++-multilib
- libc6:i386
env:
- INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release
- LIBCXX_BUILD=1
- BUILD_32_BITS=ON
- EXTRA_FLAGS="-m32"
- EXTRA_CXX_FLAGS="-stdlib=libc++"
# Clang w/ libc++, ASAN, UBSAN
- compiler: clang
dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug
- LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address"
- ENABLE_SANITIZER=1
- EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all"
- EXTRA_CXX_FLAGS="-stdlib=libc++"
- UBSAN_OPTIONS=print_stacktrace=1
# Clang w/ libc++ and MSAN
- compiler: clang
dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug
- LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins
- ENABLE_SANITIZER=1
- EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins"
- EXTRA_CXX_FLAGS="-stdlib=libc++"
# Clang w/ libc++ and MSAN
- compiler: clang
dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo
- LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread
- ENABLE_SANITIZER=1
- EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all"
- EXTRA_CXX_FLAGS="-stdlib=libc++"
- os: osx
osx_image: xcode8.3
compiler: clang
env:
- COMPILER=clang++
- BUILD_TYPE=Release
- BUILD_32_BITS=ON
- EXTRA_FLAGS="-m32"
before_script:
- if [ -n "${LIBCXX_BUILD}" ]; then
source .libcxx-setup.sh;
fi
- if [ -n "${ENABLE_SANITIZER}" ]; then
export EXTRA_OPTIONS="-DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF";
else
export EXTRA_OPTIONS="";
fi
- mkdir -p build && cd build
before_install:
- if [ -z "$BUILD_32_BITS" ]; then
export BUILD_32_BITS=OFF && echo disabling 32 bit build;
fi
- if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then
sudo add-apt-repository -y "ppa:ubuntu-toolchain-r/test";
sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60";
fi
install:
- if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then
travis_wait sudo -E apt-get -yq --no-install-suggests --no-install-recommends install g++-6;
fi
- if [ "${TRAVIS_OS_NAME}" == "linux" -a "${BUILD_32_BITS}" == "OFF" ]; then
travis_wait sudo -E apt-get -y --no-install-suggests --no-install-recommends install llvm-3.9-tools;
sudo cp /usr/lib/llvm-3.9/bin/FileCheck /usr/local/bin/;
fi
- if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then
PATH=~/.local/bin:${PATH};
pip install --user --upgrade pip;
travis_wait pip install --user cpp-coveralls;
fi
- if [ "${C_COMPILER}" == "gcc-7" -a "${TRAVIS_OS_NAME}" == "osx" ]; then
rm -f /usr/local/include/c++;
brew update;
travis_wait brew install gcc@7;
fi
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
sudo apt-get update -qq;
sudo apt-get install -qq unzip cmake3;
wget https://github.com/bazelbuild/bazel/releases/download/3.2.0/bazel-3.2.0-installer-linux-x86_64.sh --output-document bazel-installer.sh;
travis_wait sudo bash bazel-installer.sh;
fi
- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
curl -L -o bazel-installer.sh https://github.com/bazelbuild/bazel/releases/download/3.2.0/bazel-3.2.0-installer-darwin-x86_64.sh;
travis_wait sudo bash bazel-installer.sh;
fi
script:
- cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_FLAGS="${EXTRA_FLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} ${EXTRA_CXX_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} ..
- make
- ctest -C ${BUILD_TYPE} --output-on-failure
- bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/...
after_success:
- if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then
coveralls --include src --include include --gcov-options '\-lp' --root .. --build-root .;
fi

third-party/benchmark/.ycm_extra_conf.py

  • This file was added.
import os
import ycm_core
# These are the compilation flags that will be used in case there's no
# compilation database set (by default, one is not set).
# CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR.
flags = [
'-Wall',
'-Werror',
'-pedantic-errors',
'-std=c++0x',
'-fno-strict-aliasing',
'-O3',
'-DNDEBUG',
# ...and the same thing goes for the magic -x option which specifies the
# language that the files to be compiled are written in. This is mostly
# relevant for c++ headers.
# For a C project, you would set this to 'c' instead of 'c++'.
'-x', 'c++',
'-I', 'include',
'-isystem', '/usr/include',
'-isystem', '/usr/local/include',
]
# Set this to the absolute path to the folder (NOT the file!) containing the
# compile_commands.json file to use that instead of 'flags'. See here for
# more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html
#
# Most projects will NOT need to set this to anything; you can just change the
# 'flags' list of compilation flags. Notice that YCM itself uses that approach.
compilation_database_folder = ''
if os.path.exists( compilation_database_folder ):
database = ycm_core.CompilationDatabase( compilation_database_folder )
else:
database = None
SOURCE_EXTENSIONS = [ '.cc' ]
def DirectoryOfThisScript():
return os.path.dirname( os.path.abspath( __file__ ) )
def MakeRelativePathsInFlagsAbsolute( flags, working_directory ):
if not working_directory:
return list( flags )
new_flags = []
make_next_absolute = False
path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ]
for flag in flags:
new_flag = flag
if make_next_absolute:
make_next_absolute = False
if not flag.startswith( '/' ):
new_flag = os.path.join( working_directory, flag )
for path_flag in path_flags:
if flag == path_flag:
make_next_absolute = True
break
if flag.startswith( path_flag ):
path = flag[ len( path_flag ): ]
new_flag = path_flag + os.path.join( working_directory, path )
break
if new_flag:
new_flags.append( new_flag )
return new_flags
def IsHeaderFile( filename ):
extension = os.path.splitext( filename )[ 1 ]
return extension in [ '.h', '.hxx', '.hpp', '.hh' ]
def GetCompilationInfoForFile( filename ):
# The compilation_commands.json file generated by CMake does not have entries
# for header files. So we do our best by asking the db for flags for a
# corresponding source file, if any. If one exists, the flags for that file
# should be good enough.
if IsHeaderFile( filename ):
basename = os.path.splitext( filename )[ 0 ]
for extension in SOURCE_EXTENSIONS:
replacement_file = basename + extension
if os.path.exists( replacement_file ):
compilation_info = database.GetCompilationInfoForFile(
replacement_file )
if compilation_info.compiler_flags_:
return compilation_info
return None
return database.GetCompilationInfoForFile( filename )
def FlagsForFile( filename, **kwargs ):
if database:
# Bear in mind that compilation_info.compiler_flags_ does NOT return a
# python list, but a "list-like" StringVec object
compilation_info = GetCompilationInfoForFile( filename )
if not compilation_info:
return None
final_flags = MakeRelativePathsInFlagsAbsolute(
compilation_info.compiler_flags_,
compilation_info.compiler_working_dir_ )
else:
relative_to = DirectoryOfThisScript()
final_flags = MakeRelativePathsInFlagsAbsolute( flags, relative_to )
return {
'flags': final_flags,
'do_cache': True
}

third-party/benchmark/AUTHORS

Show All 15 Lines
Christian Wassermann <christian_wassermann@web.de> Christian Wassermann <christian_wassermann@web.de>
Christopher Seymour <chris.j.seymour@hotmail.com> Christopher Seymour <chris.j.seymour@hotmail.com>
Colin Braley <braley.colin@gmail.com> Colin Braley <braley.colin@gmail.com>
Daniel Harvey <danielharvey458@gmail.com> Daniel Harvey <danielharvey458@gmail.com>
David Coeurjolly <david.coeurjolly@liris.cnrs.fr> David Coeurjolly <david.coeurjolly@liris.cnrs.fr>
Deniz Evrenci <denizevrenci@gmail.com> Deniz Evrenci <denizevrenci@gmail.com>
Dirac Research Dirac Research
Dominik Czarnota <dominik.b.czarnota@gmail.com> Dominik Czarnota <dominik.b.czarnota@gmail.com>
Dominik Korman <kormandominik@gmail.com>
Donald Aingworth <donalds_junk_mail@yahoo.com>
Eric Backus <eric_backus@alum.mit.edu> Eric Backus <eric_backus@alum.mit.edu>
Eric Fiselier <eric@efcs.ca> Eric Fiselier <eric@efcs.ca>
Eugene Zhuk <eugene.zhuk@gmail.com> Eugene Zhuk <eugene.zhuk@gmail.com>
Evgeny Safronov <division494@gmail.com> Evgeny Safronov <division494@gmail.com>
Federico Ficarelli <federico.ficarelli@gmail.com> Federico Ficarelli <federico.ficarelli@gmail.com>
Felix Homann <linuxaudio@showlabor.de> Felix Homann <linuxaudio@showlabor.de>
Gergő Szitár <szitar.gergo@gmail.com> Gergő Szitár <szitar.gergo@gmail.com>
Google Inc. Google Inc.
Show All 29 Lines

third-party/benchmark/BUILD.bazel

load("@rules_cc//cc:defs.bzl", "cc_library")
licenses(["notice"]) licenses(["notice"])
config_setting( config_setting(
name = "qnx",
constraint_values = ["@platforms//os:qnx"],
values = {
"cpu": "x64_qnx",
},
visibility = [":__subpackages__"],
)
config_setting(
name = "windows", name = "windows",
constraint_values = ["@platforms//os:windows"],
values = { values = {
"cpu": "x64_windows", "cpu": "x64_windows",
}, },
visibility = [":__subpackages__"], visibility = [":__subpackages__"],
) )
cc_library( cc_library(
name = "benchmark", name = "benchmark",
Show All 30 Lines

third-party/benchmark/CMakeLists.txt

cmake_minimum_required (VERSION 3.5.1) cmake_minimum_required (VERSION 3.5.1)
foreach(p foreach(p
CMP0048 # OK to clear PROJECT_VERSION on project() CMP0048 # OK to clear PROJECT_VERSION on project()
CMP0054 # CMake 3.1 CMP0054 # CMake 3.1
CMP0056 # export EXE_LINKER_FLAGS to try_run CMP0056 # export EXE_LINKER_FLAGS to try_run
CMP0057 # Support no if() IN_LIST operator CMP0057 # Support no if() IN_LIST operator
CMP0063 # Honor visibility properties for all targets CMP0063 # Honor visibility properties for all targets
CMP0077 # Allow option() overrides in importing projects CMP0077 # Allow option() overrides in importing projects
) )
if(POLICY ${p}) if(POLICY ${p})
cmake_policy(SET ${p} NEW) cmake_policy(SET ${p} NEW)
endif() endif()
endforeach() endforeach()
project (benchmark VERSION 1.5.4 LANGUAGES CXX) project (benchmark VERSION 1.6.0 LANGUAGES CXX)
option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON)
option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON)
option(BENCHMARK_ENABLE_LTO "Enable link time optimisation of the benchmark library." OFF) option(BENCHMARK_ENABLE_LTO "Enable link time optimisation of the benchmark library." OFF)
option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard library." OFF) option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard library." OFF)
option(BENCHMARK_ENABLE_WERROR "Build Release candidates with -Werror." ON)
option(BENCHMARK_FORCE_WERROR "Build Release candidates with -Werror regardless of compiler issues." OFF)
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI")
# PGC++ maybe reporting false positives.
set(BENCHMARK_ENABLE_WERROR OFF)
endif()
if(BENCHMARK_FORCE_WERROR)
set(BENCHMARK_ENABLE_WERROR ON)
endif(BENCHMARK_FORCE_WERROR)
if(NOT MSVC) if(NOT MSVC)
option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF) option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF)
else() else()
set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL "Build a 32 bit version of the library - unsupported when using MSVC)" FORCE) set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL "Build a 32 bit version of the library - unsupported when using MSVC)" FORCE)
endif() endif()
option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark. (Projects embedding benchmark may want to turn this OFF.)" ON) option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark. (Projects embedding benchmark may want to turn this OFF.)" ON)
option(BENCHMARK_ENABLE_DOXYGEN "Build documentation with Doxygen." OFF)
option(BENCHMARK_INSTALL_DOCS "Enable installation of documentation." ON)
# Allow unmet dependencies to be met using CMake's ExternalProject mechanics, which # Allow unmet dependencies to be met using CMake's ExternalProject mechanics, which
# may require downloading the source code. # may require downloading the source code.
option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree building of unmet dependencies" OFF) option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree building of unmet dependencies" OFF)
# This option can be used to disable building and running unit tests which depend on gtest # This option can be used to disable building and running unit tests which depend on gtest
# in cases where it is not possible to build or find a valid version of gtest. # in cases where it is not possible to build or find a valid version of gtest.
option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON) option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON)
option(BENCHMARK_USE_BUNDLED_GTEST "Use bundled GoogleTest. If disabled, the find_package(GTest) will be used." ON)
option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm" OFF) option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm" OFF)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
if(MSVC) if(MSVC)
# As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and # As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and
# cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the # cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the
# undocumented, but working variable. # undocumented, but working variable.
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines
# The version of the libraries # The version of the libraries
set(GENERIC_LIB_VERSION ${VERSION}) set(GENERIC_LIB_VERSION ${VERSION})
string(SUBSTRING ${VERSION} 0 1 GENERIC_LIB_SOVERSION) string(SUBSTRING ${VERSION} 0 1 GENERIC_LIB_SOVERSION)
# Import our CMake modules # Import our CMake modules
include(CheckCXXCompilerFlag) include(CheckCXXCompilerFlag)
include(AddCXXCompilerFlag) include(AddCXXCompilerFlag)
include(CXXFeatureCheck) include(CXXFeatureCheck)
include(CheckLibraryExists)
check_library_exists(rt shm_open "" HAVE_LIB_RT)
if (BENCHMARK_BUILD_32_BITS) if (BENCHMARK_BUILD_32_BITS)
add_required_cxx_compiler_flag(-m32) add_required_cxx_compiler_flag(-m32)
endif() endif()
if (MSVC) if (MSVC)
# Turn compiler warnings up to 11 # Turn compiler warnings up to 11
string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
Show All 32 Lineselse()
if (NOT HAVE_CXX_FLAG_STD_CXX11) if (NOT HAVE_CXX_FLAG_STD_CXX11)
add_cxx_compiler_flag(-std=c++0x) add_cxx_compiler_flag(-std=c++0x)
endif() endif()
# Turn compiler warnings up to 11 # Turn compiler warnings up to 11
add_cxx_compiler_flag(-Wall) add_cxx_compiler_flag(-Wall)
add_cxx_compiler_flag(-Wextra) add_cxx_compiler_flag(-Wextra)
add_cxx_compiler_flag(-Wshadow) add_cxx_compiler_flag(-Wshadow)
if(BENCHMARK_ENABLE_WERROR)
add_cxx_compiler_flag(-Werror RELEASE) add_cxx_compiler_flag(-Werror RELEASE)
add_cxx_compiler_flag(-Werror RELWITHDEBINFO) add_cxx_compiler_flag(-Werror RELWITHDEBINFO)
add_cxx_compiler_flag(-Werror MINSIZEREL) add_cxx_compiler_flag(-Werror MINSIZEREL)
endif()
if (NOT BENCHMARK_ENABLE_TESTING) if (NOT BENCHMARK_ENABLE_TESTING)
# Disable warning when compiling tests as gtest does not use 'override'. # Disable warning when compiling tests as gtest does not use 'override'.
add_cxx_compiler_flag(-Wsuggest-override) add_cxx_compiler_flag(-Wsuggest-override)
endif() endif()
add_cxx_compiler_flag(-pedantic) add_cxx_compiler_flag(-pedantic)
add_cxx_compiler_flag(-pedantic-errors) add_cxx_compiler_flag(-pedantic-errors)
add_cxx_compiler_flag(-Wshorten-64-to-32) add_cxx_compiler_flag(-Wshorten-64-to-32)
add_cxx_compiler_flag(-fstrict-aliasing) add_cxx_compiler_flag(-fstrict-aliasing)
# Disable warnings regarding deprecated parts of the library while building # Disable warnings regarding deprecated parts of the library while building
# and testing those parts of the library. # and testing those parts of the library.
add_cxx_compiler_flag(-Wno-deprecated-declarations) add_cxx_compiler_flag(-Wno-deprecated-declarations)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
# Intel silently ignores '-Wno-deprecated-declarations', # Intel silently ignores '-Wno-deprecated-declarations',
# warning no. 1786 must be explicitly disabled. # warning no. 1786 must be explicitly disabled.
# See #631 for rationale. # See #631 for rationale.
add_cxx_compiler_flag(-wd1786) add_cxx_compiler_flag(-wd1786)
endif() endif()
# Disable deprecation warnings for release builds (when -Werror is enabled). # Disable deprecation warnings for release builds (when -Werror is enabled).
if(BENCHMARK_ENABLE_WERROR)
add_cxx_compiler_flag(-Wno-deprecated RELEASE) add_cxx_compiler_flag(-Wno-deprecated RELEASE)
add_cxx_compiler_flag(-Wno-deprecated RELWITHDEBINFO) add_cxx_compiler_flag(-Wno-deprecated RELWITHDEBINFO)
add_cxx_compiler_flag(-Wno-deprecated MINSIZEREL) add_cxx_compiler_flag(-Wno-deprecated MINSIZEREL)
endif()
if (NOT BENCHMARK_ENABLE_EXCEPTIONS) if (NOT BENCHMARK_ENABLE_EXCEPTIONS)
add_cxx_compiler_flag(-fno-exceptions) add_cxx_compiler_flag(-fno-exceptions)
endif() endif()
if (HAVE_CXX_FLAG_FSTRICT_ALIASING) if (HAVE_CXX_FLAG_FSTRICT_ALIASING)
if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing
add_cxx_compiler_flag(-Wstrict-aliasing) add_cxx_compiler_flag(-Wstrict-aliasing)
endif() endif()
▲ Show 20 Lines • Show All 107 Lines • ▼ Show 20 Lines
# Build the targets # Build the targets
add_subdirectory(src) add_subdirectory(src)
if (BENCHMARK_ENABLE_TESTING) if (BENCHMARK_ENABLE_TESTING)
enable_testing() enable_testing()
if (BENCHMARK_ENABLE_GTEST_TESTS AND if (BENCHMARK_ENABLE_GTEST_TESTS AND
NOT (TARGET gtest AND TARGET gtest_main AND NOT (TARGET gtest AND TARGET gtest_main AND
TARGET gmock AND TARGET gmock_main)) TARGET gmock AND TARGET gmock_main))
if (BENCHMARK_USE_BUNDLED_GTEST)
include(GoogleTest) include(GoogleTest)
else()
find_package(GTest CONFIG REQUIRED)
add_library(gtest ALIAS GTest::gtest)
add_library(gtest_main ALIAS GTest::gtest_main)
add_library(gmock ALIAS GTest::gmock)
add_library(gmock_main ALIAS GTest::gmock_main)
endif()
endif() endif()
add_subdirectory(test) add_subdirectory(test)
endif() endif()

third-party/benchmark/CONTRIBUTORS

Show All 32 Lines
Christopher Seymour <chris.j.seymour@hotmail.com> Christopher Seymour <chris.j.seymour@hotmail.com>
Colin Braley <braley.colin@gmail.com> Colin Braley <braley.colin@gmail.com>
Cyrille Faucheux <cyrille.faucheux@gmail.com> Cyrille Faucheux <cyrille.faucheux@gmail.com>
Daniel Harvey <danielharvey458@gmail.com> Daniel Harvey <danielharvey458@gmail.com>
David Coeurjolly <david.coeurjolly@liris.cnrs.fr> David Coeurjolly <david.coeurjolly@liris.cnrs.fr>
Deniz Evrenci <denizevrenci@gmail.com> Deniz Evrenci <denizevrenci@gmail.com>
Dominic Hamon <dma@stripysock.com> <dominic@google.com> Dominic Hamon <dma@stripysock.com> <dominic@google.com>
Dominik Czarnota <dominik.b.czarnota@gmail.com> Dominik Czarnota <dominik.b.czarnota@gmail.com>
Dominik Korman <kormandominik@gmail.com>
Donald Aingworth <donalds_junk_mail@yahoo.com>
Eric Backus <eric_backus@alum.mit.edu> Eric Backus <eric_backus@alum.mit.edu>
Eric Fiselier <eric@efcs.ca> Eric Fiselier <eric@efcs.ca>
Eugene Zhuk <eugene.zhuk@gmail.com> Eugene Zhuk <eugene.zhuk@gmail.com>
Evgeny Safronov <division494@gmail.com> Evgeny Safronov <division494@gmail.com>
Fanbo Meng <fanbo.meng@ibm.com> Fanbo Meng <fanbo.meng@ibm.com>
Federico Ficarelli <federico.ficarelli@gmail.com> Federico Ficarelli <federico.ficarelli@gmail.com>
Felix Homann <linuxaudio@showlabor.de> Felix Homann <linuxaudio@showlabor.de>
Geoffrey Martin-Noble <gcmn@google.com> <gmngeoffrey@gmail.com> Geoffrey Martin-Noble <gcmn@google.com> <gmngeoffrey@gmail.com>
Show All 37 Lines

third-party/benchmark/README.md

  • This file was copied to third-party/benchmark/docs/user_guide.md.
Show All 21 Linesstatic void BM_SomeFunction(benchmark::State& state) {
} }
} }
// Register the function as a benchmark // Register the function as a benchmark
BENCHMARK(BM_SomeFunction); BENCHMARK(BM_SomeFunction);
// Run the benchmark // Run the benchmark
BENCHMARK_MAIN(); BENCHMARK_MAIN();
``` ```
## Getting Started
To get started, see [Requirements](#requirements) and To get started, see [Requirements](#requirements) and
[Installation](#installation). See [Usage](#usage) for a full example and the [Installation](#installation). See [Usage](#usage) for a full example and the
[User Guide](#user-guide) for a more comprehensive feature overview. [User Guide](docs/user_guide.md) for a more comprehensive feature overview.
It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/docs/primer.md) It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/docs/primer.md)
as some of the structural aspects of the APIs are similar. as some of the structural aspects of the APIs are similar.
### Resources ## Resources
[Discussion group](https://groups.google.com/d/forum/benchmark-discuss) [Discussion group](https://groups.google.com/d/forum/benchmark-discuss)
IRC channels: IRC channels:
* [libera](https://libera.chat) #benchmark * [libera](https://libera.chat) #benchmark
[Additional Tooling Documentation](docs/tools.md) [Additional Tooling Documentation](docs/tools.md)
[Assembly Testing Documentation](docs/AssemblyTests.md) [Assembly Testing Documentation](docs/AssemblyTests.md)
## Requirements ## Requirements
The library can be used with C++03. However, it requires C++11 to build, The library can be used with C++03. However, it requires C++11 to build,
including compiler and standard library support. including compiler and standard library support.
The following minimum versions are required to build the library: The following minimum versions are required to build the library:
* GCC 4.8 * GCC 4.8
* Clang 3.4 * Clang 3.4
* Visual Studio 14 2015 * Visual Studio 14 2015
* Intel 2015 Update 1 * Intel 2015 Update 1
See [Platform-Specific Build Instructions](#platform-specific-build-instructions). See [Platform-Specific Build Instructions](docs/platform_specific_build_instructions.md).
## Installation ## Installation
This describes the installation process using cmake. As pre-requisites, you'll This describes the installation process using cmake. As pre-requisites, you'll
need git and cmake installed. need git and cmake installed.
_See [dependencies.md](dependencies.md) for more details regarding supported _See [dependencies.md](docs/dependencies.md) for more details regarding supported
versions of build tools._ versions of build tools._
```bash ```bash
# Check out the library. # Check out the library.
$ git clone https://github.com/google/benchmark.git $ git clone https://github.com/google/benchmark.git
# Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory.
$ git clone https://github.com/google/googletest.git benchmark/googletest
# Go to the library root directory # Go to the library root directory
$ cd benchmark $ cd benchmark
# Make a build directory to place the build output. # Make a build directory to place the build output.
$ cmake -E make_directory "build" $ cmake -E make_directory "build"
# Generate build system files with cmake. # Generate build system files with cmake, and download any dependencies.
$ cmake -E chdir "build" cmake -DCMAKE_BUILD_TYPE=Release ../ $ cmake -E chdir "build" cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on -DCMAKE_BUILD_TYPE=Release ../
# or, starting with CMake 3.13, use a simpler form: # or, starting with CMake 3.13, use a simpler form:
# cmake -DCMAKE_BUILD_TYPE=Release -S . -B "build" # cmake -DCMAKE_BUILD_TYPE=Release -S . -B "build"
# Build the library. # Build the library.
$ cmake --build "build" --config Release $ cmake --build "build" --config Release
``` ```
This builds the `benchmark` and `benchmark_main` libraries and tests. This builds the `benchmark` and `benchmark_main` libraries and tests.
On a unix system, the build directory should now look something like this: On a unix system, the build directory should now look something like this:
Show All 17 Lines
``` ```
sudo cmake --build "build" --config Release --target install sudo cmake --build "build" --config Release --target install
``` ```
Note that Google Benchmark requires Google Test to build and run the tests. This Note that Google Benchmark requires Google Test to build and run the tests. This
dependency can be provided two ways: dependency can be provided two ways:
* Checkout the Google Test sources into `benchmark/googletest` as above. * Checkout the Google Test sources into `benchmark/googletest`.
* Otherwise, if `-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON` is specified during * Otherwise, if `-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON` is specified during
configuration, the library will automatically download and build any required configuration as above, the library will automatically download and build
dependencies. any required dependencies.
If you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF` If you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF`
to `CMAKE_ARGS`. to `CMAKE_ARGS`.
### Debug vs Release ### Debug vs Release
By default, benchmark builds as a debug library. You will see a warning in the By default, benchmark builds as a debug library. You will see a warning in the
output when this is the case. To build it as a release library instead, add output when this is the case. To build it as a release library instead, add
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines
$ g++ mybenchmark.cc -std=c++11 -isystem benchmark/include \ $ g++ mybenchmark.cc -std=c++11 -isystem benchmark/include \
-Lbenchmark/build/src -lbenchmark -lpthread -o mybenchmark -Lbenchmark/build/src -lbenchmark -lpthread -o mybenchmark
``` ```
Alternatively, link against the `benchmark_main` library and remove Alternatively, link against the `benchmark_main` library and remove
`BENCHMARK_MAIN();` above to get the same behavior. `BENCHMARK_MAIN();` above to get the same behavior.
The compiled executable will run all benchmarks by default. Pass the `--help` The compiled executable will run all benchmarks by default. Pass the `--help`
flag for option information or see the guide below. flag for option information or see the [User Guide](docs/user_guide.md).
### Usage with CMake ### Usage with CMake
If using CMake, it is recommended to link against the project-provided If using CMake, it is recommended to link against the project-provided
`benchmark::benchmark` and `benchmark::benchmark_main` targets using `benchmark::benchmark` and `benchmark::benchmark_main` targets using
`target_link_libraries`. `target_link_libraries`.
It is possible to use ```find_package``` to import an installed version of the It is possible to use ```find_package``` to import an installed version of the
library. library.
```cmake ```cmake
find_package(benchmark REQUIRED) find_package(benchmark REQUIRED)
``` ```
Alternatively, ```add_subdirectory``` will incorporate the library directly in Alternatively, ```add_subdirectory``` will incorporate the library directly in
to one's CMake project. to one's CMake project.
```cmake ```cmake
add_subdirectory(benchmark) add_subdirectory(benchmark)
``` ```
Either way, link to the library as follows. Either way, link to the library as follows.
```cmake ```cmake
target_link_libraries(MyTarget benchmark::benchmark) target_link_libraries(MyTarget benchmark::benchmark)
``` ```
## Platform Specific Build Instructions
### Building with GCC
When the library is built using GCC it is necessary to link with the pthread
library due to how GCC implements `std::thread`. Failing to link to pthread will
lead to runtime exceptions (unless you're using libc++), not linker errors. See
[issue #67](https://github.com/google/benchmark/issues/67) for more details. You
can link to pthread by adding `-pthread` to your linker command. Note, you can
also use `-lpthread`, but there are potential issues with ordering of command
line parameters if you use that.
### Building with Visual Studio 2015 or 2017
The `shlwapi` library (`-lshlwapi`) is required to support a call to `CPUInfo` which reads the registry. Either add `shlwapi.lib` under `[ Configuration Properties > Linker > Input ]`, or use the following:
```
// Alternatively, can add libraries using linker options.
#ifdef _WIN32
#pragma comment ( lib, "Shlwapi.lib" )
#ifdef _DEBUG
#pragma comment ( lib, "benchmarkd.lib" )
#else
#pragma comment ( lib, "benchmark.lib" )
#endif
#endif
```
Can also use the graphical version of CMake:
* Open `CMake GUI`.
* Under `Where to build the binaries`, same path as source plus `build`.
* Under `CMAKE_INSTALL_PREFIX`, same path as source plus `install`.
* Click `Configure`, `Generate`, `Open Project`.
* If build fails, try deleting entire directory and starting again, or unticking options to build less.
### Building with Intel 2015 Update 1 or Intel System Studio Update 4
See instructions for building with Visual Studio. Once built, right click on the solution and change the build to Intel.
### Building on Solaris
If you're running benchmarks on solaris, you'll want the kstat library linked in
too (`-lkstat`).
## User Guide
### Command Line
[Output Formats](#output-formats)
[Output Files](#output-files)
[Running Benchmarks](#running-benchmarks)
[Running a Subset of Benchmarks](#running-a-subset-of-benchmarks)
[Result Comparison](#result-comparison)
[Extra Context](#extra-context)
### Library
[Runtime and Reporting Considerations](#runtime-and-reporting-considerations)
[Passing Arguments](#passing-arguments)
[Custom Benchmark Name](#custom-benchmark-name)
[Calculating Asymptotic Complexity](#asymptotic-complexity)
[Templated Benchmarks](#templated-benchmarks)
[Fixtures](#fixtures)
[Custom Counters](#custom-counters)
[Multithreaded Benchmarks](#multithreaded-benchmarks)
[CPU Timers](#cpu-timers)
[Manual Timing](#manual-timing)
[Setting the Time Unit](#setting-the-time-unit)
[Random Interleaving](docs/random_interleaving.md)
[User-Requested Performance Counters](docs/perf_counters.md)
[Preventing Optimization](#preventing-optimization)
[Reporting Statistics](#reporting-statistics)
[Custom Statistics](#custom-statistics)
[Using RegisterBenchmark](#using-register-benchmark)
[Exiting with an Error](#exiting-with-an-error)
[A Faster KeepRunning Loop](#a-faster-keep-running-loop)
[Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling)
<a name="output-formats" />
### Output Formats
The library supports multiple output formats. Use the
`--benchmark_format=<console|json|csv>` flag (or set the
`BENCHMARK_FORMAT=<console|json|csv>` environment variable) to set
the format type. `console` is the default format.
The Console format is intended to be a human readable format. By default
the format generates color output. Context is output on stderr and the
tabular data on stdout. Example tabular output looks like:
```
Benchmark Time(ns) CPU(ns) Iterations
----------------------------------------------------------------------
BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s
BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s
BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s
```
The JSON format outputs human readable json split into two top level attributes.
The `context` attribute contains information about the run in general, including
information about the CPU and the date.
The `benchmarks` attribute contains a list of every benchmark run. Example json
output looks like:
```json
{
"context": {
"date": "2015/03/17-18:40:25",
"num_cpus": 40,
"mhz_per_cpu": 2801,
"cpu_scaling_enabled": false,
"build_type": "debug"
},
"benchmarks": [
{
"name": "BM_SetInsert/1024/1",
"iterations": 94877,
"real_time": 29275,
"cpu_time": 29836,
"bytes_per_second": 134066,
"items_per_second": 33516
},
{
"name": "BM_SetInsert/1024/8",
"iterations": 21609,
"real_time": 32317,
"cpu_time": 32429,
"bytes_per_second": 986770,
"items_per_second": 246693
},
{
"name": "BM_SetInsert/1024/10",
"iterations": 21393,
"real_time": 32724,
"cpu_time": 33355,
"bytes_per_second": 1199226,
"items_per_second": 299807
}
]
}
```
The CSV format outputs comma-separated values. The `context` is output on stderr
and the CSV itself on stdout. Example CSV output looks like:
```
name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label
"BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942,
"BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115,
"BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06,
```
<a name="output-files" />
### Output Files
Write benchmark results to a file with the `--benchmark_out=<filename>` option
(or set `BENCHMARK_OUT`). Specify the output format with
`--benchmark_out_format={json|console|csv}` (or set
`BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is
deprecated and the saved `.csv` file
[is not parsable](https://github.com/google/benchmark/issues/794) by csv
parsers.
Specifying `--benchmark_out` does not suppress the console output.
<a name="running-benchmarks" />
### Running Benchmarks
Benchmarks are executed by running the produced binaries. Benchmarks binaries,
by default, accept options that may be specified either through their command
line interface or by setting environment variables before execution. For every
`--option_flag=<value>` CLI switch, a corresponding environment variable
`OPTION_FLAG=<value>` exist and is used as default if set (CLI switches always
prevails). A complete list of CLI options is available running benchmarks
with the `--help` switch.
<a name="running-a-subset-of-benchmarks" />
### Running a Subset of Benchmarks
The `--benchmark_filter=<regex>` option (or `BENCHMARK_FILTER=<regex>`
environment variable) can be used to only run the benchmarks that match
the specified `<regex>`. For example:
```bash
$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32
Run on (1 X 2300 MHz CPU )
2016-06-25 19:34:24
Benchmark Time CPU Iterations
----------------------------------------------------
BM_memcpy/32 11 ns 11 ns 79545455
BM_memcpy/32k 2181 ns 2185 ns 324074
BM_memcpy/32 12 ns 12 ns 54687500
BM_memcpy/32k 1834 ns 1837 ns 357143
```
<a name="result-comparison" />
### Result comparison
It is possible to compare the benchmarking results.
See [Additional Tooling Documentation](docs/tools.md)
<a name="extra-context" />
### Extra Context
Sometimes it's useful to add extra context to the content printed before the
results. By default this section includes information about the CPU on which
the benchmarks are running. If you do want to add more context, you can use
the `benchmark_context` command line flag:
```bash
$ ./run_benchmarks --benchmark_context=pwd=`pwd`
Run on (1 x 2300 MHz CPU)
pwd: /home/user/benchmark/
Benchmark Time CPU Iterations
----------------------------------------------------
BM_memcpy/32 11 ns 11 ns 79545455
BM_memcpy/32k 2181 ns 2185 ns 324074
```
You can get the same effect with the API:
```c++
benchmark::AddCustomContext("foo", "bar");
```
Note that attempts to add a second value with the same key will fail with an
error message.
<a name="runtime-and-reporting-considerations" />
### Runtime and Reporting Considerations
When the benchmark binary is executed, each benchmark function is run serially.
The number of iterations to run is determined dynamically by running the
benchmark a few times and measuring the time taken and ensuring that the
ultimate result will be statistically stable. As such, faster benchmark
functions will be run for more iterations than slower benchmark functions, and
the number of iterations is thus reported.
In all cases, the number of iterations for which the benchmark is run is
governed by the amount of time the benchmark takes. Concretely, the number of
iterations is at least one, not more than 1e9, until CPU time is greater than
the minimum time, or the wallclock time is 5x minimum time. The minimum time is
set per benchmark by calling `MinTime` on the registered benchmark object.
Average timings are then reported over the iterations run. If multiple
repetitions are requested using the `--benchmark_repetitions` command-line
option, or at registration time, the benchmark function will be run several
times and statistical results across these repetitions will also be reported.
As well as the per-benchmark entries, a preamble in the report will include
information about the machine on which the benchmarks are run.
<a name="passing-arguments" />
### Passing Arguments
Sometimes a family of benchmarks can be implemented with just one routine that
takes an extra argument to specify which one of the family of benchmarks to
run. For example, the following code defines a family of benchmarks for
measuring the speed of `memcpy()` calls of different lengths:
```c++
static void BM_memcpy(benchmark::State& state) {
char* src = new char[state.range(0)];
char* dst = new char[state.range(0)];
memset(src, 'x', state.range(0));
for (auto _ : state)
memcpy(dst, src, state.range(0));
state.SetBytesProcessed(int64_t(state.iterations()) *
int64_t(state.range(0)));
delete[] src;
delete[] dst;
}
BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10);
```
The preceding code is quite repetitive, and can be replaced with the following
short-hand. The following invocation will pick a few appropriate arguments in
the specified range and will generate a benchmark for each such argument.
```c++
BENCHMARK(BM_memcpy)->Range(8, 8<<10);
```
By default the arguments in the range are generated in multiples of eight and
the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the
range multiplier is changed to multiples of two.
```c++
BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10);
```
Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ].
The preceding code shows a method of defining a sparse range. The following
example shows a method of defining a dense range. It is then used to benchmark
the performance of `std::vector` initialization for uniformly increasing sizes.
```c++
static void BM_DenseRange(benchmark::State& state) {
for(auto _ : state) {
std::vector<int> v(state.range(0), state.range(0));
benchmark::DoNotOptimize(v.data());
benchmark::ClobberMemory();
}
}
BENCHMARK(BM_DenseRange)->DenseRange(0, 1024, 128);
```
Now arguments generated are [ 0, 128, 256, 384, 512, 640, 768, 896, 1024 ].
You might have a benchmark that depends on two or more inputs. For example, the
following code defines a family of benchmarks for measuring the speed of set
insertion.
```c++
static void BM_SetInsert(benchmark::State& state) {
std::set<int> data;
for (auto _ : state) {
state.PauseTiming();
data = ConstructRandomSet(state.range(0));
state.ResumeTiming();
for (int j = 0; j < state.range(1); ++j)
data.insert(RandomNumber());
}
}
BENCHMARK(BM_SetInsert)
->Args({1<<10, 128})
->Args({2<<10, 128})
->Args({4<<10, 128})
->Args({8<<10, 128})
->Args({1<<10, 512})
->Args({2<<10, 512})
->Args({4<<10, 512})
->Args({8<<10, 512});
```
The preceding code is quite repetitive, and can be replaced with the following
short-hand. The following macro will pick a few appropriate arguments in the
product of the two specified ranges and will generate a benchmark for each such
pair.
```c++
BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}});
```
Some benchmarks may require specific argument values that cannot be expressed
with `Ranges`. In this case, `ArgsProduct` offers the ability to generate a
benchmark input for each combination in the product of the supplied vectors.
```c++
BENCHMARK(BM_SetInsert)
->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}})
// would generate the same benchmark arguments as
BENCHMARK(BM_SetInsert)
->Args({1<<10, 20})
->Args({3<<10, 20})
->Args({8<<10, 20})
->Args({3<<10, 40})
->Args({8<<10, 40})
->Args({1<<10, 40})
->Args({1<<10, 60})
->Args({3<<10, 60})
->Args({8<<10, 60})
->Args({1<<10, 80})
->Args({3<<10, 80})
->Args({8<<10, 80});
```
For more complex patterns of inputs, passing a custom function to `Apply` allows
programmatic specification of an arbitrary set of arguments on which to run the
benchmark. The following example enumerates a dense range on one parameter,
and a sparse range on the second.
```c++
static void CustomArguments(benchmark::internal::Benchmark* b) {
for (int i = 0; i <= 10; ++i)
for (int j = 32; j <= 1024*1024; j *= 8)
b->Args({i, j});
}
BENCHMARK(BM_SetInsert)->Apply(CustomArguments);
```
#### Passing Arbitrary Arguments to a Benchmark
In C++11 it is possible to define a benchmark that takes an arbitrary number
of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)`
macro creates a benchmark that invokes `func` with the `benchmark::State` as
the first argument followed by the specified `args...`.
The `test_case_name` is appended to the name of the benchmark and
should describe the values passed.
```c++
template <class ...ExtraArgs>
void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) {
[...]
}
// Registers a benchmark named "BM_takes_args/int_string_test" that passes
// the specified values to `extra_args`.
BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc"));
```
Note that elements of `...args` may refer to global variables. Users should
avoid modifying global state inside of a benchmark.
<a name="asymptotic-complexity" />
### Calculating Asymptotic Complexity (Big O)
Asymptotic complexity might be calculated for a family of benchmarks. The
following code will calculate the coefficient for the high-order term in the
running time and the normalized root-mean square error of string comparison.
```c++
static void BM_StringCompare(benchmark::State& state) {
std::string s1(state.range(0), '-');
std::string s2(state.range(0), '-');
for (auto _ : state) {
benchmark::DoNotOptimize(s1.compare(s2));
}
state.SetComplexityN(state.range(0));
}
BENCHMARK(BM_StringCompare)
->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN);
```
As shown in the following invocation, asymptotic complexity might also be
calculated automatically.
```c++
BENCHMARK(BM_StringCompare)
->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity();
```
The following code will specify asymptotic complexity with a lambda function,
that might be used to customize high-order term calculation.
```c++
BENCHMARK(BM_StringCompare)->RangeMultiplier(2)
->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; });
```
<a name="custom-benchmark-name" />
### Custom Benchmark Name
You can change the benchmark's name as follows:
```c++
BENCHMARK(BM_memcpy)->Name("memcpy")->RangeMultiplier(2)->Range(8, 8<<10);
```
The invocation will execute the benchmark as before using `BM_memcpy` but changes
the prefix in the report to `memcpy`.
<a name="templated-benchmarks" />
### Templated Benchmarks
This example produces and consumes messages of size `sizeof(v)` `range_x`
times. It also outputs throughput in the absence of multiprogramming.
```c++
template <class Q> void BM_Sequential(benchmark::State& state) {
Q q;
typename Q::value_type v;
for (auto _ : state) {
for (int i = state.range(0); i--; )
q.push(v);
for (int e = state.range(0); e--; )
q.Wait(&v);
}
// actually messages, not bytes:
state.SetBytesProcessed(
static_cast<int64_t>(state.iterations())*state.range(0));
}
BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue<int>)->Range(1<<0, 1<<10);
```
Three macros are provided for adding benchmark templates.
```c++
#ifdef BENCHMARK_HAS_CXX11
#define BENCHMARK_TEMPLATE(func, ...) // Takes any number of parameters.
#else // C++ < C++11
#define BENCHMARK_TEMPLATE(func, arg1)
#endif
#define BENCHMARK_TEMPLATE1(func, arg1)
#define BENCHMARK_TEMPLATE2(func, arg1, arg2)
```
<a name="fixtures" />
### Fixtures
Fixture tests are created by first defining a type that derives from
`::benchmark::Fixture` and then creating/registering the tests using the
following macros:
* `BENCHMARK_F(ClassName, Method)`
* `BENCHMARK_DEFINE_F(ClassName, Method)`
* `BENCHMARK_REGISTER_F(ClassName, Method)`
For Example:
```c++
class MyFixture : public benchmark::Fixture {
public:
void SetUp(const ::benchmark::State& state) {
}
void TearDown(const ::benchmark::State& state) {
}
};
BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) {
for (auto _ : st) {
...
}
}
BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) {
for (auto _ : st) {
...
}
}
/* BarTest is NOT registered */
BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2);
/* BarTest is now registered */
```
#### Templated Fixtures
Also you can create templated fixture by using the following macros:
* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)`
* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)`
For example:
```c++
template<typename T>
class MyFixture : public benchmark::Fixture {};
BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) {
for (auto _ : st) {
...
}
}
BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) {
for (auto _ : st) {
...
}
}
BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2);
```
<a name="custom-counters" />
### Custom Counters
You can add your own counters with user-defined names. The example below
will add columns "Foo", "Bar" and "Baz" in its output:
```c++
static void UserCountersExample1(benchmark::State& state) {
double numFoos = 0, numBars = 0, numBazs = 0;
for (auto _ : state) {
// ... count Foo,Bar,Baz events
}
state.counters["Foo"] = numFoos;
state.counters["Bar"] = numBars;
state.counters["Baz"] = numBazs;
}
```
The `state.counters` object is a `std::map` with `std::string` keys
and `Counter` values. The latter is a `double`-like class, via an implicit
conversion to `double&`. Thus you can use all of the standard arithmetic
assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter.
In multithreaded benchmarks, each counter is set on the calling thread only.
When the benchmark finishes, the counters from each thread will be summed;
the resulting sum is the value which will be shown for the benchmark.
The `Counter` constructor accepts three parameters: the value as a `double`
; a bit flag which allows you to show counters as rates, and/or as per-thread
iteration, and/or as per-thread averages, and/or iteration invariants,
and/or finally inverting the result; and a flag specifying the 'unit' - i.e.
is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024
(`benchmark::Counter::OneK::kIs1024`)?
```c++
// sets a simple counter
state.counters["Foo"] = numFoos;
// Set the counter as a rate. It will be presented divided
// by the duration of the benchmark.
// Meaning: per one second, how many 'foo's are processed?
state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate);
// Set the counter as a rate. It will be presented divided
// by the duration of the benchmark, and the result inverted.
// Meaning: how many seconds it takes to process one 'foo'?
state.counters["FooInvRate"] = Counter(numFoos, benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
// Set the counter as a thread-average quantity. It will
// be presented divided by the number of threads.
state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads);
// There's also a combined flag:
state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate);
// This says that we process with the rate of state.range(0) bytes every iteration:
state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024);
```
When you're compiling in C++11 mode or later you can use `insert()` with
`std::initializer_list`:
```c++
// With C++11, this can be done:
state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}});
// ... instead of:
state.counters["Foo"] = numFoos;
state.counters["Bar"] = numBars;
state.counters["Baz"] = numBazs;
```
#### Counter Reporting
When using the console reporter, by default, user counters are printed at
the end after the table, the same way as ``bytes_processed`` and
``items_processed``. This is best for cases in which there are few counters,
or where there are only a couple of lines per benchmark. Here's an example of
the default output:
```
------------------------------------------------------------------------------
Benchmark Time CPU Iterations UserCounters...
------------------------------------------------------------------------------
BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8
BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m
BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2
BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4
BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8
BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16
BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32
BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4
BM_Factorial 26 ns 26 ns 26608979 40320
BM_Factorial/real_time 26 ns 26 ns 26587936 40320
BM_CalculatePiRange/1 16 ns 16 ns 45704255 0
BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374
BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746
BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355
```
If this doesn't suit you, you can print each counter as a table column by
passing the flag `--benchmark_counters_tabular=true` to the benchmark
application. This is best for cases in which there are a lot of counters, or
a lot of lines per individual benchmark. Note that this will trigger a
reprinting of the table header any time the counter set changes between
individual benchmarks. Here's an example of corresponding output when
`--benchmark_counters_tabular=true` is passed:
```
---------------------------------------------------------------------------------------
Benchmark Time CPU Iterations Bar Bat Baz Foo
---------------------------------------------------------------------------------------
BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8
BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1
BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2
BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4
BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8
BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16
BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32
BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4
--------------------------------------------------------------
Benchmark Time CPU Iterations
--------------------------------------------------------------
BM_Factorial 26 ns 26 ns 26392245 40320
BM_Factorial/real_time 26 ns 26 ns 26494107 40320
BM_CalculatePiRange/1 15 ns 15 ns 45571597 0
BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374
BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746
BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355
BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184
BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162
BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416
BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159
BM_CalculatePi/threads:8 2255 ns 9943 ns 70936
```
Note above the additional header printed when the benchmark changes from
``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does
not have the same counter set as ``BM_UserCounter``.
<a name="multithreaded-benchmarks"/>
### Multithreaded Benchmarks
In a multithreaded test (benchmark invoked by multiple threads simultaneously),
it is guaranteed that none of the threads will start until all have reached
the start of the benchmark loop, and all will have finished before any thread
exits the benchmark loop. (This behavior is also provided by the `KeepRunning()`
API) As such, any global setup or teardown can be wrapped in a check against the thread
index:
```c++
static void BM_MultiThreaded(benchmark::State& state) {
if (state.thread_index == 0) {
// Setup code here.
}
for (auto _ : state) {
// Run the test as normal.
}
if (state.thread_index == 0) {
// Teardown code here.
}
}
BENCHMARK(BM_MultiThreaded)->Threads(2);
```
If the benchmarked code itself uses threads and you want to compare it to
single-threaded code, you may want to use real-time ("wallclock") measurements
for latency comparisons:
```c++
BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime();
```
Without `UseRealTime`, CPU time is used by default.
<a name="cpu-timers" />
### CPU Timers
By default, the CPU timer only measures the time spent by the main thread.
If the benchmark itself uses threads internally, this measurement may not
be what you are looking for. Instead, there is a way to measure the total
CPU usage of the process, by all the threads.
```c++
void callee(int i);
static void MyMain(int size) {
#pragma omp parallel for
for(int i = 0; i < size; i++)
callee(i);
}
static void BM_OpenMP(benchmark::State& state) {
for (auto _ : state)
MyMain(state.range(0));
}
// Measure the time spent by the main thread, use it to decide for how long to
// run the benchmark loop. Depending on the internal implementation detail may
// measure to anywhere from near-zero (the overhead spent before/after work
// handoff to worker thread[s]) to the whole single-thread time.
BENCHMARK(BM_OpenMP)->Range(8, 8<<10);
// Measure the user-visible time, the wall clock (literally, the time that
// has passed on the clock on the wall), use it to decide for how long to
// run the benchmark loop. This will always be meaningful, an will match the
// time spent by the main thread in single-threaded case, in general decreasing
// with the number of internal threads doing the work.
BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime();
// Measure the total CPU consumption, use it to decide for how long to
// run the benchmark loop. This will always measure to no less than the
// time spent by the main thread in single-threaded case.
BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime();
// A mixture of the last two. Measure the total CPU consumption, but use the
// wall clock to decide for how long to run the benchmark loop.
BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime();
```
#### Controlling Timers
Normally, the entire duration of the work loop (`for (auto _ : state) {}`)
is measured. But sometimes, it is necessary to do some work inside of
that loop, every iteration, but without counting that time to the benchmark time.
That is possible, although it is not recommended, since it has high overhead.
```c++
static void BM_SetInsert_With_Timer_Control(benchmark::State& state) {
std::set<int> data;
for (auto _ : state) {
state.PauseTiming(); // Stop timers. They will not count until they are resumed.
data = ConstructRandomSet(state.range(0)); // Do something that should not be measured
state.ResumeTiming(); // And resume timers. They are now counting again.
// The rest will be measured.
for (int j = 0; j < state.range(1); ++j)
data.insert(RandomNumber());
}
}
BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}});
```
<a name="manual-timing" />
### Manual Timing
For benchmarking something for which neither CPU time nor real-time are
correct or accurate enough, completely manual timing is supported using
the `UseManualTime` function.
When `UseManualTime` is used, the benchmarked code must call
`SetIterationTime` once per iteration of the benchmark loop to
report the manually measured time.
An example use case for this is benchmarking GPU execution (e.g. OpenCL
or CUDA kernels, OpenGL or Vulkan or Direct3D draw calls), which cannot
be accurately measured using CPU time or real-time. Instead, they can be
measured accurately using a dedicated API, and these measurement results
can be reported back with `SetIterationTime`.
```c++
static void BM_ManualTiming(benchmark::State& state) {
int microseconds = state.range(0);
std::chrono::duration<double, std::micro> sleep_duration {
static_cast<double>(microseconds)
};
for (auto _ : state) {
auto start = std::chrono::high_resolution_clock::now();
// Simulate some useful workload with a sleep
std::this_thread::sleep_for(sleep_duration);
auto end = std::chrono::high_resolution_clock::now();
auto elapsed_seconds =
std::chrono::duration_cast<std::chrono::duration<double>>(
end - start);
state.SetIterationTime(elapsed_seconds.count());
}
}
BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime();
```
<a name="setting-the-time-unit" />
### Setting the Time Unit
If a benchmark runs a few milliseconds it may be hard to visually compare the
measured times, since the output data is given in nanoseconds per default. In
order to manually set the time unit, you can specify it manually:
```c++
BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
```
<a name="preventing-optimization" />
### Preventing Optimization
To prevent a value or expression from being optimized away by the compiler
the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()`
functions can be used.
```c++
static void BM_test(benchmark::State& state) {
for (auto _ : state) {
int x = 0;
for (int i=0; i < 64; ++i) {
benchmark::DoNotOptimize(x += i);
}
}
}
```
`DoNotOptimize(<expr>)` forces the *result* of `<expr>` to be stored in either
memory or a register. For GNU based compilers it acts as read/write barrier
for global memory. More specifically it forces the compiler to flush pending
writes to memory and reload any other values as necessary.
Note that `DoNotOptimize(<expr>)` does not prevent optimizations on `<expr>`
in any way. `<expr>` may even be removed entirely when the result is already
known. For example:
```c++
/* Example 1: `<expr>` is removed entirely. */
int foo(int x) { return x + 42; }
while (...) DoNotOptimize(foo(0)); // Optimized to DoNotOptimize(42);
/* Example 2: Result of '<expr>' is only reused */
int bar(int) __attribute__((const));
while (...) DoNotOptimize(bar(0)); // Optimized to:
// int __result__ = bar(0);
// while (...) DoNotOptimize(__result__);
```
The second tool for preventing optimizations is `ClobberMemory()`. In essence
`ClobberMemory()` forces the compiler to perform all pending writes to global
memory. Memory managed by block scope objects must be "escaped" using
`DoNotOptimize(...)` before it can be clobbered. In the below example
`ClobberMemory()` prevents the call to `v.push_back(42)` from being optimized
away.
```c++
static void BM_vector_push_back(benchmark::State& state) {
for (auto _ : state) {
std::vector<int> v;
v.reserve(1);
benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered.
v.push_back(42);
benchmark::ClobberMemory(); // Force 42 to be written to memory.
}
}
```
Note that `ClobberMemory()` is only available for GNU or MSVC based compilers.
<a name="reporting-statistics" />
### Statistics: Reporting the Mean, Median and Standard Deviation of Repeated Benchmarks
By default each benchmark is run once and that single result is reported.
However benchmarks are often noisy and a single result may not be representative
of the overall behavior. For this reason it's possible to repeatedly rerun the
benchmark.
The number of runs of each benchmark is specified globally by the
`--benchmark_repetitions` flag or on a per benchmark basis by calling
`Repetitions` on the registered benchmark object. When a benchmark is run more
than once the mean, median and standard deviation of the runs will be reported.
Additionally the `--benchmark_report_aggregates_only={true|false}`,
`--benchmark_display_aggregates_only={true|false}` flags or
`ReportAggregatesOnly(bool)`, `DisplayAggregatesOnly(bool)` functions can be
used to change how repeated tests are reported. By default the result of each
repeated run is reported. When `report aggregates only` option is `true`,
only the aggregates (i.e. mean, median and standard deviation, maybe complexity
measurements if they were requested) of the runs is reported, to both the
reporters - standard output (console), and the file.
However when only the `display aggregates only` option is `true`,
only the aggregates are displayed in the standard output, while the file
output still contains everything.
Calling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a
registered benchmark object overrides the value of the appropriate flag for that
benchmark.
<a name="custom-statistics" />
### Custom Statistics
While having mean, median and standard deviation is nice, this may not be
enough for everyone. For example you may want to know what the largest
observation is, e.g. because you have some real-time constraints. This is easy.
The following code will specify a custom statistic to be calculated, defined
by a lambda function.
```c++
void BM_spin_empty(benchmark::State& state) {
for (auto _ : state) {
for (int x = 0; x < state.range(0); ++x) {
benchmark::DoNotOptimize(x);
}
}
}
BENCHMARK(BM_spin_empty)
->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
return *(std::max_element(std::begin(v), std::end(v)));
})
->Arg(512);
```
<a name="using-register-benchmark" />
### Using RegisterBenchmark(name, fn, args...)
The `RegisterBenchmark(name, func, args...)` function provides an alternative
way to create and register benchmarks.
`RegisterBenchmark(name, func, args...)` creates, registers, and returns a
pointer to a new benchmark with the specified `name` that invokes
`func(st, args...)` where `st` is a `benchmark::State` object.
Unlike the `BENCHMARK` registration macros, which can only be used at the global
scope, the `RegisterBenchmark` can be called anywhere. This allows for
benchmark tests to be registered programmatically.
Additionally `RegisterBenchmark` allows any callable object to be registered
as a benchmark. Including capturing lambdas and function objects.
For Example:
```c++
auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ };
int main(int argc, char** argv) {
for (auto& test_input : { /* ... */ })
benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input);
benchmark::Initialize(&argc, argv);
benchmark::RunSpecifiedBenchmarks();
benchmark::Shutdown();
}
```
<a name="exiting-with-an-error" />
### Exiting with an Error
When errors caused by external influences, such as file I/O and network
communication, occur within a benchmark the
`State::SkipWithError(const char* msg)` function can be used to skip that run
of benchmark and report the error. Note that only future iterations of the
`KeepRunning()` are skipped. For the ranged-for version of the benchmark loop
Users must explicitly exit the loop, otherwise all iterations will be performed.
Users may explicitly return to exit the benchmark immediately.
The `SkipWithError(...)` function may be used at any point within the benchmark,
including before and after the benchmark loop. Moreover, if `SkipWithError(...)`
has been used, it is not required to reach the benchmark loop and one may return
from the benchmark function early.
For example:
```c++
static void BM_test(benchmark::State& state) {
auto resource = GetResource();
if (!resource.good()) {
state.SkipWithError("Resource is not good!");
// KeepRunning() loop will not be entered.
}
while (state.KeepRunning()) {
auto data = resource.read_data();
if (!resource.good()) {
state.SkipWithError("Failed to read data!");
break; // Needed to skip the rest of the iteration.
}
do_stuff(data);
}
}
static void BM_test_ranged_fo(benchmark::State & state) {
auto resource = GetResource();
if (!resource.good()) {
state.SkipWithError("Resource is not good!");
return; // Early return is allowed when SkipWithError() has been used.
}
for (auto _ : state) {
auto data = resource.read_data();
if (!resource.good()) {
state.SkipWithError("Failed to read data!");
break; // REQUIRED to prevent all further iterations.
}
do_stuff(data);
}
}
```
<a name="a-faster-keep-running-loop" />
### A Faster KeepRunning Loop
In C++11 mode, a ranged-based for loop should be used in preference to
the `KeepRunning` loop for running the benchmarks. For example:
```c++
static void BM_Fast(benchmark::State &state) {
for (auto _ : state) {
FastOperation();
}
}
BENCHMARK(BM_Fast);
```
The reason the ranged-for loop is faster than using `KeepRunning`, is
because `KeepRunning` requires a memory load and store of the iteration count
ever iteration, whereas the ranged-for variant is able to keep the iteration count
in a register.
For example, an empty inner loop of using the ranged-based for method looks like:
```asm
# Loop Init
mov rbx, qword ptr [r14 + 104]
call benchmark::State::StartKeepRunning()
test rbx, rbx
je .LoopEnd
.LoopHeader: # =>This Inner Loop Header: Depth=1
add rbx, -1
jne .LoopHeader
.LoopEnd:
```
Compared to an empty `KeepRunning` loop, which looks like:
```asm
.LoopHeader: # in Loop: Header=BB0_3 Depth=1
cmp byte ptr [rbx], 1
jne .LoopInit
.LoopBody: # =>This Inner Loop Header: Depth=1
mov rax, qword ptr [rbx + 8]
lea rcx, [rax + 1]
mov qword ptr [rbx + 8], rcx
cmp rax, qword ptr [rbx + 104]
jb .LoopHeader
jmp .LoopEnd
.LoopInit:
mov rdi, rbx
call benchmark::State::StartKeepRunning()
jmp .LoopBody
.LoopEnd:
```
Unless C++03 compatibility is required, the ranged-for variant of writing
the benchmark loop should be preferred.
<a name="disabling-cpu-frequency-scaling" />
### Disabling CPU Frequency Scaling
If you see this error:
```
***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead.
```
you might want to disable the CPU frequency scaling while running the benchmark:
```bash
sudo cpupower frequency-set --governor performance
./mybench
sudo cpupower frequency-set --governor powersave
```

third-party/benchmark/WORKSPACE

workspace(name = "com_github_google_benchmark") workspace(name = "com_github_google_benchmark")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
http_archive(
name = "rules_cc",
strip_prefix = "rules_cc-a508235df92e71d537fcbae0c7c952ea6957a912",
urls = ["https://github.com/bazelbuild/rules_cc/archive/a508235df92e71d537fcbae0c7c952ea6957a912.zip"],
sha256 = "d7dc12c1d5bc1a87474de8e3d17b7731a4dcebcfb8aa3990fe8ac7734ef12f2f",
)
http_archive( http_archive(
name = "com_google_absl", name = "com_google_absl",
sha256 = "f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111", sha256 = "f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111",
strip_prefix = "abseil-cpp-20200225.2", strip_prefix = "abseil-cpp-20200225.2",
urls = ["https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz"], urls = ["https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz"],
) )
http_archive( git_repository(
name = "com_google_googletest", name = "com_google_googletest",
strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", remote = "https://github.com/google/googletest.git",
urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], tag = "release-1.11.0",
sha256 = "8f827dd550db8b4fdf73904690df0be9fccc161017c9038a724bc9a0617a1bc8",
) )
http_archive( http_archive(
name = "pybind11", name = "pybind11",
build_file = "@//bindings/python:pybind11.BUILD", build_file = "@//bindings/python:pybind11.BUILD",
sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d", sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d",
strip_prefix = "pybind11-2.4.3", strip_prefix = "pybind11-2.4.3",
urls = ["https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"], urls = ["https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"],
Show All 20 Lines

third-party/benchmark/bindings/python/google_benchmark/benchmark.cc

Show First 20 Lines • Show All 159 Lines • ▼ Show 20 Lines py::class_<State>(m, "State")
.def_property_readonly("error_occurred", &State::error_occurred) .def_property_readonly("error_occurred", &State::error_occurred)
.def("set_iteration_time", &State::SetIterationTime) .def("set_iteration_time", &State::SetIterationTime)
.def_property("bytes_processed", &State::bytes_processed, .def_property("bytes_processed", &State::bytes_processed,
&State::SetBytesProcessed) &State::SetBytesProcessed)
.def_property("complexity_n", &State::complexity_length_n, .def_property("complexity_n", &State::complexity_length_n,
&State::SetComplexityN) &State::SetComplexityN)
.def_property("items_processed", &State::items_processed, .def_property("items_processed", &State::items_processed,
&State::SetItemsProcessed) &State::SetItemsProcessed)
.def("set_label", (void (State::*)(const char*)) & State::SetLabel) .def("set_label", (void(State::*)(const char*)) & State::SetLabel)
.def("range", &State::range, py::arg("pos") = 0) .def("range", &State::range, py::arg("pos") = 0)
.def_property_readonly("iterations", &State::iterations) .def_property_readonly("iterations", &State::iterations)
.def_readwrite("counters", &State::counters) .def_readwrite("counters", &State::counters)
.def_readonly("thread_index", &State::thread_index) .def_property_readonly("thread_index", &State::thread_index)
.def_readonly("threads", &State::threads); .def_property_readonly("threads", &State::threads);
m.def("Initialize", Initialize); m.def("Initialize", Initialize);
m.def("RegisterBenchmark", RegisterBenchmark, m.def("RegisterBenchmark", RegisterBenchmark,
py::return_value_policy::reference); py::return_value_policy::reference);
m.def("RunSpecifiedBenchmarks", m.def("RunSpecifiedBenchmarks",
[]() { benchmark::RunSpecifiedBenchmarks(); }); []() { benchmark::RunSpecifiedBenchmarks(); });
}; };
} // namespace } // namespace

third-party/benchmark/bindings/python/google_benchmark/example.py

Show First 20 Lines • Show All 96 Lines • ▼ Show 20 Lines
@benchmark.option.use_real_time() @benchmark.option.use_real_time()
def with_options(state): def with_options(state):
while state: while state:
sum(range(1_000_000)) sum(range(1_000_000))
@benchmark.register(name="sum_million_microseconds") @benchmark.register(name="sum_million_microseconds")
@benchmark.option.unit(benchmark.kMicrosecond) @benchmark.option.unit(benchmark.kMicrosecond)
def with_options(state): def with_options2(state):
while state: while state:
sum(range(1_000_000)) sum(range(1_000_000))
@benchmark.register @benchmark.register
@benchmark.option.arg(100) @benchmark.option.arg(100)
@benchmark.option.arg(1000) @benchmark.option.arg(1000)
def passing_argument(state): def passing_argument(state):
Show All 23 Lines

third-party/benchmark/cmake/Config.cmake.in

@PACKAGE_INIT@
include (CMakeFindDependencyMacro)
find_dependency (Threads)
include("${CMAKE_CURRENT_LIST_DIR}/@targets_export_name@.cmake") include("${CMAKE_CURRENT_LIST_DIR}/@targets_export_name@.cmake")

third-party/benchmark/cmake/GoogleTest.cmake

Show All 23 Lines
endif() endif()
# Prevent overriding the parent project's compiler/linker # Prevent overriding the parent project's compiler/linker
# settings on Windows # settings on Windows
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
include(${GOOGLETEST_PREFIX}/googletest-paths.cmake) include(${GOOGLETEST_PREFIX}/googletest-paths.cmake)
# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves.
add_compile_options(-w)
# Add googletest directly to our build. This defines # Add googletest directly to our build. This defines
# the gtest and gtest_main targets. # the gtest and gtest_main targets.
add_subdirectory(${GOOGLETEST_SOURCE_DIR} add_subdirectory(${GOOGLETEST_SOURCE_DIR}
${GOOGLETEST_BINARY_DIR} ${GOOGLETEST_BINARY_DIR}
EXCLUDE_FROM_ALL) EXCLUDE_FROM_ALL)
set_target_properties(gtest PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES>) if(NOT DEFINED GTEST_COMPILE_COMMANDS)
set_target_properties(gtest_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gtest_main,INTERFACE_INCLUDE_DIRECTORIES>) set(GTEST_COMPILE_COMMANDS ON)
set_target_properties(gmock PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gmock,INTERFACE_INCLUDE_DIRECTORIES>) endif()
set_target_properties(gmock_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gmock_main,INTERFACE_INCLUDE_DIRECTORIES>)
set_target_properties(gtest PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES> EXPORT_COMPILE_COMMANDS ${GTEST_COMPILE_COMMANDS})
set_target_properties(gtest_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gtest_main,INTERFACE_INCLUDE_DIRECTORIES> EXPORT_COMPILE_COMMANDS ${GTEST_COMPILE_COMMANDS})
set_target_properties(gmock PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gmock,INTERFACE_INCLUDE_DIRECTORIES> EXPORT_COMPILE_COMMANDS ${GTEST_COMPILE_COMMANDS})
set_target_properties(gmock_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gmock_main,INTERFACE_INCLUDE_DIRECTORIES> EXPORT_COMPILE_COMMANDS ${GTEST_COMPILE_COMMANDS})

third-party/benchmark/cmake/GoogleTest.cmake.in

Show All 25 Lines ExternalProject_Add(
BINARY_DIR "${CMAKE_BINARY_DIR}/build" BINARY_DIR "${CMAKE_BINARY_DIR}/build"
CONFIGURE_COMMAND "" CONFIGURE_COMMAND ""
BUILD_COMMAND "" BUILD_COMMAND ""
INSTALL_COMMAND "" INSTALL_COMMAND ""
TEST_COMMAND "" TEST_COMMAND ""
) )
else() else()
if(NOT ALLOW_DOWNLOADING_GOOGLETEST) if(NOT ALLOW_DOWNLOADING_GOOGLETEST)
message(SEND_ERROR "Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable BENCHMARK_DOWNLOAD_DEPENDENCIES, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.") message(SEND_ERROR "Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable BENCHMARK_DOWNLOAD_DEPENDENCIES, or disable BENCHMARK_USE_BUNDLED_GTEST, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.")
return()
else() else()
message(WARNING "Did not find Google Test sources! Fetching from web...") message(WARNING "Did not find Google Test sources! Fetching from web...")
ExternalProject_Add( ExternalProject_Add(
googletest googletest
GIT_REPOSITORY https://github.com/google/googletest.git GIT_REPOSITORY https://github.com/google/googletest.git
GIT_TAG master GIT_TAG "release-1.11.0"
PREFIX "${CMAKE_BINARY_DIR}" PREFIX "${CMAKE_BINARY_DIR}"
STAMP_DIR "${CMAKE_BINARY_DIR}/stamp" STAMP_DIR "${CMAKE_BINARY_DIR}/stamp"
DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download" DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download"
SOURCE_DIR "${CMAKE_BINARY_DIR}/src" SOURCE_DIR "${CMAKE_BINARY_DIR}/src"
BINARY_DIR "${CMAKE_BINARY_DIR}/build" BINARY_DIR "${CMAKE_BINARY_DIR}/build"
CONFIGURE_COMMAND "" CONFIGURE_COMMAND ""
BUILD_COMMAND "" BUILD_COMMAND ""
INSTALL_COMMAND "" INSTALL_COMMAND ""
Show All 10 Lines

third-party/benchmark/cmake/Modules/FindLLVMAr.cmake

  • This file was added.
include(FeatureSummary)
find_program(LLVMAR_EXECUTABLE
NAMES llvm-ar
DOC "The llvm-ar executable"
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(LLVMAr
DEFAULT_MSG
LLVMAR_EXECUTABLE)
SET_PACKAGE_PROPERTIES(LLVMAr PROPERTIES
URL https://llvm.org/docs/CommandGuide/llvm-ar.html
DESCRIPTION "create, modify, and extract from archives"
)

third-party/benchmark/cmake/Modules/FindLLVMNm.cmake

  • This file was added.
include(FeatureSummary)
find_program(LLVMNM_EXECUTABLE
NAMES llvm-nm
DOC "The llvm-nm executable"
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(LLVMNm
DEFAULT_MSG
LLVMNM_EXECUTABLE)
SET_PACKAGE_PROPERTIES(LLVMNm PROPERTIES
URL https://llvm.org/docs/CommandGuide/llvm-nm.html
DESCRIPTION "list LLVM bitcode and object file’s symbol table"
)

third-party/benchmark/cmake/Modules/FindLLVMRanLib.cmake

  • This file was added.
include(FeatureSummary)
find_program(LLVMRANLIB_EXECUTABLE
NAMES llvm-ranlib
DOC "The llvm-ranlib executable"
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(LLVMRanLib
DEFAULT_MSG
LLVMRANLIB_EXECUTABLE)
SET_PACKAGE_PROPERTIES(LLVMRanLib PROPERTIES
DESCRIPTION "generate index for LLVM archive"
)

third-party/benchmark/cmake/Modules/FindPFM.cmake

  • This file was added.
# If successful, the following variables will be defined:
# HAVE_LIBPFM.
# Set BENCHMARK_ENABLE_LIBPFM to 0 to disable, regardless of libpfm presence.
include(CheckIncludeFile)
include(CheckLibraryExists)
include(FeatureSummary)
enable_language(C)
set_package_properties(PFM PROPERTIES
URL http://perfmon2.sourceforge.net/
DESCRIPTION "a helper library to develop monitoring tools"
PURPOSE "Used to program specific performance monitoring events")
check_library_exists(libpfm.a pfm_initialize "" HAVE_LIBPFM_INITIALIZE)
if(HAVE_LIBPFM_INITIALIZE)
check_include_file(perfmon/perf_event.h HAVE_PERFMON_PERF_EVENT_H)
check_include_file(perfmon/pfmlib.h HAVE_PERFMON_PFMLIB_H)
check_include_file(perfmon/pfmlib_perf_event.h HAVE_PERFMON_PFMLIB_PERF_EVENT_H)
if(HAVE_PERFMON_PERF_EVENT_H AND HAVE_PERFMON_PFMLIB_H AND HAVE_PERFMON_PFMLIB_PERF_EVENT_H)
message("Using Perf Counters.")
set(HAVE_LIBPFM 1)
set(PFM_FOUND 1)
endif()
else()
message("Perf Counters support requested, but was unable to find libpfm.")
endif()

third-party/benchmark/dependencies.md

  • This file was moved to third-party/benchmark/docs/dependencies.md.

third-party/benchmark/docs/_config.yml

theme: jekyll-theme-hacker theme: jekyll-theme-minimal
No newline at end of file No newline at end of file

third-party/benchmark/docs/dependencies.md

  • This file was moved from third-party/benchmark/dependencies.md.
# Build tool dependency policy # Build tool dependency policy
To ensure the broadest compatibility when building the benchmark library, but To ensure the broadest compatibility when building the benchmark library, but
still allow forward progress, we require any build tooling to be available for: still allow forward progress, we require any build tooling to be available for:
* Debian stable AND * Debian stable _and_
* The last two Ubuntu LTS releases AND * The last two Ubuntu LTS releases
Currently, this means using build tool versions that are available for Ubuntu Currently, this means using build tool versions that are available for Ubuntu
16.04 (Xenial), Ubuntu 18.04 (Bionic), and Debian stretch. 18.04 (Bionic Beaver), Ubuntu 20.04 (Focal Fossa), and Debian 11 (bullseye).
_Note, [travis](.travis.yml) runs under Ubuntu 14.04 (Trusty) for linux builds._ _Note, CI also runs ubuntu-16.04 and ubuntu-14.04 to ensure best effort support
for older versions._
## cmake ## cmake
The current supported version is cmake 3.5.1 as of 2018-06-06. The current supported version is cmake 3.5.1 as of 2018-06-06.
_Note, this version is also available for Ubuntu 14.04, the previous Ubuntu LTS _Note, this version is also available for Ubuntu 14.04, an older Ubuntu LTS
release, as `cmake3`._ release, as `cmake3`._

third-party/benchmark/docs/index.md

  • This file was added.
# Benchmark
* [Assembly Tests](AssemblyTests.md)
* [Dependencies](dependencies.md)
* [Perf Counters](perf_counters.md)
* [Platform Specific Build Instructions](platform_specific_build_instructions.md)
* [Random Interleaving](random_interleaving.md)
* [Releasing](releasing.md)
* [Tools](tools.md)
* [User Guide](user_guide.md)
No newline at end of file

third-party/benchmark/docs/platform_specific_build_instructions.md

  • This file was added.
# Platform Specific Build Instructions
## Building with GCC
When the library is built using GCC it is necessary to link with the pthread
library due to how GCC implements `std::thread`. Failing to link to pthread will
lead to runtime exceptions (unless you're using libc++), not linker errors. See
[issue #67](https://github.com/google/benchmark/issues/67) for more details. You
can link to pthread by adding `-pthread` to your linker command. Note, you can
also use `-lpthread`, but there are potential issues with ordering of command
line parameters if you use that.
On QNX, the pthread library is part of libc and usually included automatically
(see
[`pthread_create()`](https://www.qnx.com/developers/docs/7.1/index.html#com.qnx.doc.neutrino.lib_ref/topic/p/pthread_create.html)).
There's no separate pthread library to link.
## Building with Visual Studio 2015 or 2017
The `shlwapi` library (`-lshlwapi`) is required to support a call to `CPUInfo` which reads the registry. Either add `shlwapi.lib` under `[ Configuration Properties > Linker > Input ]`, or use the following:
```
// Alternatively, can add libraries using linker options.
#ifdef _WIN32
#pragma comment ( lib, "Shlwapi.lib" )
#ifdef _DEBUG
#pragma comment ( lib, "benchmarkd.lib" )
#else
#pragma comment ( lib, "benchmark.lib" )
#endif
#endif
```
Can also use the graphical version of CMake:
* Open `CMake GUI`.
* Under `Where to build the binaries`, same path as source plus `build`.
* Under `CMAKE_INSTALL_PREFIX`, same path as source plus `install`.
* Click `Configure`, `Generate`, `Open Project`.
* If build fails, try deleting entire directory and starting again, or unticking options to build less.
## Building with Intel 2015 Update 1 or Intel System Studio Update 4
See instructions for building with Visual Studio. Once built, right click on the solution and change the build to Intel.
## Building on Solaris
If you're running benchmarks on solaris, you'll want the kstat library linked in
too (`-lkstat`).
No newline at end of file

third-party/benchmark/docs/releasing.md

# How to release # How to release
* Make sure you're on main and synced to HEAD * Make sure you're on main and synced to HEAD
* Ensure the project builds and tests run (sanity check only, obviously) * Ensure the project builds and tests run (sanity check only, obviously)
* `parallel -j0 exec ::: test/*_test` can help ensure everything at least * `parallel -j0 exec ::: test/*_test` can help ensure everything at least
passes passes
* Prepare release notes * Prepare release notes
* `git log $(git describe --abbrev=0 --tags)..HEAD` gives you the list of * `git log $(git describe --abbrev=0 --tags)..HEAD` gives you the list of
commits between the last annotated tag and HEAD commits between the last annotated tag and HEAD
* Pick the most interesting. * Pick the most interesting.
* Create one last commit that updates the version saved in `CMakeLists.txt` to the release version you're creating. (This version will be used if benchmark is installed from the archive you'll be creating in the next step.) * Create one last commit that updates the version saved in `CMakeLists.txt` and the
`__version__` variable in `bindings/python/google_benchmark/__init__.py`to the release
version you're creating. (This version will be used if benchmark is installed from the
archive you'll be creating in the next step.)
``` ```
project (benchmark VERSION 1.5.3 LANGUAGES CXX) project (benchmark VERSION 1.6.0 LANGUAGES CXX)
```
```python
# bindings/python/google_benchmark/__init__.py
# ...
__version__ = "1.6.0" # <-- change this to the release version you are creating
# ...
``` ```
* Create a release through github's interface * Create a release through github's interface
* Note this will create a lightweight tag. * Note this will create a lightweight tag.
* Update this to an annotated tag: * Update this to an annotated tag:
* `git pull --tags` * `git pull --tags`
* `git tag -a -f <tag> <tag>` * `git tag -a -f <tag> <tag>`
* `git push --force origin` * `git push --force --tags origin`

third-party/benchmark/docs/user_guide.md

  • This file was copied from third-party/benchmark/README.md.
# Benchmark # User Guide
[![build-and-test](https://github.com/google/benchmark/workflows/build-and-test/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Abuild-and-test) ## Command Line
[![bazel](https://github.com/google/benchmark/actions/workflows/bazel.yml/badge.svg)](https://github.com/google/benchmark/actions/workflows/bazel.yml)
[![pylint](https://github.com/google/benchmark/workflows/pylint/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Apylint)
[![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings)
[![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark)
[![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark)
A library to benchmark code snippets, similar to unit tests. Example:
```c++
#include <benchmark/benchmark.h>
static void BM_SomeFunction(benchmark::State& state) {
// Perform setup here
for (auto _ : state) {
// This code gets timed
SomeFunction();
}
}
// Register the function as a benchmark
BENCHMARK(BM_SomeFunction);
// Run the benchmark
BENCHMARK_MAIN();
```
To get started, see [Requirements](#requirements) and
[Installation](#installation). See [Usage](#usage) for a full example and the
[User Guide](#user-guide) for a more comprehensive feature overview.
It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/docs/primer.md)
as some of the structural aspects of the APIs are similar.
### Resources
[Discussion group](https://groups.google.com/d/forum/benchmark-discuss)
IRC channels:
* [libera](https://libera.chat) #benchmark
[Additional Tooling Documentation](docs/tools.md)
[Assembly Testing Documentation](docs/AssemblyTests.md)
## Requirements
The library can be used with C++03. However, it requires C++11 to build,
including compiler and standard library support.
The following minimum versions are required to build the library:
* GCC 4.8
* Clang 3.4
* Visual Studio 14 2015
* Intel 2015 Update 1
See [Platform-Specific Build Instructions](#platform-specific-build-instructions).
## Installation
This describes the installation process using cmake. As pre-requisites, you'll
need git and cmake installed.
_See [dependencies.md](dependencies.md) for more details regarding supported
versions of build tools._
```bash
# Check out the library.
$ git clone https://github.com/google/benchmark.git
# Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory.
$ git clone https://github.com/google/googletest.git benchmark/googletest
# Go to the library root directory
$ cd benchmark
# Make a build directory to place the build output.
$ cmake -E make_directory "build"
# Generate build system files with cmake.
$ cmake -E chdir "build" cmake -DCMAKE_BUILD_TYPE=Release ../
# or, starting with CMake 3.13, use a simpler form:
# cmake -DCMAKE_BUILD_TYPE=Release -S . -B "build"
# Build the library.
$ cmake --build "build" --config Release
```
This builds the `benchmark` and `benchmark_main` libraries and tests.
On a unix system, the build directory should now look something like this:
```
/benchmark
/build
/src
/libbenchmark.a
/libbenchmark_main.a
/test
...
```
Next, you can run the tests to check the build.
```bash
$ cmake -E chdir "build" ctest --build-config Release
```
If you want to install the library globally, also run:
```
sudo cmake --build "build" --config Release --target install
```
Note that Google Benchmark requires Google Test to build and run the tests. This
dependency can be provided two ways:
* Checkout the Google Test sources into `benchmark/googletest` as above.
* Otherwise, if `-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON` is specified during
configuration, the library will automatically download and build any required
dependencies.
If you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF`
to `CMAKE_ARGS`.
### Debug vs Release
By default, benchmark builds as a debug library. You will see a warning in the
output when this is the case. To build it as a release library instead, add
`-DCMAKE_BUILD_TYPE=Release` when generating the build system files, as shown
above. The use of `--config Release` in build commands is needed to properly
support multi-configuration tools (like Visual Studio for example) and can be
skipped for other build systems (like Makefile).
To enable link-time optimisation, also add `-DBENCHMARK_ENABLE_LTO=true` when
generating the build system files.
If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake
cache variables, if autodetection fails.
If you are using clang, you may need to set `LLVMAR_EXECUTABLE`,
`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables.
### Stable and Experimental Library Versions
The main branch contains the latest stable version of the benchmarking library;
the API of which can be considered largely stable, with source breaking changes
being made only upon the release of a new major version.
Newer, experimental, features are implemented and tested on the
[`v2` branch](https://github.com/google/benchmark/tree/v2). Users who wish
to use, test, and provide feedback on the new features are encouraged to try
this branch. However, this branch provides no stability guarantees and reserves
the right to change and break the API at any time.
## Usage
### Basic usage
Define a function that executes the code to measure, register it as a benchmark
function using the `BENCHMARK` macro, and ensure an appropriate `main` function
is available:
```c++
#include <benchmark/benchmark.h>
static void BM_StringCreation(benchmark::State& state) {
for (auto _ : state)
std::string empty_string;
}
// Register the function as a benchmark
BENCHMARK(BM_StringCreation);
// Define another benchmark
static void BM_StringCopy(benchmark::State& state) {
std::string x = "hello";
for (auto _ : state)
std::string copy(x);
}
BENCHMARK(BM_StringCopy);
BENCHMARK_MAIN();
```
To run the benchmark, compile and link against the `benchmark` library
(libbenchmark.a/.so). If you followed the build steps above, this library will
be under the build directory you created.
```bash
# Example on linux after running the build steps above. Assumes the
# `benchmark` and `build` directories are under the current directory.
$ g++ mybenchmark.cc -std=c++11 -isystem benchmark/include \
-Lbenchmark/build/src -lbenchmark -lpthread -o mybenchmark
```
Alternatively, link against the `benchmark_main` library and remove
`BENCHMARK_MAIN();` above to get the same behavior.
The compiled executable will run all benchmarks by default. Pass the `--help`
flag for option information or see the guide below.
### Usage with CMake
If using CMake, it is recommended to link against the project-provided
`benchmark::benchmark` and `benchmark::benchmark_main` targets using
`target_link_libraries`.
It is possible to use ```find_package``` to import an installed version of the
library.
```cmake
find_package(benchmark REQUIRED)
```
Alternatively, ```add_subdirectory``` will incorporate the library directly in
to one's CMake project.
```cmake
add_subdirectory(benchmark)
```
Either way, link to the library as follows.
```cmake
target_link_libraries(MyTarget benchmark::benchmark)
```
## Platform Specific Build Instructions
### Building with GCC
When the library is built using GCC it is necessary to link with the pthread
library due to how GCC implements `std::thread`. Failing to link to pthread will
lead to runtime exceptions (unless you're using libc++), not linker errors. See
[issue #67](https://github.com/google/benchmark/issues/67) for more details. You
can link to pthread by adding `-pthread` to your linker command. Note, you can
also use `-lpthread`, but there are potential issues with ordering of command
line parameters if you use that.
### Building with Visual Studio 2015 or 2017
The `shlwapi` library (`-lshlwapi`) is required to support a call to `CPUInfo` which reads the registry. Either add `shlwapi.lib` under `[ Configuration Properties > Linker > Input ]`, or use the following:
```
// Alternatively, can add libraries using linker options.
#ifdef _WIN32
#pragma comment ( lib, "Shlwapi.lib" )
#ifdef _DEBUG
#pragma comment ( lib, "benchmarkd.lib" )
#else
#pragma comment ( lib, "benchmark.lib" )
#endif
#endif
```
Can also use the graphical version of CMake:
* Open `CMake GUI`.
* Under `Where to build the binaries`, same path as source plus `build`.
* Under `CMAKE_INSTALL_PREFIX`, same path as source plus `install`.
* Click `Configure`, `Generate`, `Open Project`.
* If build fails, try deleting entire directory and starting again, or unticking options to build less.
### Building with Intel 2015 Update 1 or Intel System Studio Update 4
See instructions for building with Visual Studio. Once built, right click on the solution and change the build to Intel.
### Building on Solaris
If you're running benchmarks on solaris, you'll want the kstat library linked in
too (`-lkstat`).
## User Guide
### Command Line
[Output Formats](#output-formats) [Output Formats](#output-formats)
[Output Files](#output-files) [Output Files](#output-files)
[Running Benchmarks](#running-benchmarks) [Running Benchmarks](#running-benchmarks)
[Running a Subset of Benchmarks](#running-a-subset-of-benchmarks) [Running a Subset of Benchmarks](#running-a-subset-of-benchmarks)
[Result Comparison](#result-comparison) [Result Comparison](#result-comparison)
[Extra Context](#extra-context) [Extra Context](#extra-context)
### Library ## Library
[Runtime and Reporting Considerations](#runtime-and-reporting-considerations) [Runtime and Reporting Considerations](#runtime-and-reporting-considerations)
[Setup/Teardown](#setupteardown)
[Passing Arguments](#passing-arguments) [Passing Arguments](#passing-arguments)
[Custom Benchmark Name](#custom-benchmark-name) [Custom Benchmark Name](#custom-benchmark-name)
[Calculating Asymptotic Complexity](#asymptotic-complexity) [Calculating Asymptotic Complexity](#asymptotic-complexity)
[Templated Benchmarks](#templated-benchmarks) [Templated Benchmarks](#templated-benchmarks)
[Fixtures](#fixtures) [Fixtures](#fixtures)
[Custom Counters](#custom-counters) [Custom Counters](#custom-counters)
[Multithreaded Benchmarks](#multithreaded-benchmarks) [Multithreaded Benchmarks](#multithreaded-benchmarks)
[CPU Timers](#cpu-timers) [CPU Timers](#cpu-timers)
[Manual Timing](#manual-timing) [Manual Timing](#manual-timing)
[Setting the Time Unit](#setting-the-time-unit) [Setting the Time Unit](#setting-the-time-unit)
[Random Interleaving](docs/random_interleaving.md) [Random Interleaving](random_interleaving.md)
[User-Requested Performance Counters](docs/perf_counters.md) [User-Requested Performance Counters](perf_counters.md)
[Preventing Optimization](#preventing-optimization) [Preventing Optimization](#preventing-optimization)
[Reporting Statistics](#reporting-statistics) [Reporting Statistics](#reporting-statistics)
[Custom Statistics](#custom-statistics) [Custom Statistics](#custom-statistics)
[Using RegisterBenchmark](#using-register-benchmark) [Using RegisterBenchmark](#using-register-benchmark)
[Exiting with an Error](#exiting-with-an-error) [Exiting with an Error](#exiting-with-an-error)
[A Faster KeepRunning Loop](#a-faster-keep-running-loop) [A Faster KeepRunning Loop](#a-faster-keep-running-loop)
[Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling) [Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling)
<a name="output-formats" /> <a name="output-formats" />
### Output Formats ## Output Formats
The library supports multiple output formats. Use the The library supports multiple output formats. Use the
`--benchmark_format=<console|json|csv>` flag (or set the `--benchmark_format=<console|json|csv>` flag (or set the
`BENCHMARK_FORMAT=<console|json|csv>` environment variable) to set `BENCHMARK_FORMAT=<console|json|csv>` environment variable) to set
the format type. `console` is the default format. the format type. `console` is the default format.
The Console format is intended to be a human readable format. By default The Console format is intended to be a human readable format. By default
the format generates color output. Context is output on stderr and the the format generates color output. Context is output on stderr and the
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines
name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label
"BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942, "BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942,
"BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115, "BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115,
"BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, "BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06,
``` ```
<a name="output-files" /> <a name="output-files" />
### Output Files ## Output Files
Write benchmark results to a file with the `--benchmark_out=<filename>` option Write benchmark results to a file with the `--benchmark_out=<filename>` option
(or set `BENCHMARK_OUT`). Specify the output format with (or set `BENCHMARK_OUT`). Specify the output format with
`--benchmark_out_format={json|console|csv}` (or set `--benchmark_out_format={json|console|csv}` (or set
`BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is `BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is
deprecated and the saved `.csv` file deprecated and the saved `.csv` file
[is not parsable](https://github.com/google/benchmark/issues/794) by csv [is not parsable](https://github.com/google/benchmark/issues/794) by csv
parsers. parsers.
Specifying `--benchmark_out` does not suppress the console output. Specifying `--benchmark_out` does not suppress the console output.
<a name="running-benchmarks" /> <a name="running-benchmarks" />
### Running Benchmarks ## Running Benchmarks
Benchmarks are executed by running the produced binaries. Benchmarks binaries, Benchmarks are executed by running the produced binaries. Benchmarks binaries,
by default, accept options that may be specified either through their command by default, accept options that may be specified either through their command
line interface or by setting environment variables before execution. For every line interface or by setting environment variables before execution. For every
`--option_flag=<value>` CLI switch, a corresponding environment variable `--option_flag=<value>` CLI switch, a corresponding environment variable
`OPTION_FLAG=<value>` exist and is used as default if set (CLI switches always `OPTION_FLAG=<value>` exist and is used as default if set (CLI switches always
prevails). A complete list of CLI options is available running benchmarks prevails). A complete list of CLI options is available running benchmarks
with the `--help` switch. with the `--help` switch.
<a name="running-a-subset-of-benchmarks" /> <a name="running-a-subset-of-benchmarks" />
### Running a Subset of Benchmarks ## Running a Subset of Benchmarks
The `--benchmark_filter=<regex>` option (or `BENCHMARK_FILTER=<regex>` The `--benchmark_filter=<regex>` option (or `BENCHMARK_FILTER=<regex>`
environment variable) can be used to only run the benchmarks that match environment variable) can be used to only run the benchmarks that match
the specified `<regex>`. For example: the specified `<regex>`. For example:
```bash ```bash
$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32 $ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32
Run on (1 X 2300 MHz CPU ) Run on (1 X 2300 MHz CPU )
2016-06-25 19:34:24 2016-06-25 19:34:24
Benchmark Time CPU Iterations Benchmark Time CPU Iterations
---------------------------------------------------- ----------------------------------------------------
BM_memcpy/32 11 ns 11 ns 79545455 BM_memcpy/32 11 ns 11 ns 79545455
BM_memcpy/32k 2181 ns 2185 ns 324074 BM_memcpy/32k 2181 ns 2185 ns 324074
BM_memcpy/32 12 ns 12 ns 54687500 BM_memcpy/32 12 ns 12 ns 54687500
BM_memcpy/32k 1834 ns 1837 ns 357143 BM_memcpy/32k 1834 ns 1837 ns 357143
``` ```
<a name="result-comparison" /> <a name="result-comparison" />
### Result comparison ## Result comparison
It is possible to compare the benchmarking results. It is possible to compare the benchmarking results.
See [Additional Tooling Documentation](docs/tools.md) See [Additional Tooling Documentation](tools.md)
<a name="extra-context" /> <a name="extra-context" />
### Extra Context ## Extra Context
Sometimes it's useful to add extra context to the content printed before the Sometimes it's useful to add extra context to the content printed before the
results. By default this section includes information about the CPU on which results. By default this section includes information about the CPU on which
the benchmarks are running. If you do want to add more context, you can use the benchmarks are running. If you do want to add more context, you can use
the `benchmark_context` command line flag: the `benchmark_context` command line flag:
```bash ```bash
$ ./run_benchmarks --benchmark_context=pwd=`pwd` $ ./run_benchmarks --benchmark_context=pwd=`pwd`
Show All 11 Lines```c++
benchmark::AddCustomContext("foo", "bar"); benchmark::AddCustomContext("foo", "bar");
``` ```
Note that attempts to add a second value with the same key will fail with an Note that attempts to add a second value with the same key will fail with an
error message. error message.
<a name="runtime-and-reporting-considerations" /> <a name="runtime-and-reporting-considerations" />
### Runtime and Reporting Considerations ## Runtime and Reporting Considerations
When the benchmark binary is executed, each benchmark function is run serially. When the benchmark binary is executed, each benchmark function is run serially.
The number of iterations to run is determined dynamically by running the The number of iterations to run is determined dynamically by running the
benchmark a few times and measuring the time taken and ensuring that the benchmark a few times and measuring the time taken and ensuring that the
ultimate result will be statistically stable. As such, faster benchmark ultimate result will be statistically stable. As such, faster benchmark
functions will be run for more iterations than slower benchmark functions, and functions will be run for more iterations than slower benchmark functions, and
the number of iterations is thus reported. the number of iterations is thus reported.
In all cases, the number of iterations for which the benchmark is run is In all cases, the number of iterations for which the benchmark is run is
governed by the amount of time the benchmark takes. Concretely, the number of governed by the amount of time the benchmark takes. Concretely, the number of
iterations is at least one, not more than 1e9, until CPU time is greater than iterations is at least one, not more than 1e9, until CPU time is greater than
the minimum time, or the wallclock time is 5x minimum time. The minimum time is the minimum time, or the wallclock time is 5x minimum time. The minimum time is
set per benchmark by calling `MinTime` on the registered benchmark object. set per benchmark by calling `MinTime` on the registered benchmark object.
Average timings are then reported over the iterations run. If multiple Average timings are then reported over the iterations run. If multiple
repetitions are requested using the `--benchmark_repetitions` command-line repetitions are requested using the `--benchmark_repetitions` command-line
option, or at registration time, the benchmark function will be run several option, or at registration time, the benchmark function will be run several
times and statistical results across these repetitions will also be reported. times and statistical results across these repetitions will also be reported.
As well as the per-benchmark entries, a preamble in the report will include As well as the per-benchmark entries, a preamble in the report will include
information about the machine on which the benchmarks are run. information about the machine on which the benchmarks are run.
<a name="setup-teardown" />
## Setup/Teardown
Global setup/teardown specific to each benchmark can be done by
passing a callback to Setup/Teardown:
The setup/teardown callbacks will be invoked once for each benchmark.
If the benchmark is multi-threaded (will run in k threads), they will be invoked exactly once before
each run with k threads.
If the benchmark uses different size groups of threads, the above will be true for each size group.
Eg.,
```c++
static void DoSetup(const benchmark::State& state) {
}
static void DoTeardown(const benchmark::State& state) {
}
static void BM_func(benchmark::State& state) {...}
BENCHMARK(BM_func)->Arg(1)->Arg(3)->Threads(16)->Threads(32)->Setup(DoSetup)->Teardown(DoTeardown);
```
In this example, `DoSetup` and `DoTearDown` will be invoked 4 times each,
specifically, once for each of this family:
- BM_func_Arg_1_Threads_16, BM_func_Arg_1_Threads_32
- BM_func_Arg_3_Threads_16, BM_func_Arg_3_Threads_32
<a name="passing-arguments" /> <a name="passing-arguments" />
### Passing Arguments ## Passing Arguments
Sometimes a family of benchmarks can be implemented with just one routine that Sometimes a family of benchmarks can be implemented with just one routine that
takes an extra argument to specify which one of the family of benchmarks to takes an extra argument to specify which one of the family of benchmarks to
run. For example, the following code defines a family of benchmarks for run. For example, the following code defines a family of benchmarks for
measuring the speed of `memcpy()` calls of different lengths: measuring the speed of `memcpy()` calls of different lengths:
```c++ ```c++
static void BM_memcpy(benchmark::State& state) { static void BM_memcpy(benchmark::State& state) {
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 LinesBENCHMARK(BM_SetInsert)
->Args({8<<10, 512}); ->Args({8<<10, 512});
``` ```
The preceding code is quite repetitive, and can be replaced with the following The preceding code is quite repetitive, and can be replaced with the following
short-hand. The following macro will pick a few appropriate arguments in the short-hand. The following macro will pick a few appropriate arguments in the
product of the two specified ranges and will generate a benchmark for each such product of the two specified ranges and will generate a benchmark for each such
pair. pair.
{% raw %}
```c++ ```c++
BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}});
``` ```
{% endraw %}
Some benchmarks may require specific argument values that cannot be expressed Some benchmarks may require specific argument values that cannot be expressed
with `Ranges`. In this case, `ArgsProduct` offers the ability to generate a with `Ranges`. In this case, `ArgsProduct` offers the ability to generate a
benchmark input for each combination in the product of the supplied vectors. benchmark input for each combination in the product of the supplied vectors.
{% raw %}
```c++ ```c++
BENCHMARK(BM_SetInsert) BENCHMARK(BM_SetInsert)
->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}}) ->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}})
// would generate the same benchmark arguments as // would generate the same benchmark arguments as
BENCHMARK(BM_SetInsert) BENCHMARK(BM_SetInsert)
->Args({1<<10, 20}) ->Args({1<<10, 20})
->Args({3<<10, 20}) ->Args({3<<10, 20})
->Args({8<<10, 20}) ->Args({8<<10, 20})
->Args({3<<10, 40}) ->Args({3<<10, 40})
->Args({8<<10, 40}) ->Args({8<<10, 40})
->Args({1<<10, 40}) ->Args({1<<10, 40})
->Args({1<<10, 60}) ->Args({1<<10, 60})
->Args({3<<10, 60}) ->Args({3<<10, 60})
->Args({8<<10, 60}) ->Args({8<<10, 60})
->Args({1<<10, 80}) ->Args({1<<10, 80})
->Args({3<<10, 80}) ->Args({3<<10, 80})
->Args({8<<10, 80}); ->Args({8<<10, 80});
``` ```
{% endraw %}
For the most common scenarios, helper methods for creating a list of
integers for a given sparse or dense range are provided.
```c++
BENCHMARK(BM_SetInsert)
->ArgsProduct({
benchmark::CreateRange(8, 128, /*multi=*/2),
benchmark::CreateDenseRange(1, 4, /*step=*/1)
})
// would generate the same benchmark arguments as
BENCHMARK(BM_SetInsert)
->ArgsProduct({
{8, 16, 32, 64, 128},
{1, 2, 3, 4}
});
```
For more complex patterns of inputs, passing a custom function to `Apply` allows For more complex patterns of inputs, passing a custom function to `Apply` allows
programmatic specification of an arbitrary set of arguments on which to run the programmatic specification of an arbitrary set of arguments on which to run the
benchmark. The following example enumerates a dense range on one parameter, benchmark. The following example enumerates a dense range on one parameter,
and a sparse range on the second. and a sparse range on the second.
```c++ ```c++
static void CustomArguments(benchmark::internal::Benchmark* b) { static void CustomArguments(benchmark::internal::Benchmark* b) {
for (int i = 0; i <= 10; ++i) for (int i = 0; i <= 10; ++i)
for (int j = 32; j <= 1024*1024; j *= 8) for (int j = 32; j <= 1024*1024; j *= 8)
b->Args({i, j}); b->Args({i, j});
} }
BENCHMARK(BM_SetInsert)->Apply(CustomArguments); BENCHMARK(BM_SetInsert)->Apply(CustomArguments);
``` ```
#### Passing Arbitrary Arguments to a Benchmark ### Passing Arbitrary Arguments to a Benchmark
In C++11 it is possible to define a benchmark that takes an arbitrary number In C++11 it is possible to define a benchmark that takes an arbitrary number
of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)` of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)`
macro creates a benchmark that invokes `func` with the `benchmark::State` as macro creates a benchmark that invokes `func` with the `benchmark::State` as
the first argument followed by the specified `args...`. the first argument followed by the specified `args...`.
The `test_case_name` is appended to the name of the benchmark and The `test_case_name` is appended to the name of the benchmark and
should describe the values passed. should describe the values passed.
```c++ ```c++
template <class ...ExtraArgs> template <class ...ExtraArgs>
void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) {
[...] [...]
} }
// Registers a benchmark named "BM_takes_args/int_string_test" that passes // Registers a benchmark named "BM_takes_args/int_string_test" that passes
// the specified values to `extra_args`. // the specified values to `extra_args`.
BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc"));
``` ```
Note that elements of `...args` may refer to global variables. Users should Note that elements of `...args` may refer to global variables. Users should
avoid modifying global state inside of a benchmark. avoid modifying global state inside of a benchmark.
<a name="asymptotic-complexity" /> <a name="asymptotic-complexity" />
### Calculating Asymptotic Complexity (Big O) ## Calculating Asymptotic Complexity (Big O)
Asymptotic complexity might be calculated for a family of benchmarks. The Asymptotic complexity might be calculated for a family of benchmarks. The
following code will calculate the coefficient for the high-order term in the following code will calculate the coefficient for the high-order term in the
running time and the normalized root-mean square error of string comparison. running time and the normalized root-mean square error of string comparison.
```c++ ```c++
static void BM_StringCompare(benchmark::State& state) { static void BM_StringCompare(benchmark::State& state) {
std::string s1(state.range(0), '-'); std::string s1(state.range(0), '-');
Show All 20 Lines
```c++ ```c++
BENCHMARK(BM_StringCompare)->RangeMultiplier(2) BENCHMARK(BM_StringCompare)->RangeMultiplier(2)
->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; }); ->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; });
``` ```
<a name="custom-benchmark-name" /> <a name="custom-benchmark-name" />
### Custom Benchmark Name ## Custom Benchmark Name
You can change the benchmark's name as follows: You can change the benchmark's name as follows:
```c++ ```c++
BENCHMARK(BM_memcpy)->Name("memcpy")->RangeMultiplier(2)->Range(8, 8<<10); BENCHMARK(BM_memcpy)->Name("memcpy")->RangeMultiplier(2)->Range(8, 8<<10);
``` ```
The invocation will execute the benchmark as before using `BM_memcpy` but changes The invocation will execute the benchmark as before using `BM_memcpy` but changes
the prefix in the report to `memcpy`. the prefix in the report to `memcpy`.
<a name="templated-benchmarks" /> <a name="templated-benchmarks" />
### Templated Benchmarks ## Templated Benchmarks
This example produces and consumes messages of size `sizeof(v)` `range_x` This example produces and consumes messages of size `sizeof(v)` `range_x`
times. It also outputs throughput in the absence of multiprogramming. times. It also outputs throughput in the absence of multiprogramming.
```c++ ```c++
template <class Q> void BM_Sequential(benchmark::State& state) { template <class Q> void BM_Sequential(benchmark::State& state) {
Q q; Q q;
typename Q::value_type v; typename Q::value_type v;
for (auto _ : state) { for (auto _ : state) {
for (int i = state.range(0); i--; ) for (int i = state.range(0); i--; )
q.push(v); q.push(v);
for (int e = state.range(0); e--; ) for (int e = state.range(0); e--; )
q.Wait(&v); q.Wait(&v);
} }
// actually messages, not bytes: // actually messages, not bytes:
state.SetBytesProcessed( state.SetBytesProcessed(
static_cast<int64_t>(state.iterations())*state.range(0)); static_cast<int64_t>(state.iterations())*state.range(0));
} }
// C++03
BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue<int>)->Range(1<<0, 1<<10); BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue<int>)->Range(1<<0, 1<<10);
// C++11 or newer, you can use the BENCHMARK macro with template parameters:
BENCHMARK(BM_Sequential<WaitQueue<int>>)->Range(1<<0, 1<<10);
``` ```
Three macros are provided for adding benchmark templates. Three macros are provided for adding benchmark templates.
```c++ ```c++
#ifdef BENCHMARK_HAS_CXX11 #ifdef BENCHMARK_HAS_CXX11
#define BENCHMARK_TEMPLATE(func, ...) // Takes any number of parameters. #define BENCHMARK(func<...>) // Takes any number of parameters.
#else // C++ < C++11 #else // C++ < C++11
#define BENCHMARK_TEMPLATE(func, arg1) #define BENCHMARK_TEMPLATE(func, arg1)
#endif #endif
#define BENCHMARK_TEMPLATE1(func, arg1) #define BENCHMARK_TEMPLATE1(func, arg1)
#define BENCHMARK_TEMPLATE2(func, arg1, arg2) #define BENCHMARK_TEMPLATE2(func, arg1, arg2)
``` ```
<a name="fixtures" /> <a name="fixtures" />
### Fixtures ## Fixtures
Fixture tests are created by first defining a type that derives from Fixture tests are created by first defining a type that derives from
`::benchmark::Fixture` and then creating/registering the tests using the `::benchmark::Fixture` and then creating/registering the tests using the
following macros: following macros:
* `BENCHMARK_F(ClassName, Method)` * `BENCHMARK_F(ClassName, Method)`
* `BENCHMARK_DEFINE_F(ClassName, Method)` * `BENCHMARK_DEFINE_F(ClassName, Method)`
* `BENCHMARK_REGISTER_F(ClassName, Method)` * `BENCHMARK_REGISTER_F(ClassName, Method)`
Show All 21 Lines for (auto _ : st) {
... ...
} }
} }
/* BarTest is NOT registered */ /* BarTest is NOT registered */
BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2);
/* BarTest is now registered */ /* BarTest is now registered */
``` ```
#### Templated Fixtures ### Templated Fixtures
Also you can create templated fixture by using the following macros: Also you can create templated fixture by using the following macros:
* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)` * `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)`
* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)` * `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)`
For example: For example:
Show All 13 LinesBENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) {
} }
} }
BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2); BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2);
``` ```
<a name="custom-counters" /> <a name="custom-counters" />
### Custom Counters ## Custom Counters
You can add your own counters with user-defined names. The example below You can add your own counters with user-defined names. The example below
will add columns "Foo", "Bar" and "Baz" in its output: will add columns "Foo", "Bar" and "Baz" in its output:
```c++ ```c++
static void UserCountersExample1(benchmark::State& state) { static void UserCountersExample1(benchmark::State& state) {
double numFoos = 0, numBars = 0, numBazs = 0; double numFoos = 0, numBars = 0, numBazs = 0;
for (auto _ : state) { for (auto _ : state) {
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines```c++
// This says that we process with the rate of state.range(0) bytes every iteration: // This says that we process with the rate of state.range(0) bytes every iteration:
state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024); state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024);
``` ```
When you're compiling in C++11 mode or later you can use `insert()` with When you're compiling in C++11 mode or later you can use `insert()` with
`std::initializer_list`: `std::initializer_list`:
{% raw %}
```c++ ```c++
// With C++11, this can be done: // With C++11, this can be done:
state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}});
// ... instead of: // ... instead of:
state.counters["Foo"] = numFoos; state.counters["Foo"] = numFoos;
state.counters["Bar"] = numBars; state.counters["Bar"] = numBars;
state.counters["Baz"] = numBazs; state.counters["Baz"] = numBazs;
``` ```
{% endraw %}
#### Counter Reporting ### Counter Reporting
When using the console reporter, by default, user counters are printed at When using the console reporter, by default, user counters are printed at
the end after the table, the same way as ``bytes_processed`` and the end after the table, the same way as ``bytes_processed`` and
``items_processed``. This is best for cases in which there are few counters, ``items_processed``. This is best for cases in which there are few counters,
or where there are only a couple of lines per benchmark. Here's an example of or where there are only a couple of lines per benchmark. Here's an example of
the default output: the default output:
``` ```
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines
``` ```
Note above the additional header printed when the benchmark changes from Note above the additional header printed when the benchmark changes from
``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does ``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does
not have the same counter set as ``BM_UserCounter``. not have the same counter set as ``BM_UserCounter``.
<a name="multithreaded-benchmarks"/> <a name="multithreaded-benchmarks"/>
### Multithreaded Benchmarks ## Multithreaded Benchmarks
In a multithreaded test (benchmark invoked by multiple threads simultaneously), In a multithreaded test (benchmark invoked by multiple threads simultaneously),
it is guaranteed that none of the threads will start until all have reached it is guaranteed that none of the threads will start until all have reached
the start of the benchmark loop, and all will have finished before any thread the start of the benchmark loop, and all will have finished before any thread
exits the benchmark loop. (This behavior is also provided by the `KeepRunning()` exits the benchmark loop. (This behavior is also provided by the `KeepRunning()`
API) As such, any global setup or teardown can be wrapped in a check against the thread API) As such, any global setup or teardown can be wrapped in a check against the thread
index: index:
```c++ ```c++
static void BM_MultiThreaded(benchmark::State& state) { static void BM_MultiThreaded(benchmark::State& state) {
if (state.thread_index == 0) { if (state.thread_index() == 0) {
// Setup code here. // Setup code here.
} }
for (auto _ : state) { for (auto _ : state) {
// Run the test as normal. // Run the test as normal.
} }
if (state.thread_index == 0) { if (state.thread_index() == 0) {
// Teardown code here. // Teardown code here.
} }
} }
BENCHMARK(BM_MultiThreaded)->Threads(2); BENCHMARK(BM_MultiThreaded)->Threads(2);
``` ```
If the benchmarked code itself uses threads and you want to compare it to If the benchmarked code itself uses threads and you want to compare it to
single-threaded code, you may want to use real-time ("wallclock") measurements single-threaded code, you may want to use real-time ("wallclock") measurements
for latency comparisons: for latency comparisons:
```c++ ```c++
BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime();
``` ```
Without `UseRealTime`, CPU time is used by default. Without `UseRealTime`, CPU time is used by default.
<a name="cpu-timers" /> <a name="cpu-timers" />
### CPU Timers ## CPU Timers
By default, the CPU timer only measures the time spent by the main thread. By default, the CPU timer only measures the time spent by the main thread.
If the benchmark itself uses threads internally, this measurement may not If the benchmark itself uses threads internally, this measurement may not
be what you are looking for. Instead, there is a way to measure the total be what you are looking for. Instead, there is a way to measure the total
CPU usage of the process, by all the threads. CPU usage of the process, by all the threads.
```c++ ```c++
void callee(int i); void callee(int i);
Show All 27 Lines
// time spent by the main thread in single-threaded case. // time spent by the main thread in single-threaded case.
BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime(); BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime();
// A mixture of the last two. Measure the total CPU consumption, but use the // A mixture of the last two. Measure the total CPU consumption, but use the
// wall clock to decide for how long to run the benchmark loop. // wall clock to decide for how long to run the benchmark loop.
BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime(); BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime();
``` ```
#### Controlling Timers ### Controlling Timers
Normally, the entire duration of the work loop (`for (auto _ : state) {}`) Normally, the entire duration of the work loop (`for (auto _ : state) {}`)
is measured. But sometimes, it is necessary to do some work inside of is measured. But sometimes, it is necessary to do some work inside of
that loop, every iteration, but without counting that time to the benchmark time. that loop, every iteration, but without counting that time to the benchmark time.
That is possible, although it is not recommended, since it has high overhead. That is possible, although it is not recommended, since it has high overhead.
{% raw %}
```c++ ```c++
static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { static void BM_SetInsert_With_Timer_Control(benchmark::State& state) {
std::set<int> data; std::set<int> data;
for (auto _ : state) { for (auto _ : state) {
state.PauseTiming(); // Stop timers. They will not count until they are resumed. state.PauseTiming(); // Stop timers. They will not count until they are resumed.
data = ConstructRandomSet(state.range(0)); // Do something that should not be measured data = ConstructRandomSet(state.range(0)); // Do something that should not be measured
state.ResumeTiming(); // And resume timers. They are now counting again. state.ResumeTiming(); // And resume timers. They are now counting again.
// The rest will be measured. // The rest will be measured.
for (int j = 0; j < state.range(1); ++j) for (int j = 0; j < state.range(1); ++j)
data.insert(RandomNumber()); data.insert(RandomNumber());
} }
} }
BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}});
``` ```
{% endraw %}
<a name="manual-timing" /> <a name="manual-timing" />
### Manual Timing ## Manual Timing
For benchmarking something for which neither CPU time nor real-time are For benchmarking something for which neither CPU time nor real-time are
correct or accurate enough, completely manual timing is supported using correct or accurate enough, completely manual timing is supported using
the `UseManualTime` function. the `UseManualTime` function.
When `UseManualTime` is used, the benchmarked code must call When `UseManualTime` is used, the benchmarked code must call
`SetIterationTime` once per iteration of the benchmark loop to `SetIterationTime` once per iteration of the benchmark loop to
report the manually measured time. report the manually measured time.
Show All 24 Lines for (auto _ : state) {
state.SetIterationTime(elapsed_seconds.count()); state.SetIterationTime(elapsed_seconds.count());
} }
} }
BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime(); BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime();
``` ```
<a name="setting-the-time-unit" /> <a name="setting-the-time-unit" />
### Setting the Time Unit ## Setting the Time Unit
If a benchmark runs a few milliseconds it may be hard to visually compare the If a benchmark runs a few milliseconds it may be hard to visually compare the
measured times, since the output data is given in nanoseconds per default. In measured times, since the output data is given in nanoseconds per default. In
order to manually set the time unit, you can specify it manually: order to manually set the time unit, you can specify it manually:
```c++ ```c++
BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
``` ```
<a name="preventing-optimization" /> <a name="preventing-optimization" />
### Preventing Optimization ## Preventing Optimization
To prevent a value or expression from being optimized away by the compiler To prevent a value or expression from being optimized away by the compiler
the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()` the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()`
functions can be used. functions can be used.
```c++ ```c++
static void BM_test(benchmark::State& state) { static void BM_test(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Linesstatic void BM_vector_push_back(benchmark::State& state) {
} }
} }
``` ```
Note that `ClobberMemory()` is only available for GNU or MSVC based compilers. Note that `ClobberMemory()` is only available for GNU or MSVC based compilers.
<a name="reporting-statistics" /> <a name="reporting-statistics" />
### Statistics: Reporting the Mean, Median and Standard Deviation of Repeated Benchmarks ## Statistics: Reporting the Mean, Median and Standard Deviation / Coefficient of variation of Repeated Benchmarks
By default each benchmark is run once and that single result is reported. By default each benchmark is run once and that single result is reported.
However benchmarks are often noisy and a single result may not be representative However benchmarks are often noisy and a single result may not be representative
of the overall behavior. For this reason it's possible to repeatedly rerun the of the overall behavior. For this reason it's possible to repeatedly rerun the
benchmark. benchmark.
The number of runs of each benchmark is specified globally by the The number of runs of each benchmark is specified globally by the
`--benchmark_repetitions` flag or on a per benchmark basis by calling `--benchmark_repetitions` flag or on a per benchmark basis by calling
`Repetitions` on the registered benchmark object. When a benchmark is run more `Repetitions` on the registered benchmark object. When a benchmark is run more
than once the mean, median and standard deviation of the runs will be reported. than once the mean, median, standard deviation and coefficient of variation
of the runs will be reported.
Additionally the `--benchmark_report_aggregates_only={true|false}`, Additionally the `--benchmark_report_aggregates_only={true|false}`,
`--benchmark_display_aggregates_only={true|false}` flags or `--benchmark_display_aggregates_only={true|false}` flags or
`ReportAggregatesOnly(bool)`, `DisplayAggregatesOnly(bool)` functions can be `ReportAggregatesOnly(bool)`, `DisplayAggregatesOnly(bool)` functions can be
used to change how repeated tests are reported. By default the result of each used to change how repeated tests are reported. By default the result of each
repeated run is reported. When `report aggregates only` option is `true`, repeated run is reported. When `report aggregates only` option is `true`,
only the aggregates (i.e. mean, median and standard deviation, maybe complexity only the aggregates (i.e. mean, median, standard deviation and coefficient
measurements if they were requested) of the runs is reported, to both the of variation, maybe complexity measurements if they were requested) of the runs
reporters - standard output (console), and the file. is reported, to both the reporters - standard output (console), and the file.
However when only the `display aggregates only` option is `true`, However when only the `display aggregates only` option is `true`,
only the aggregates are displayed in the standard output, while the file only the aggregates are displayed in the standard output, while the file
output still contains everything. output still contains everything.
Calling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a Calling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a
registered benchmark object overrides the value of the appropriate flag for that registered benchmark object overrides the value of the appropriate flag for that
benchmark. benchmark.
<a name="custom-statistics" /> <a name="custom-statistics" />
### Custom Statistics ## Custom Statistics
While having mean, median and standard deviation is nice, this may not be While having these aggregates is nice, this may not be enough for everyone.
enough for everyone. For example you may want to know what the largest For example you may want to know what the largest observation is, e.g. because
observation is, e.g. because you have some real-time constraints. This is easy. you have some real-time constraints. This is easy. The following code will
The following code will specify a custom statistic to be calculated, defined specify a custom statistic to be calculated, defined by a lambda function.
by a lambda function.
```c++ ```c++
void BM_spin_empty(benchmark::State& state) { void BM_spin_empty(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
for (int x = 0; x < state.range(0); ++x) { for (int x = 0; x < state.range(0); ++x) {
benchmark::DoNotOptimize(x); benchmark::DoNotOptimize(x);
} }
} }
} }
BENCHMARK(BM_spin_empty) BENCHMARK(BM_spin_empty)
->ComputeStatistics("max", [](const std::vector<double>& v) -> double { ->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
return *(std::max_element(std::begin(v), std::end(v))); return *(std::max_element(std::begin(v), std::end(v)));
}) })
->Arg(512); ->Arg(512);
``` ```
While usually the statistics produce values in time units,
you can also produce percentages:
```c++
void BM_spin_empty(benchmark::State& state) {
for (auto _ : state) {
for (int x = 0; x < state.range(0); ++x) {
benchmark::DoNotOptimize(x);
}
}
}
BENCHMARK(BM_spin_empty)
->ComputeStatistics("ratio", [](const std::vector<double>& v) -> double {
return std::begin(v) / std::end(v);
}, benchmark::StatisticUnit::Percentage)
->Arg(512);
```
<a name="using-register-benchmark" /> <a name="using-register-benchmark" />
### Using RegisterBenchmark(name, fn, args...) ## Using RegisterBenchmark(name, fn, args...)
The `RegisterBenchmark(name, func, args...)` function provides an alternative The `RegisterBenchmark(name, func, args...)` function provides an alternative
way to create and register benchmarks. way to create and register benchmarks.
`RegisterBenchmark(name, func, args...)` creates, registers, and returns a `RegisterBenchmark(name, func, args...)` creates, registers, and returns a
pointer to a new benchmark with the specified `name` that invokes pointer to a new benchmark with the specified `name` that invokes
`func(st, args...)` where `st` is a `benchmark::State` object. `func(st, args...)` where `st` is a `benchmark::State` object.
Unlike the `BENCHMARK` registration macros, which can only be used at the global Unlike the `BENCHMARK` registration macros, which can only be used at the global
Show All 13 Linesint main(int argc, char** argv) {
benchmark::Initialize(&argc, argv); benchmark::Initialize(&argc, argv);
benchmark::RunSpecifiedBenchmarks(); benchmark::RunSpecifiedBenchmarks();
benchmark::Shutdown(); benchmark::Shutdown();
} }
``` ```
<a name="exiting-with-an-error" /> <a name="exiting-with-an-error" />
### Exiting with an Error ## Exiting with an Error
When errors caused by external influences, such as file I/O and network When errors caused by external influences, such as file I/O and network
communication, occur within a benchmark the communication, occur within a benchmark the
`State::SkipWithError(const char* msg)` function can be used to skip that run `State::SkipWithError(const char* msg)` function can be used to skip that run
of benchmark and report the error. Note that only future iterations of the of benchmark and report the error. Note that only future iterations of the
`KeepRunning()` are skipped. For the ranged-for version of the benchmark loop `KeepRunning()` are skipped. For the ranged-for version of the benchmark loop
Users must explicitly exit the loop, otherwise all iterations will be performed. Users must explicitly exit the loop, otherwise all iterations will be performed.
Users may explicitly return to exit the benchmark immediately. Users may explicitly return to exit the benchmark immediately.
Show All 35 Lines if (!resource.good()) {
break; // REQUIRED to prevent all further iterations. break; // REQUIRED to prevent all further iterations.
} }
do_stuff(data); do_stuff(data);
} }
} }
``` ```
<a name="a-faster-keep-running-loop" /> <a name="a-faster-keep-running-loop" />
### A Faster KeepRunning Loop ## A Faster KeepRunning Loop
In C++11 mode, a ranged-based for loop should be used in preference to In C++11 mode, a ranged-based for loop should be used in preference to
the `KeepRunning` loop for running the benchmarks. For example: the `KeepRunning` loop for running the benchmarks. For example:
```c++ ```c++
static void BM_Fast(benchmark::State &state) { static void BM_Fast(benchmark::State &state) {
for (auto _ : state) { for (auto _ : state) {
FastOperation(); FastOperation();
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines
.LoopEnd: .LoopEnd:
``` ```
Unless C++03 compatibility is required, the ranged-for variant of writing Unless C++03 compatibility is required, the ranged-for variant of writing
the benchmark loop should be preferred. the benchmark loop should be preferred.
<a name="disabling-cpu-frequency-scaling" /> <a name="disabling-cpu-frequency-scaling" />
### Disabling CPU Frequency Scaling ## Disabling CPU Frequency Scaling
If you see this error: If you see this error:
``` ```
***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead.
``` ```
you might want to disable the CPU frequency scaling while running the benchmark: you might want to disable the CPU frequency scaling while running the benchmark:
```bash ```bash
sudo cpupower frequency-set --governor performance sudo cpupower frequency-set --governor performance
./mybench ./mybench
sudo cpupower frequency-set --governor powersave sudo cpupower frequency-set --governor powersave
``` ```

third-party/benchmark/include/benchmark/benchmark.h

Show All 28 Lines
static void BM_StringCopy(benchmark::State& state) { static void BM_StringCopy(benchmark::State& state) {
std::string x = "hello"; std::string x = "hello";
for (auto _ : state) for (auto _ : state)
std::string copy(x); std::string copy(x);
} }
BENCHMARK(BM_StringCopy); BENCHMARK(BM_StringCopy);
// Augment the main() program to invoke benchmarks if specified // Augment the main() program to invoke benchmarks if specified
// via the --benchmarks command line flag. E.g., // via the --benchmark_filter command line flag. E.g.,
// my_unittest --benchmark_filter=all // my_unittest --benchmark_filter=all
// my_unittest --benchmark_filter=BM_StringCreation // my_unittest --benchmark_filter=BM_StringCreation
// my_unittest --benchmark_filter=String // my_unittest --benchmark_filter=String
// my_unittest --benchmark_filter='Copy|Creation' // my_unittest --benchmark_filter='Copy|Creation'
int main(int argc, char** argv) { int main(int argc, char** argv) {
benchmark::Initialize(&argc, argv); benchmark::Initialize(&argc, argv);
benchmark::RunSpecifiedBenchmarks(); benchmark::RunSpecifiedBenchmarks();
benchmark::Shutdown(); benchmark::Shutdown();
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines
BENCHMARK(BM_test)->MinTime(2.0); // Run for at least 2 seconds. BENCHMARK(BM_test)->MinTime(2.0); // Run for at least 2 seconds.
In a multithreaded test, it is guaranteed that none of the threads will start In a multithreaded test, it is guaranteed that none of the threads will start
until all have reached the loop start, and all will have finished before any until all have reached the loop start, and all will have finished before any
thread exits the loop body. As such, any global setup or teardown you want to thread exits the loop body. As such, any global setup or teardown you want to
do can be wrapped in a check against the thread index: do can be wrapped in a check against the thread index:
static void BM_MultiThreaded(benchmark::State& state) { static void BM_MultiThreaded(benchmark::State& state) {
if (state.thread_index == 0) { if (state.thread_index() == 0) {
// Setup code here. // Setup code here.
} }
for (auto _ : state) { for (auto _ : state) {
// Run the test as normal. // Run the test as normal.
} }
if (state.thread_index == 0) { if (state.thread_index() == 0) {
// Teardown code here. // Teardown code here.
} }
} }
BENCHMARK(BM_MultiThreaded)->Threads(4); BENCHMARK(BM_MultiThreaded)->Threads(4);
If a benchmark runs a few milliseconds it may be hard to visually compare the If a benchmark runs a few milliseconds it may be hard to visually compare the
measured times, since the output data is given in nanoseconds per default. In measured times, since the output data is given in nanoseconds per default. In
Show All 17 Lines
#endif #endif
#include <stdint.h> #include <stdint.h>
#include <algorithm> #include <algorithm>
#include <cassert> #include <cassert>
#include <cstddef> #include <cstddef>
#include <iosfwd> #include <iosfwd>
#include <limits>
#include <map> #include <map>
#include <set> #include <set>
#include <string> #include <string>
#include <utility> #include <utility>
#include <vector> #include <vector>
#if defined(BENCHMARK_HAS_CXX11) #if defined(BENCHMARK_HAS_CXX11)
#include <atomic>
#include <initializer_list> #include <initializer_list>
#include <type_traits> #include <type_traits>
#include <utility> #include <utility>
#endif #endif
#if defined(_MSC_VER) #if defined(_MSC_VER)
#include <intrin.h> // for _ReadWriteBarrier #include <intrin.h> // for _ReadWriteBarrier
#endif #endif
Show All 34 Lines
#define BENCHMARK_ALWAYS_INLINE #define BENCHMARK_ALWAYS_INLINE
#define BENCHMARK_NOEXCEPT #define BENCHMARK_NOEXCEPT
#define BENCHMARK_NOEXCEPT_OP(x) #define BENCHMARK_NOEXCEPT_OP(x)
#endif #endif
#define BENCHMARK_INTERNAL_TOSTRING2(x) #x #define BENCHMARK_INTERNAL_TOSTRING2(x) #x
#define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) #define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x)
// clang-format off
#if defined(__GNUC__) || defined(__clang__) #if defined(__GNUC__) || defined(__clang__)
#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) #define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y)
#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) #define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))
#define BENCHMARK_DISABLE_DEPRECATED_WARNING \
_Pragma("GCC diagnostic push") \
_Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("GCC diagnostic pop")
#else #else
#define BENCHMARK_BUILTIN_EXPECT(x, y) x #define BENCHMARK_BUILTIN_EXPECT(x, y) x
#define BENCHMARK_DEPRECATED_MSG(msg) #define BENCHMARK_DEPRECATED_MSG(msg)
#define BENCHMARK_WARNING_MSG(msg) \ #define BENCHMARK_WARNING_MSG(msg) \
__pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING( \ __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING( \
__LINE__) ") : warning note: " msg)) __LINE__) ") : warning note: " msg))
#define BENCHMARK_DISABLE_DEPRECATED_WARNING
#define BENCHMARK_RESTORE_DEPRECATED_WARNING
#endif #endif
// clang-format on
#if defined(__GNUC__) && !defined(__clang__) #if defined(__GNUC__) && !defined(__clang__)
#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) #define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#endif #endif
#ifndef __has_builtin #ifndef __has_builtin
#define __has_builtin(x) 0 #define __has_builtin(x) 0
#endif #endif
Show All 9 Lines
#ifdef BENCHMARK_HAS_CXX11 #ifdef BENCHMARK_HAS_CXX11
#define BENCHMARK_OVERRIDE override #define BENCHMARK_OVERRIDE override
#else #else
#define BENCHMARK_OVERRIDE #define BENCHMARK_OVERRIDE
#endif #endif
namespace benchmark { namespace benchmark {
class BenchmarkReporter; class BenchmarkReporter;
class MemoryManager;
void Initialize(int* argc, char** argv); void Initialize(int* argc, char** argv);
void Shutdown(); void Shutdown();
// Report to stdout all arguments in 'argv' as unrecognized except the first. // Report to stdout all arguments in 'argv' as unrecognized except the first.
// Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). // Returns true there is at least on unrecognized argument (i.e. 'argc' > 1).
bool ReportUnrecognizedArguments(int argc, char** argv); bool ReportUnrecognizedArguments(int argc, char** argv);
// Returns the current value of --benchmark_filter.
std::string GetBenchmarkFilter();
// Generate a list of benchmarks matching the specified --benchmark_filter flag // Generate a list of benchmarks matching the specified --benchmark_filter flag
// and if --benchmark_list_tests is specified return after printing the name // and if --benchmark_list_tests is specified return after printing the name
// of each matching benchmark. Otherwise run each matching benchmark and // of each matching benchmark. Otherwise run each matching benchmark and
// report the results. // report the results.
// //
// spec : Specify the benchmarks to run. If users do not specify this arg,
// then the value of FLAGS_benchmark_filter
// will be used.
//
// The second and third overload use the specified 'display_reporter' and // The second and third overload use the specified 'display_reporter' and
// 'file_reporter' respectively. 'file_reporter' will write to the file // 'file_reporter' respectively. 'file_reporter' will write to the file
// specified // specified
// by '--benchmark_output'. If '--benchmark_output' is not given the // by '--benchmark_output'. If '--benchmark_output' is not given the
// 'file_reporter' is ignored. // 'file_reporter' is ignored.
// //
// RETURNS: The number of matching benchmarks. // RETURNS: The number of matching benchmarks.
size_t RunSpecifiedBenchmarks(); size_t RunSpecifiedBenchmarks();
size_t RunSpecifiedBenchmarks(std::string spec);
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter);
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
std::string spec);
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter); BenchmarkReporter* file_reporter);
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter,
std::string spec);
// If a MemoryManager is registered (via RegisterMemoryManager()),
// it can be used to collect and report allocation metrics for a run of the
// benchmark.
class MemoryManager {
public:
static const int64_t TombstoneValue;
struct Result {
Result()
: num_allocs(0),
max_bytes_used(0),
total_allocated_bytes(TombstoneValue),
net_heap_growth(TombstoneValue) {}
// The number of allocations made in total between Start and Stop.
int64_t num_allocs;
// The peak memory use between Start and Stop.
int64_t max_bytes_used;
// The total memory allocated, in bytes, between Start and Stop.
// Init'ed to TombstoneValue if metric not available.
int64_t total_allocated_bytes;
// The net changes in memory, in bytes, between Start and Stop.
// ie., total_allocated_bytes - total_deallocated_bytes.
// Init'ed to TombstoneValue if metric not available.
int64_t net_heap_growth;
};
virtual ~MemoryManager() {}
// Implement this to start recording allocation information.
virtual void Start() = 0;
// Implement this to stop recording and fill out the given Result structure.
BENCHMARK_DEPRECATED_MSG("Use Stop(Result&) instead")
virtual void Stop(Result* result) = 0;
// FIXME(vyng): Make this pure virtual once we've migrated current users.
BENCHMARK_DISABLE_DEPRECATED_WARNING
virtual void Stop(Result& result) { Stop(&result); }
BENCHMARK_RESTORE_DEPRECATED_WARNING
};
// Register a MemoryManager instance that will be used to collect and report // Register a MemoryManager instance that will be used to collect and report
// allocation measurements for benchmark runs. // allocation measurements for benchmark runs.
void RegisterMemoryManager(MemoryManager* memory_manager); void RegisterMemoryManager(MemoryManager* memory_manager);
// Add a key-value pair to output as part of the context stanza in the report. // Add a key-value pair to output as part of the context stanza in the report.
void AddCustomContext(const std::string& key, const std::string& value); void AddCustomContext(const std::string& key, const std::string& value);
Show All 14 Lines
} // namespace internal } // namespace internal
#if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \ #if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \
defined(__EMSCRIPTEN__) defined(__EMSCRIPTEN__)
#define BENCHMARK_HAS_NO_INLINE_ASSEMBLY #define BENCHMARK_HAS_NO_INLINE_ASSEMBLY
#endif #endif
// Force the compiler to flush pending writes to global memory. Acts as an
// effective read/write barrier
#ifdef BENCHMARK_HAS_CXX11
inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() {
std::atomic_signal_fence(std::memory_order_acq_rel);
}
#endif
// The DoNotOptimize(...) function can be used to prevent a value or // The DoNotOptimize(...) function can be used to prevent a value or
// expression from being optimized away by the compiler. This function is // expression from being optimized away by the compiler. This function is
// intended to add little to no overhead. // intended to add little to no overhead.
// See: https://youtu.be/nXaxk27zwlk?t=2441 // See: https://youtu.be/nXaxk27zwlk?t=2441
#ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY #ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY
template <class Tp> template <class Tp>
inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) {
asm volatile("" : : "r,m"(value) : "memory"); asm volatile("" : : "r,m"(value) : "memory");
} }
template <class Tp> template <class Tp>
inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) {
#if defined(__clang__) #if defined(__clang__)
asm volatile("" : "+r,m"(value) : : "memory"); asm volatile("" : "+r,m"(value) : : "memory");
#else #else
asm volatile("" : "+m,r"(value) : : "memory"); asm volatile("" : "+m,r"(value) : : "memory");
#endif #endif
} }
// Force the compiler to flush pending writes to global memory. Acts as an #ifndef BENCHMARK_HAS_CXX11
// effective read/write barrier
inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() {
asm volatile("" : : : "memory"); asm volatile("" : : : "memory");
} }
#endif
#elif defined(_MSC_VER) #elif defined(_MSC_VER)
template <class Tp> template <class Tp>
inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) {
internal::UseCharPointer(&reinterpret_cast<char const volatile&>(value)); internal::UseCharPointer(&reinterpret_cast<char const volatile&>(value));
_ReadWriteBarrier(); _ReadWriteBarrier();
} }
#ifndef BENCHMARK_HAS_CXX11
inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); } inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); }
#endif
#else #else
template <class Tp> template <class Tp>
inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) {
internal::UseCharPointer(&reinterpret_cast<char const volatile&>(value)); internal::UseCharPointer(&reinterpret_cast<char const volatile&>(value));
} }
// FIXME Add ClobberMemory() for non-gnu and non-msvc compilers // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers, before C++11.
#endif #endif
// This class is used for user-defined counters. // This class is used for user-defined counters.
class Counter { class Counter {
public: public:
enum Flags { enum Flags {
kDefaults = 0, kDefaults = 0,
// Mark the counter as a rate. It will be presented divided // Mark the counter as a rate. It will be presented divided
// by the duration of the benchmark. // by the duration of the benchmark.
kIsRate = 1U << 0U, kIsRate = 1 << 0,
// Mark the counter as a thread-average quantity. It will be // Mark the counter as a thread-average quantity. It will be
// presented divided by the number of threads. // presented divided by the number of threads.
kAvgThreads = 1U << 1U, kAvgThreads = 1 << 1,
// Mark the counter as a thread-average rate. See above. // Mark the counter as a thread-average rate. See above.
kAvgThreadsRate = kIsRate | kAvgThreads, kAvgThreadsRate = kIsRate | kAvgThreads,
// Mark the counter as a constant value, valid/same for *every* iteration. // Mark the counter as a constant value, valid/same for *every* iteration.
// When reporting, it will be *multiplied* by the iteration count. // When reporting, it will be *multiplied* by the iteration count.
kIsIterationInvariant = 1U << 2U, kIsIterationInvariant = 1 << 2,
// Mark the counter as a constant rate. // Mark the counter as a constant rate.
// When reporting, it will be *multiplied* by the iteration count // When reporting, it will be *multiplied* by the iteration count
// and then divided by the duration of the benchmark. // and then divided by the duration of the benchmark.
kIsIterationInvariantRate = kIsRate | kIsIterationInvariant, kIsIterationInvariantRate = kIsRate | kIsIterationInvariant,
// Mark the counter as a iteration-average quantity. // Mark the counter as a iteration-average quantity.
// It will be presented divided by the number of iterations. // It will be presented divided by the number of iterations.
kAvgIterations = 1U << 3U, kAvgIterations = 1 << 3,
// Mark the counter as a iteration-average rate. See above. // Mark the counter as a iteration-average rate. See above.
kAvgIterationsRate = kIsRate | kAvgIterations, kAvgIterationsRate = kIsRate | kAvgIterations,
// In the end, invert the result. This is always done last! // In the end, invert the result. This is always done last!
kInvert = 1U << 31U kInvert = 1 << 31
}; };
enum OneK { enum OneK {
// 1'000 items per 1k // 1'000 items per 1k
kIs1000 = 1000, kIs1000 = 1000,
// 1'024 items per 1k // 1'024 items per 1k
kIs1024 = 1024 kIs1024 = 1024
}; };
double value; double value;
Flags flags; Flags flags;
OneK oneK; OneK oneK;
BENCHMARK_ALWAYS_INLINE BENCHMARK_ALWAYS_INLINE
Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000) Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000)
: value(v), flags(f), oneK(k) {} : value(v), flags(f), oneK(k) {}
BENCHMARK_ALWAYS_INLINE operator double const&() const { return value; } BENCHMARK_ALWAYS_INLINE operator double const &() const { return value; }
Lint: Pre-merge checks
Inline Actions

clang-format: please reformat the code

-  BENCHMARK_ALWAYS_INLINE operator double const &() const { return value; }
+  BENCHMARK_ALWAYS_INLINE operator double const&() const { return value; }
Lint: Pre-merge checks: clang-format: please reformat the code ``` - BENCHMARK_ALWAYS_INLINE operator double const &()…
BENCHMARK_ALWAYS_INLINE operator double&() { return value; } BENCHMARK_ALWAYS_INLINE operator double&() { return value; }
}; };
// A helper for user code to create unforeseen combinations of Flags, without // A helper for user code to create unforeseen combinations of Flags, without
// having to do this cast manually each time, or providing this operator. // having to do this cast manually each time, or providing this operator.
Counter::Flags inline operator|(const Counter::Flags& LHS, Counter::Flags inline operator|(const Counter::Flags& LHS,
const Counter::Flags& RHS) { const Counter::Flags& RHS) {
return static_cast<Counter::Flags>(static_cast<int>(LHS) | return static_cast<Counter::Flags>(static_cast<int>(LHS) |
Show All 10 Lines
// BigO is passed to a benchmark in order to specify the asymptotic // BigO is passed to a benchmark in order to specify the asymptotic
// computational // computational
// complexity for the benchmark. In case oAuto is selected, complexity will be // complexity for the benchmark. In case oAuto is selected, complexity will be
// calculated automatically to the best fit. // calculated automatically to the best fit.
enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda };
typedef uint64_t IterationCount; typedef uint64_t IterationCount;
enum StatisticUnit { kTime, kPercentage };
// BigOFunc is passed to a benchmark in order to specify the asymptotic // BigOFunc is passed to a benchmark in order to specify the asymptotic
// computational complexity for the benchmark. // computational complexity for the benchmark.
typedef double(BigOFunc)(IterationCount); typedef double(BigOFunc)(IterationCount);
// StatisticsFunc is passed to a benchmark in order to compute some descriptive // StatisticsFunc is passed to a benchmark in order to compute some descriptive
// statistics over all the measurements of some type // statistics over all the measurements of some type
typedef double(StatisticsFunc)(const std::vector<double>&); typedef double(StatisticsFunc)(const std::vector<double>&);
namespace internal { namespace internal {
struct Statistics { struct Statistics {
std::string name_; std::string name_;
StatisticsFunc* compute_; StatisticsFunc* compute_;
StatisticUnit unit_;
Statistics(const std::string& name, StatisticsFunc* compute) Statistics(const std::string& name, StatisticsFunc* compute,
: name_(name), compute_(compute) {} StatisticUnit unit = kTime)
: name_(name), compute_(compute), unit_(unit) {}
}; };
class BenchmarkInstance; class BenchmarkInstance;
class ThreadTimer; class ThreadTimer;
class ThreadManager; class ThreadManager;
class PerfCountersMeasurement; class PerfCountersMeasurement;
enum AggregationReportMode enum AggregationReportMode
▲ Show 20 Lines • Show All 186 Lines • ▼ Show 20 Lines public:
} }
BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead") BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead")
int64_t range_x() const { return range(0); } int64_t range_x() const { return range(0); }
BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead") BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead")
int64_t range_y() const { return range(1); } int64_t range_y() const { return range(1); }
// Number of threads concurrently executing the benchmark.
BENCHMARK_ALWAYS_INLINE
int threads() const { return threads_; }
// Index of the executing thread. Values from [0, threads).
BENCHMARK_ALWAYS_INLINE
int thread_index() const { return thread_index_; }
BENCHMARK_ALWAYS_INLINE BENCHMARK_ALWAYS_INLINE
IterationCount iterations() const { IterationCount iterations() const {
if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) {
return 0; return 0;
} }
return max_iterations - total_iterations_ + batch_leftover_; return max_iterations - total_iterations_ + batch_leftover_;
} }
private private:
: // items we expect on the first cache line (ie 64 bytes of the struct) // items we expect on the first cache line (ie 64 bytes of the struct)
// When total_iterations_ is 0, KeepRunning() and friends will return false. // When total_iterations_ is 0, KeepRunning() and friends will return false.
// May be larger than max_iterations. // May be larger than max_iterations.
IterationCount total_iterations_; IterationCount total_iterations_;
// When using KeepRunningBatch(), batch_leftover_ holds the number of // When using KeepRunningBatch(), batch_leftover_ holds the number of
// iterations beyond max_iters that were run. Used to track // iterations beyond max_iters that were run. Used to track
// completed_iterations_ accurately. // completed_iterations_ accurately.
IterationCount batch_leftover_; IterationCount batch_leftover_;
public: public:
const IterationCount max_iterations; const IterationCount max_iterations;
private: private:
bool started_; bool started_;
bool finished_; bool finished_;
bool error_occurred_; bool error_occurred_;
private: // items we don't need on the first cache line // items we don't need on the first cache line
std::vector<int64_t> range_; std::vector<int64_t> range_;
int64_t complexity_n_; int64_t complexity_n_;
public: public:
// Container for user-defined counters. // Container for user-defined counters.
UserCounters counters; UserCounters counters;
// Index of the executing thread. Values from [0, threads).
const int thread_index;
// Number of threads concurrently executing the benchmark.
const int threads;
private: private:
State(IterationCount max_iters, const std::vector<int64_t>& ranges, State(IterationCount max_iters, const std::vector<int64_t>& ranges,
int thread_i, int n_threads, internal::ThreadTimer* timer, int thread_i, int n_threads, internal::ThreadTimer* timer,
internal::ThreadManager* manager, internal::ThreadManager* manager,
internal::PerfCountersMeasurement* perf_counters_measurement); internal::PerfCountersMeasurement* perf_counters_measurement);
void StartKeepRunning(); void StartKeepRunning();
// Implementation of KeepRunning() and KeepRunningBatch(). // Implementation of KeepRunning() and KeepRunningBatch().
// is_batch must be true unless n is 1. // is_batch must be true unless n is 1.
bool KeepRunningInternal(IterationCount n, bool is_batch); bool KeepRunningInternal(IterationCount n, bool is_batch);
void FinishKeepRunning(); void FinishKeepRunning();
const int thread_index_;
const int threads_;
internal::ThreadTimer* const timer_; internal::ThreadTimer* const timer_;
internal::ThreadManager* const manager_; internal::ThreadManager* const manager_;
internal::PerfCountersMeasurement* const perf_counters_measurement_; internal::PerfCountersMeasurement* const perf_counters_measurement_;
friend class internal::BenchmarkInstance; friend class internal::BenchmarkInstance;
}; };
inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() {
▲ Show 20 Lines • Show All 155 Lines • ▼ Show 20 Lines public:
// New code should use 'Ranges'. // New code should use 'Ranges'.
Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) { Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) {
std::vector<std::pair<int64_t, int64_t> > ranges; std::vector<std::pair<int64_t, int64_t> > ranges;
ranges.push_back(std::make_pair(lo1, hi1)); ranges.push_back(std::make_pair(lo1, hi1));
ranges.push_back(std::make_pair(lo2, hi2)); ranges.push_back(std::make_pair(lo2, hi2));
return Ranges(ranges); return Ranges(ranges);
} }
// Have "setup" and/or "teardown" invoked once for every benchmark run.
// If the benchmark is multi-threaded (will run in k threads concurrently),
// the setup callback will be be invoked exactly once (not k times) before
// each run with k threads. Time allowing (e.g. for a short benchmark), there
// may be multiple such runs per benchmark, each run with its own
// "setup"/"teardown".
//
// If the benchmark uses different size groups of threads (e.g. via
// ThreadRange), the above will be true for each size group.
//
// The callback will be passed a State object, which includes the number
// of threads, thread-index, benchmark arguments, etc.
//
// The callback must not be NULL or self-deleting.
Benchmark* Setup(void (*setup)(const benchmark::State&));
Benchmark* Teardown(void (*teardown)(const benchmark::State&));
// Pass this benchmark object to *func, which can customize // Pass this benchmark object to *func, which can customize
// the benchmark by calling various methods like Arg, Args, // the benchmark by calling various methods like Arg, Args,
// Threads, etc. // Threads, etc.
Benchmark* Apply(void (*func)(Benchmark* benchmark)); Benchmark* Apply(void (*func)(Benchmark* benchmark));
// Set the range multiplier for non-dense range. If not called, the range // Set the range multiplier for non-dense range. If not called, the range
// multiplier kRangeMultiplier will be used. // multiplier kRangeMultiplier will be used.
Benchmark* RangeMultiplier(int multiplier); Benchmark* RangeMultiplier(int multiplier);
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines public:
// the asymptotic computational complexity will be shown on the output. // the asymptotic computational complexity will be shown on the output.
Benchmark* Complexity(BigO complexity = benchmark::oAuto); Benchmark* Complexity(BigO complexity = benchmark::oAuto);
// Set the asymptotic computational complexity for the benchmark. If called // Set the asymptotic computational complexity for the benchmark. If called
// the asymptotic computational complexity will be shown on the output. // the asymptotic computational complexity will be shown on the output.
Benchmark* Complexity(BigOFunc* complexity); Benchmark* Complexity(BigOFunc* complexity);
// Add this statistics to be computed over all the values of benchmark run // Add this statistics to be computed over all the values of benchmark run
Benchmark* ComputeStatistics(std::string name, StatisticsFunc* statistics); Benchmark* ComputeStatistics(const std::string& name,
StatisticsFunc* statistics,
StatisticUnit unit = kTime);
// Support for running multiple copies of the same benchmark concurrently // Support for running multiple copies of the same benchmark concurrently
// in multiple threads. This may be useful when measuring the scaling // in multiple threads. This may be useful when measuring the scaling
// of some piece of code. // of some piece of code.
// Run one instance of this benchmark concurrently in t threads. // Run one instance of this benchmark concurrently in t threads.
Benchmark* Threads(int t); Benchmark* Threads(int t);
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines private:
bool measure_process_cpu_time_; bool measure_process_cpu_time_;
bool use_real_time_; bool use_real_time_;
bool use_manual_time_; bool use_manual_time_;
BigO complexity_; BigO complexity_;
BigOFunc* complexity_lambda_; BigOFunc* complexity_lambda_;
std::vector<Statistics> statistics_; std::vector<Statistics> statistics_;
std::vector<int> thread_counts_; std::vector<int> thread_counts_;
typedef void (*callback_function)(const benchmark::State&);
callback_function setup_;
callback_function teardown_;
Benchmark& operator=(Benchmark const&); Benchmark& operator=(Benchmark const&);
}; };
} // namespace internal } // namespace internal
// Create and register a benchmark with the specified 'name' that invokes // Create and register a benchmark with the specified 'name' that invokes
// the specified functor 'fn'. // the specified functor 'fn'.
// //
Show All 32 Lines
private: private:
template <class OLambda> template <class OLambda>
LambdaBenchmark(const char* name, OLambda&& lam) LambdaBenchmark(const char* name, OLambda&& lam)
: Benchmark(name), lambda_(std::forward<OLambda>(lam)) {} : Benchmark(name), lambda_(std::forward<OLambda>(lam)) {}
LambdaBenchmark(LambdaBenchmark const&) = delete; LambdaBenchmark(LambdaBenchmark const&) = delete;
private: template <class Lam> // NOLINTNEXTLINE(readability-redundant-declaration)
template <class Lam>
friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&);
Lambda lambda_; Lambda lambda_;
}; };
#endif #endif
} // namespace internal } // namespace internal
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines
// empty. If X is empty the expression becomes (+1 == +0). // empty. If X is empty the expression becomes (+1 == +0).
#if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0) #if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0)
#define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__ #define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__
#else #else
#define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__ #define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__
#endif #endif
// Helpers for generating unique variable names // Helpers for generating unique variable names
#ifdef BENCHMARK_HAS_CXX11
#define BENCHMARK_PRIVATE_NAME(...) \
BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, \
__VA_ARGS__)
#else
#define BENCHMARK_PRIVATE_NAME(n) \ #define BENCHMARK_PRIVATE_NAME(n) \
BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, n) BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, n)
#endif // BENCHMARK_HAS_CXX11
#define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) #define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c)
#define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c #define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c
// Helper for concatenation with macro name expansion // Helper for concatenation with macro name expansion
#define BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method) \ #define BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method) \
BaseClass##_##Method##_Benchmark BaseClass##_##Method##_Benchmark
#define BENCHMARK_PRIVATE_DECLARE(n) \ #define BENCHMARK_PRIVATE_DECLARE(n) \
static ::benchmark::internal::Benchmark* BENCHMARK_PRIVATE_NAME(n) \ static ::benchmark::internal::Benchmark* BENCHMARK_PRIVATE_NAME(n) \
BENCHMARK_UNUSED BENCHMARK_UNUSED
#ifdef BENCHMARK_HAS_CXX11
#define BENCHMARK(...) \
BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \
(::benchmark::internal::RegisterBenchmarkInternal( \
new ::benchmark::internal::FunctionBenchmark(#__VA_ARGS__, \
&__VA_ARGS__)))
#else
#define BENCHMARK(n) \ #define BENCHMARK(n) \
BENCHMARK_PRIVATE_DECLARE(n) = \ BENCHMARK_PRIVATE_DECLARE(n) = \
(::benchmark::internal::RegisterBenchmarkInternal( \ (::benchmark::internal::RegisterBenchmarkInternal( \
new ::benchmark::internal::FunctionBenchmark(#n, n))) new ::benchmark::internal::FunctionBenchmark(#n, n)))
#endif // BENCHMARK_HAS_CXX11
// Old-style macros // Old-style macros
#define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) #define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a))
#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)}) #define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)})
#define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) #define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t))
#define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) #define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi))
#define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ #define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \
BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}}) BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}})
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines BENCHMARK_PRIVATE_DECLARE(n) = \
#n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))) #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>)))
#else #else
#define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) #define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a)
#endif #endif
#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ #define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \
class BaseClass##_##Method##_Benchmark : public BaseClass { \ class BaseClass##_##Method##_Benchmark : public BaseClass { \
public: \ public: \
BaseClass##_##Method##_Benchmark() : BaseClass() { \ BaseClass##_##Method##_Benchmark() { \
this->SetName(#BaseClass "/" #Method); \ this->SetName(#BaseClass "/" #Method); \
} \ } \
\ \
protected: \ protected: \
virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \
}; };
#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ #define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \
class BaseClass##_##Method##_Benchmark : public BaseClass<a> { \ class BaseClass##_##Method##_Benchmark : public BaseClass<a> { \
public: \ public: \
BaseClass##_##Method##_Benchmark() : BaseClass<a>() { \ BaseClass##_##Method##_Benchmark() { \
this->SetName(#BaseClass "<" #a ">/" #Method); \ this->SetName(#BaseClass "<" #a ">/" #Method); \
} \ } \
\ \
protected: \ protected: \
virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \
}; };
#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ #define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \
class BaseClass##_##Method##_Benchmark : public BaseClass<a, b> { \ class BaseClass##_##Method##_Benchmark : public BaseClass<a, b> { \
public: \ public: \
BaseClass##_##Method##_Benchmark() : BaseClass<a, b>() { \ BaseClass##_##Method##_Benchmark() { \
this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \
} \ } \
\ \
protected: \ protected: \
virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \
}; };
#ifdef BENCHMARK_HAS_CXX11 #ifdef BENCHMARK_HAS_CXX11
#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, ...) \ #define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, ...) \
class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \ class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \
public: \ public: \
BaseClass##_##Method##_Benchmark() : BaseClass<__VA_ARGS__>() { \ BaseClass##_##Method##_Benchmark() { \
this->SetName(#BaseClass "<" #__VA_ARGS__ ">/" #Method); \ this->SetName(#BaseClass "<" #__VA_ARGS__ ">/" #Method); \
} \ } \
\ \
protected: \ protected: \
virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \
}; };
#else #else
#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ #define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \
▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines
struct CPUInfo { struct CPUInfo {
struct CacheInfo { struct CacheInfo {
std::string type; std::string type;
int level; int level;
int size; int size;
int num_sharing; int num_sharing;
}; };
enum Scaling { enum Scaling { UNKNOWN, ENABLED, DISABLED };
UNKNOWN,
ENABLED,
DISABLED
};
int num_cpus; int num_cpus;
Scaling scaling; Scaling scaling;
double cycles_per_second; double cycles_per_second;
std::vector<CacheInfo> caches; std::vector<CacheInfo> caches;
std::vector<double> load_avg; std::vector<double> load_avg;
static const CPUInfo& Get(); static const CPUInfo& Get();
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines public:
}; };
struct Run { struct Run {
static const int64_t no_repetition_index = -1; static const int64_t no_repetition_index = -1;
enum RunType { RT_Iteration, RT_Aggregate }; enum RunType { RT_Iteration, RT_Aggregate };
Run() Run()
: run_type(RT_Iteration), : run_type(RT_Iteration),
aggregate_unit(kTime),
error_occurred(false), error_occurred(false),
iterations(1), iterations(1),
threads(1), threads(1),
time_unit(kNanosecond), time_unit(kNanosecond),
real_accumulated_time(0), real_accumulated_time(0),
cpu_accumulated_time(0), cpu_accumulated_time(0),
max_heapbytes_used(0), max_heapbytes_used(0),
complexity(oNone), complexity(oNone),
complexity_lambda(), complexity_lambda(),
complexity_n(0), complexity_n(0),
report_big_o(false), report_big_o(false),
report_rms(false), report_rms(false),
counters(), memory_result(NULL),
has_memory_result(false), allocs_per_iter(0.0) {}
allocs_per_iter(0.0),
max_bytes_used(0) {}
std::string benchmark_name() const; std::string benchmark_name() const;
BenchmarkName run_name; BenchmarkName run_name;
int64_t family_index; int64_t family_index;
int64_t per_family_instance_index; int64_t per_family_instance_index;
RunType run_type; RunType run_type;
std::string aggregate_name; std::string aggregate_name;
StatisticUnit aggregate_unit;
std::string report_label; // Empty if not set by benchmark. std::string report_label; // Empty if not set by benchmark.
bool error_occurred; bool error_occurred;
std::string error_message; std::string error_message;
IterationCount iterations; IterationCount iterations;
int64_t threads; int64_t threads;
int64_t repetition_index; int64_t repetition_index;
int64_t repetitions; int64_t repetitions;
Show All 26 Lines struct Run {
// Inform print function whether the current run is a complexity report // Inform print function whether the current run is a complexity report
bool report_big_o; bool report_big_o;
bool report_rms; bool report_rms;
UserCounters counters; UserCounters counters;
// Memory metrics. // Memory metrics.
bool has_memory_result; const MemoryManager::Result* memory_result;
double allocs_per_iter; double allocs_per_iter;
int64_t max_bytes_used;
}; };
struct PerFamilyRunReports { struct PerFamilyRunReports {
PerFamilyRunReports() : num_runs_total(0), num_runs_done(0) {} PerFamilyRunReports() : num_runs_total(0), num_runs_done(0) {}
// How many runs will all instances of this benchmark perform? // How many runs will all instances of this benchmark perform?
int num_runs_total; int num_runs_total;
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines public:
enum OutputOptions { enum OutputOptions {
OO_None = 0, OO_None = 0,
OO_Color = 1, OO_Color = 1,
OO_Tabular = 2, OO_Tabular = 2,
OO_ColorTabular = OO_Color | OO_Tabular, OO_ColorTabular = OO_Color | OO_Tabular,
OO_Defaults = OO_ColorTabular OO_Defaults = OO_ColorTabular
}; };
explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults)
: output_options_(opts_), : output_options_(opts_), name_field_width_(0), printed_header_(false) {}
name_field_width_(0),
prev_counters_(),
printed_header_(false) {}
virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE;
virtual void ReportRuns(const std::vector<Run>& reports) BENCHMARK_OVERRIDE; virtual void ReportRuns(const std::vector<Run>& reports) BENCHMARK_OVERRIDE;
protected: protected:
virtual void PrintRunData(const Run& report); virtual void PrintRunData(const Run& report);
virtual void PrintHeader(const Run& report); virtual void PrintHeader(const Run& report);
Show All 26 Lines
private: private:
void PrintRunData(const Run& report); void PrintRunData(const Run& report);
bool printed_header_; bool printed_header_;
std::set<std::string> user_counter_names_; std::set<std::string> user_counter_names_;
}; };
// If a MemoryManager is registered, it can be used to collect and report
// allocation metrics for a run of the benchmark.
class MemoryManager {
public:
struct Result {
Result() : num_allocs(0), max_bytes_used(0) {}
// The number of allocations made in total between Start and Stop.
int64_t num_allocs;
// The peak memory use between Start and Stop.
int64_t max_bytes_used;
};
virtual ~MemoryManager() {}
// Implement this to start recording allocation information.
virtual void Start() = 0;
// Implement this to stop recording and fill out the given Result structure.
virtual void Stop(Result* result) = 0;
};
inline const char* GetTimeUnitString(TimeUnit unit) { inline const char* GetTimeUnitString(TimeUnit unit) {
switch (unit) { switch (unit) {
case kSecond: case kSecond:
return "s"; return "s";
case kMillisecond: case kMillisecond:
return "ms"; return "ms";
case kMicrosecond: case kMicrosecond:
return "us"; return "us";
Show All 12 Lines switch (unit) {
case kMicrosecond: case kMicrosecond:
return 1e6; return 1e6;
case kNanosecond: case kNanosecond:
return 1e9; return 1e9;
} }
BENCHMARK_UNREACHABLE(); BENCHMARK_UNREACHABLE();
} }
// Creates a list of integer values for the given range and multiplier.
// This can be used together with ArgsProduct() to allow multiple ranges
// with different multiplers.
// Example:
// ArgsProduct({
// CreateRange(0, 1024, /*multi=*/32),
// CreateRange(0, 100, /*multi=*/4),
// CreateDenseRange(0, 4, /*step=*/1),
// });
std::vector<int64_t> CreateRange(int64_t lo, int64_t hi, int multi);
// Creates a list of integer values for the given range and step.
std::vector<int64_t> CreateDenseRange(int64_t start, int64_t limit, int step);
} // namespace benchmark } // namespace benchmark
#endif // BENCHMARK_BENCHMARK_H_ #endif // BENCHMARK_BENCHMARK_H_

third-party/benchmark/requirements.txt

numpy == 1.19.4 numpy == 1.19.4
scipy == 1.5.4 scipy == 1.5.4
pandas == 1.1.5

third-party/benchmark/setup.py

import os import os
import posixpath import posixpath
import platform
import re import re
import shutil import shutil
import sys import sys
from distutils import sysconfig from distutils import sysconfig
import setuptools import setuptools
from setuptools.command import build_ext from setuptools.command import build_ext
▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines def bazel_build(self, ext):
"--symlink_prefix=" + os.path.join(self.build_temp, "bazel-"), "--symlink_prefix=" + os.path.join(self.build_temp, "bazel-"),
"--compilation_mode=" + ("dbg" if self.debug else "opt"), "--compilation_mode=" + ("dbg" if self.debug else "opt"),
] ]
if IS_WINDOWS: if IS_WINDOWS:
# Link with python*.lib. # Link with python*.lib.
for library_dir in self.library_dirs: for library_dir in self.library_dirs:
bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) bazel_argv.append("--linkopt=/LIBPATH:" + library_dir)
elif sys.platform == "darwin" and platform.machine() == "x86_64":
bazel_argv.append("--macos_minimum_os=10.9")
self.spawn(bazel_argv) self.spawn(bazel_argv)
shared_lib_suffix = '.dll' if IS_WINDOWS else '.so' shared_lib_suffix = '.dll' if IS_WINDOWS else '.so'
ext_bazel_bin_path = os.path.join( ext_bazel_bin_path = os.path.join(
self.build_temp, 'bazel-bin', self.build_temp, 'bazel-bin',
ext.relpath, ext.target_name + shared_lib_suffix) ext.relpath, ext.target_name + shared_lib_suffix)
▲ Show 20 Lines • Show All 41 Lines • Show Last 20 Lines

third-party/benchmark/src/CMakeLists.txt

Show All 19 Lines
add_library(benchmark ${SOURCE_FILES}) add_library(benchmark ${SOURCE_FILES})
add_library(benchmark::benchmark ALIAS benchmark) add_library(benchmark::benchmark ALIAS benchmark)
set_target_properties(benchmark PROPERTIES set_target_properties(benchmark PROPERTIES
OUTPUT_NAME "benchmark" OUTPUT_NAME "benchmark"
VERSION ${GENERIC_LIB_VERSION} VERSION ${GENERIC_LIB_VERSION}
SOVERSION ${GENERIC_LIB_SOVERSION} SOVERSION ${GENERIC_LIB_SOVERSION}
) )
target_include_directories(benchmark PUBLIC target_include_directories(benchmark PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>)
)
# libpfm, if available # libpfm, if available
if (HAVE_LIBPFM) if (HAVE_LIBPFM)
target_link_libraries(benchmark libpfm.a) target_link_libraries(benchmark PRIVATE pfm)
add_definitions(-DHAVE_LIBPFM) add_definitions(-DHAVE_LIBPFM)
endif() endif()
# Link threads. # Link threads.
target_link_libraries(benchmark ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(benchmark PRIVATE Threads::Threads)
find_library(LIBRT rt)
if(LIBRT) target_link_libraries(benchmark PRIVATE ${BENCHMARK_CXX_LIBRARIES})
target_link_libraries(benchmark ${LIBRT})
endif() if(HAVE_LIB_RT)
target_link_libraries(benchmark PRIVATE rt)
endif(HAVE_LIB_RT)
if(CMAKE_BUILD_TYPE)
string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER)
endif()
if(NOT CMAKE_THREAD_LIBS_INIT AND "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}" MATCHES ".*-fsanitize=[^ ]*address.*")
message(WARNING "CMake's FindThreads.cmake did not fail, but CMAKE_THREAD_LIBS_INIT ended up being empty. This was fixed in https://github.com/Kitware/CMake/commit/d53317130e84898c5328c237186dbd995aaf1c12 Let's guess that -pthread is sufficient.")
target_link_libraries(benchmark -pthread)
endif()
# We need extra libraries on Windows # We need extra libraries on Windows
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
target_link_libraries(benchmark shlwapi) target_link_libraries(benchmark PRIVATE shlwapi)
endif() endif()
# We need extra libraries on Solaris # We need extra libraries on Solaris
if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS") if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
target_link_libraries(benchmark kstat) target_link_libraries(benchmark PRIVATE kstat)
endif() endif()
# Benchmark main library # Benchmark main library
add_library(benchmark_main "benchmark_main.cc") add_library(benchmark_main "benchmark_main.cc")
add_library(benchmark::benchmark_main ALIAS benchmark_main) add_library(benchmark::benchmark_main ALIAS benchmark_main)
set_target_properties(benchmark_main PROPERTIES set_target_properties(benchmark_main PROPERTIES
OUTPUT_NAME "benchmark_main" OUTPUT_NAME "benchmark_main"
VERSION ${GENERIC_LIB_VERSION} VERSION ${GENERIC_LIB_VERSION}
SOVERSION ${GENERIC_LIB_SOVERSION} SOVERSION ${GENERIC_LIB_SOVERSION}
) )
target_include_directories(benchmark PUBLIC target_link_libraries(benchmark_main PUBLIC benchmark::benchmark)
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>
)
target_link_libraries(benchmark_main benchmark::benchmark)
set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") set(generated_dir "${PROJECT_BINARY_DIR}")
set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake")
set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake") set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake")
set(pkg_config "${generated_dir}/${PROJECT_NAME}.pc") set(pkg_config "${generated_dir}/${PROJECT_NAME}.pc")
set(targets_to_export benchmark benchmark_main)
set(targets_export_name "${PROJECT_NAME}Targets") set(targets_export_name "${PROJECT_NAME}Targets")
set(namespace "${PROJECT_NAME}::") set(namespace "${PROJECT_NAME}::")
include(CMakePackageConfigHelpers) include(CMakePackageConfigHelpers)
configure_package_config_file (
${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in
${project_config}
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
NO_SET_AND_CHECK_MACRO
NO_CHECK_REQUIRED_COMPONENTS_MACRO
)
write_basic_package_version_file( write_basic_package_version_file(
"${version_config}" VERSION ${GENERIC_LIB_VERSION} COMPATIBILITY SameMajorVersion "${version_config}" VERSION ${GENERIC_LIB_VERSION} COMPATIBILITY SameMajorVersion
) )
configure_file("${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in" "${project_config}" @ONLY)
configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark.pc.in" "${pkg_config}" @ONLY) configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark.pc.in" "${pkg_config}" @ONLY)
export (
TARGETS ${targets_to_export}
NAMESPACE "${namespace}"
FILE ${generated_dir}/${targets_export_name}.cmake
)
if (BENCHMARK_ENABLE_INSTALL) if (BENCHMARK_ENABLE_INSTALL)
# Install target (will install the library to specified CMAKE_INSTALL_PREFIX variable) # Install target (will install the library to specified CMAKE_INSTALL_PREFIX variable)
install( install(
TARGETS benchmark benchmark_main TARGETS ${targets_to_export}
EXPORT ${targets_export_name} EXPORT ${targets_export_name}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
install( install(
DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark" DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark"
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
FILES_MATCHING PATTERN "*.*h") FILES_MATCHING PATTERN "*.*h")
install( install(
FILES "${project_config}" "${version_config}" FILES "${project_config}" "${version_config}"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
install( install(
FILES "${pkg_config}" FILES "${pkg_config}"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
install( install(
EXPORT "${targets_export_name}" EXPORT "${targets_export_name}"
NAMESPACE "${namespace}" NAMESPACE "${namespace}"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
endif() endif()
if (BENCHMARK_ENABLE_DOXYGEN)
find_package(Doxygen REQUIRED)
set(DOXYGEN_QUIET YES)
set(DOXYGEN_RECURSIVE YES)
set(DOXYGEN_GENERATE_HTML YES)
set(DOXYGEN_GENERATE_MAN NO)
set(DOXYGEN_MARKDOWN_SUPPORT YES)
set(DOXYGEN_BUILTIN_STL_SUPPORT YES)
set(DOXYGEN_EXTRACT_PACKAGE YES)
set(DOXYGEN_EXTRACT_STATIC YES)
set(DOXYGEN_SHOW_INCLUDE_FILES YES)
set(DOXYGEN_BINARY_TOC YES)
set(DOXYGEN_TOC_EXPAND YES)
set(DOXYGEN_USE_MDFILE_AS_MAINPAGE "index.md")
doxygen_add_docs(benchmark_doxygen
docs
include
src
ALL
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMENT "Building documentation with Doxygen.")
if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS)
install(
DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html/"
DESTINATION ${CMAKE_INSTALL_DOCDIR})
endif()
else()
if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS)
install(
DIRECTORY "${PROJECT_SOURCE_DIR}/docs/"
DESTINATION ${CMAKE_INSTALL_DOCDIR})
endif()
endif()
No newline at end of file

third-party/benchmark/src/benchmark.cc

Show First 20 Lines • Show All 50 Lines • ▼ Show 20 Lines
#include "mutex.h" #include "mutex.h"
#include "perf_counters.h" #include "perf_counters.h"
#include "re.h" #include "re.h"
#include "statistics.h" #include "statistics.h"
#include "string_util.h" #include "string_util.h"
#include "thread_manager.h" #include "thread_manager.h"
#include "thread_timer.h" #include "thread_timer.h"
namespace benchmark {
// Print a list of benchmarks. This option overrides all other options. // Print a list of benchmarks. This option overrides all other options.
DEFINE_bool(benchmark_list_tests, false); BM_DEFINE_bool(benchmark_list_tests, false);
// A regular expression that specifies the set of benchmarks to execute. If // A regular expression that specifies the set of benchmarks to execute. If
// this flag is empty, or if this flag is the string \"all\", all benchmarks // this flag is empty, or if this flag is the string \"all\", all benchmarks
// linked into the binary are run. // linked into the binary are run.
DEFINE_string(benchmark_filter, "."); BM_DEFINE_string(benchmark_filter, "");
// Minimum number of seconds we should run benchmark before results are // Minimum number of seconds we should run benchmark before results are
// considered significant. For cpu-time based tests, this is the lower bound // considered significant. For cpu-time based tests, this is the lower bound
// on the total cpu time used by all threads that make up the test. For // on the total cpu time used by all threads that make up the test. For
// real-time based tests, this is the lower bound on the elapsed time of the // real-time based tests, this is the lower bound on the elapsed time of the
// benchmark execution, regardless of number of threads. // benchmark execution, regardless of number of threads.
DEFINE_double(benchmark_min_time, 0.5); BM_DEFINE_double(benchmark_min_time, 0.5);
// The number of runs of each benchmark. If greater than 1, the mean and // The number of runs of each benchmark. If greater than 1, the mean and
// standard deviation of the runs will be reported. // standard deviation of the runs will be reported.
DEFINE_int32(benchmark_repetitions, 1); BM_DEFINE_int32(benchmark_repetitions, 1);
// If set, enable random interleaving of repetitions of all benchmarks. // If set, enable random interleaving of repetitions of all benchmarks.
// See http://github.com/google/benchmark/issues/1051 for details. // See http://github.com/google/benchmark/issues/1051 for details.
DEFINE_bool(benchmark_enable_random_interleaving, false); BM_DEFINE_bool(benchmark_enable_random_interleaving, false);
// Report the result of each benchmark repetitions. When 'true' is specified // Report the result of each benchmark repetitions. When 'true' is specified
// only the mean, standard deviation, and other statistics are reported for // only the mean, standard deviation, and other statistics are reported for
// repeated benchmarks. Affects all reporters. // repeated benchmarks. Affects all reporters.
DEFINE_bool(benchmark_report_aggregates_only, false); BM_DEFINE_bool(benchmark_report_aggregates_only, false);
// Display the result of each benchmark repetitions. When 'true' is specified // Display the result of each benchmark repetitions. When 'true' is specified
// only the mean, standard deviation, and other statistics are displayed for // only the mean, standard deviation, and other statistics are displayed for
// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects // repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects
// the display reporter, but *NOT* file reporter, which will still contain // the display reporter, but *NOT* file reporter, which will still contain
// all the output. // all the output.
DEFINE_bool(benchmark_display_aggregates_only, false); BM_DEFINE_bool(benchmark_display_aggregates_only, false);
// The format to use for console output. // The format to use for console output.
// Valid values are 'console', 'json', or 'csv'. // Valid values are 'console', 'json', or 'csv'.
DEFINE_string(benchmark_format, "console"); BM_DEFINE_string(benchmark_format, "console");
// The format to use for file output. // The format to use for file output.
// Valid values are 'console', 'json', or 'csv'. // Valid values are 'console', 'json', or 'csv'.
DEFINE_string(benchmark_out_format, "json"); BM_DEFINE_string(benchmark_out_format, "json");
// The file to write additional output to. // The file to write additional output to.
DEFINE_string(benchmark_out, ""); BM_DEFINE_string(benchmark_out, "");
// Whether to use colors in the output. Valid values: // Whether to use colors in the output. Valid values:
// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if // 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if
// the output is being sent to a terminal and the TERM environment variable is // the output is being sent to a terminal and the TERM environment variable is
// set to a terminal type that supports colors. // set to a terminal type that supports colors.
DEFINE_string(benchmark_color, "auto"); BM_DEFINE_string(benchmark_color, "auto");
// Whether to use tabular format when printing user counters to the console. // Whether to use tabular format when printing user counters to the console.
// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. // Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false.
DEFINE_bool(benchmark_counters_tabular, false); BM_DEFINE_bool(benchmark_counters_tabular, false);
// The level of verbose logging to output
DEFINE_int32(v, 0);
// List of additional perf counters to collect, in libpfm format. For more // List of additional perf counters to collect, in libpfm format. For more
// information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html // information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html
DEFINE_string(benchmark_perf_counters, ""); BM_DEFINE_string(benchmark_perf_counters, "");
namespace benchmark {
namespace internal {
// Extra context to include in the output formatted as comma-separated key-value // Extra context to include in the output formatted as comma-separated key-value
// pairs. Kept internal as it's only used for parsing from env/command line. // pairs. Kept internal as it's only used for parsing from env/command line.
DEFINE_kvpairs(benchmark_context, {}); BM_DEFINE_kvpairs(benchmark_context, {});
// The level of verbose logging to output
BM_DEFINE_int32(v, 0);
namespace internal {
std::map<std::string, std::string>* global_context = nullptr; std::map<std::string, std::string>* global_context = nullptr;
// FIXME: wouldn't LTO mess this up? // FIXME: wouldn't LTO mess this up?
void UseCharPointer(char const volatile*) {} void UseCharPointer(char const volatile*) {}
} // namespace internal } // namespace internal
State::State(IterationCount max_iters, const std::vector<int64_t>& ranges, State::State(IterationCount max_iters, const std::vector<int64_t>& ranges,
int thread_i, int n_threads, internal::ThreadTimer* timer, int thread_i, int n_threads, internal::ThreadTimer* timer,
internal::ThreadManager* manager, internal::ThreadManager* manager,
internal::PerfCountersMeasurement* perf_counters_measurement) internal::PerfCountersMeasurement* perf_counters_measurement)
: total_iterations_(0), : total_iterations_(0),
batch_leftover_(0), batch_leftover_(0),
max_iterations(max_iters), max_iterations(max_iters),
started_(false), started_(false),
finished_(false), finished_(false),
error_occurred_(false), error_occurred_(false),
range_(ranges), range_(ranges),
complexity_n_(0), complexity_n_(0),
counters(), thread_index_(thread_i),
thread_index(thread_i), threads_(n_threads),
threads(n_threads),
timer_(timer), timer_(timer),
manager_(manager), manager_(manager),
perf_counters_measurement_(perf_counters_measurement) { perf_counters_measurement_(perf_counters_measurement) {
CHECK(max_iterations != 0) << "At least one iteration must be run"; BM_CHECK(max_iterations != 0) << "At least one iteration must be run";
CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; BM_CHECK_LT(thread_index_, threads_)
<< "thread_index must be less than threads";
// Note: The use of offsetof below is technically undefined until C++17 // Note: The use of offsetof below is technically undefined until C++17
// because State is not a standard layout type. However, all compilers // because State is not a standard layout type. However, all compilers
// currently provide well-defined behavior as an extension (which is // currently provide well-defined behavior as an extension (which is
// demonstrated since constexpr evaluation must diagnose all undefined // demonstrated since constexpr evaluation must diagnose all undefined
// behavior). However, GCC and Clang also warn about this use of offsetof, // behavior). However, GCC and Clang also warn about this use of offsetof,
// which must be suppressed. // which must be suppressed.
#if defined(__INTEL_COMPILER) #if defined(__INTEL_COMPILER)
Show All 12 Lines
#pragma warning pop #pragma warning pop
#elif defined(__GNUC__) #elif defined(__GNUC__)
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
#endif #endif
} }
void State::PauseTiming() { void State::PauseTiming() {
// Add in time accumulated so far // Add in time accumulated so far
CHECK(started_ && !finished_ && !error_occurred_); BM_CHECK(started_ && !finished_ && !error_occurred_);
timer_->StopTimer(); timer_->StopTimer();
if (perf_counters_measurement_) { if (perf_counters_measurement_) {
auto measurements = perf_counters_measurement_->StopAndGetMeasurements(); auto measurements = perf_counters_measurement_->StopAndGetMeasurements();
for (const auto& name_and_measurement : measurements) { for (const auto& name_and_measurement : measurements) {
auto name = name_and_measurement.first; auto name = name_and_measurement.first;
auto measurement = name_and_measurement.second; auto measurement = name_and_measurement.second;
CHECK_EQ(counters[name], 0.0); BM_CHECK_EQ(counters[name], 0.0);
counters[name] = Counter(measurement, Counter::kAvgIterations); counters[name] = Counter(measurement, Counter::kAvgIterations);
} }
} }
} }
void State::ResumeTiming() { void State::ResumeTiming() {
CHECK(started_ && !finished_ && !error_occurred_); BM_CHECK(started_ && !finished_ && !error_occurred_);
timer_->StartTimer(); timer_->StartTimer();
if (perf_counters_measurement_) { if (perf_counters_measurement_) {
perf_counters_measurement_->Start(); perf_counters_measurement_->Start();
} }
} }
void State::SkipWithError(const char* msg) { void State::SkipWithError(const char* msg) {
CHECK(msg); BM_CHECK(msg);
error_occurred_ = true; error_occurred_ = true;
{ {
MutexLock l(manager_->GetBenchmarkMutex()); MutexLock l(manager_->GetBenchmarkMutex());
if (manager_->results.has_error_ == false) { if (manager_->results.has_error_ == false) {
manager_->results.error_message_ = msg; manager_->results.error_message_ = msg;
manager_->results.has_error_ = true; manager_->results.has_error_ = true;
} }
} }
total_iterations_ = 0; total_iterations_ = 0;
if (timer_->running()) timer_->StopTimer(); if (timer_->running()) timer_->StopTimer();
} }
void State::SetIterationTime(double seconds) { void State::SetIterationTime(double seconds) {
timer_->SetIterationTime(seconds); timer_->SetIterationTime(seconds);
} }
void State::SetLabel(const char* label) { void State::SetLabel(const char* label) {
MutexLock l(manager_->GetBenchmarkMutex()); MutexLock l(manager_->GetBenchmarkMutex());
manager_->results.report_label_ = label; manager_->results.report_label_ = label;
} }
void State::StartKeepRunning() { void State::StartKeepRunning() {
CHECK(!started_ && !finished_); BM_CHECK(!started_ && !finished_);
started_ = true; started_ = true;
total_iterations_ = error_occurred_ ? 0 : max_iterations; total_iterations_ = error_occurred_ ? 0 : max_iterations;
manager_->StartStopBarrier(); manager_->StartStopBarrier();
if (!error_occurred_) ResumeTiming(); if (!error_occurred_) ResumeTiming();
} }
void State::FinishKeepRunning() { void State::FinishKeepRunning() {
CHECK(started_ && (!finished_ || error_occurred_)); BM_CHECK(started_ && (!finished_ || error_occurred_));
if (!error_occurred_) { if (!error_occurred_) {
PauseTiming(); PauseTiming();
} }
// Total iterations has now wrapped around past 0. Fix this. // Total iterations has now wrapped around past 0. Fix this.
total_iterations_ = 0; total_iterations_ = 0;
finished_ = true; finished_ = true;
manager_->StartStopBarrier(); manager_->StartStopBarrier();
} }
Show All 31 Linesvoid Report(BenchmarkReporter* display_reporter,
FlushStreams(display_reporter); FlushStreams(display_reporter);
FlushStreams(file_reporter); FlushStreams(file_reporter);
} }
void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks, void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
BenchmarkReporter* display_reporter, BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter) { BenchmarkReporter* file_reporter) {
// Note the file_reporter can be null. // Note the file_reporter can be null.
CHECK(display_reporter != nullptr); BM_CHECK(display_reporter != nullptr);
// Determine the width of the name field using a minimum width of 10. // Determine the width of the name field using a minimum width of 10.
bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;
size_t name_field_width = 10; size_t name_field_width = 10;
size_t stat_field_width = 0; size_t stat_field_width = 0;
for (const BenchmarkInstance& benchmark : benchmarks) { for (const BenchmarkInstance& benchmark : benchmarks) {
name_field_width = name_field_width =
std::max<size_t>(name_field_width, benchmark.name().str().size()); std::max<size_t>(name_field_width, benchmark.name().str().size());
Show All 29 Lines for (const BenchmarkInstance& benchmark : benchmarks) {
runners.emplace_back(benchmark, reports_for_family); runners.emplace_back(benchmark, reports_for_family);
int num_repeats_of_this_instance = runners.back().GetNumRepeats(); int num_repeats_of_this_instance = runners.back().GetNumRepeats();
num_repetitions_total += num_repeats_of_this_instance; num_repetitions_total += num_repeats_of_this_instance;
if (reports_for_family) if (reports_for_family)
reports_for_family->num_runs_total += num_repeats_of_this_instance; reports_for_family->num_runs_total += num_repeats_of_this_instance;
} }
assert(runners.size() == benchmarks.size() && "Unexpected runner count."); assert(runners.size() == benchmarks.size() && "Unexpected runner count.");
std::vector<int> repetition_indices; std::vector<size_t> repetition_indices;
repetition_indices.reserve(num_repetitions_total); repetition_indices.reserve(num_repetitions_total);
for (size_t runner_index = 0, num_runners = runners.size(); for (size_t runner_index = 0, num_runners = runners.size();
runner_index != num_runners; ++runner_index) { runner_index != num_runners; ++runner_index) {
const internal::BenchmarkRunner& runner = runners[runner_index]; const internal::BenchmarkRunner& runner = runners[runner_index];
std::fill_n(std::back_inserter(repetition_indices), std::fill_n(std::back_inserter(repetition_indices),
runner.GetNumRepeats(), runner_index); runner.GetNumRepeats(), runner_index);
} }
assert(repetition_indices.size() == num_repetitions_total && assert(repetition_indices.size() == num_repetitions_total &&
Show All 17 Lines for (size_t repetition_index : repetition_indices) {
if (const auto* reports_for_family = runner.GetReportsForFamily()) { if (const auto* reports_for_family = runner.GetReportsForFamily()) {
if (reports_for_family->num_runs_done == if (reports_for_family->num_runs_done ==
reports_for_family->num_runs_total) { reports_for_family->num_runs_total) {
auto additional_run_stats = ComputeBigO(reports_for_family->Runs); auto additional_run_stats = ComputeBigO(reports_for_family->Runs);
run_results.aggregates_only.insert(run_results.aggregates_only.end(), run_results.aggregates_only.insert(run_results.aggregates_only.end(),
additional_run_stats.begin(), additional_run_stats.begin(),
additional_run_stats.end()); additional_run_stats.end());
per_family_reports.erase( per_family_reports.erase(
(int)reports_for_family->Runs.front().family_index); static_cast<int>(reports_for_family->Runs.front().family_index));
} }
} }
Report(display_reporter, file_reporter, run_results); Report(display_reporter, file_reporter, run_results);
} }
} }
display_reporter->Finalize(); display_reporter->Finalize();
if (file_reporter) file_reporter->Finalize(); if (file_reporter) file_reporter->Finalize();
FlushStreams(display_reporter); FlushStreams(display_reporter);
FlushStreams(file_reporter); FlushStreams(file_reporter);
} }
// Disable deprecated warnings temporarily because we need to reference // Disable deprecated warnings temporarily because we need to reference
// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations // CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations
#ifdef __GNUC__ BENCHMARK_DISABLE_DEPRECATED_WARNING
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
std::unique_ptr<BenchmarkReporter> CreateReporter( std::unique_ptr<BenchmarkReporter> CreateReporter(
std::string const& name, ConsoleReporter::OutputOptions output_opts) { std::string const& name, ConsoleReporter::OutputOptions output_opts) {
typedef std::unique_ptr<BenchmarkReporter> PtrType; typedef std::unique_ptr<BenchmarkReporter> PtrType;
if (name == "console") { if (name == "console") {
return PtrType(new ConsoleReporter(output_opts)); return PtrType(new ConsoleReporter(output_opts));
} else if (name == "json") { } else if (name == "json") {
return PtrType(new JSONReporter); return PtrType(new JSONReporter);
} else if (name == "csv") { } else if (name == "csv") {
return PtrType(new CSVReporter); return PtrType(new CSVReporter);
} else { } else {
std::cerr << "Unexpected format: '" << name << "'\n"; std::cerr << "Unexpected format: '" << name << "'\n";
std::exit(1); std::exit(1);
} }
} }
#ifdef __GNUC__ BENCHMARK_RESTORE_DEPRECATED_WARNING
#pragma GCC diagnostic pop
#endif
} // end namespace } // end namespace
bool IsZero(double n) { bool IsZero(double n) {
return std::abs(n) < std::numeric_limits<double>::epsilon(); return std::abs(n) < std::numeric_limits<double>::epsilon();
} }
ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) {
Show All 18 Lines if (FLAGS_benchmark_counters_tabular) {
output_opts &= ~ConsoleReporter::OO_Tabular; output_opts &= ~ConsoleReporter::OO_Tabular;
} }
return static_cast<ConsoleReporter::OutputOptions>(output_opts); return static_cast<ConsoleReporter::OutputOptions>(output_opts);
} }
} // end namespace internal } // end namespace internal
size_t RunSpecifiedBenchmarks() { size_t RunSpecifiedBenchmarks() {
return RunSpecifiedBenchmarks(nullptr, nullptr); return RunSpecifiedBenchmarks(nullptr, nullptr, FLAGS_benchmark_filter);
}
size_t RunSpecifiedBenchmarks(std::string spec) {
return RunSpecifiedBenchmarks(nullptr, nullptr, std::move(spec));
} }
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) { size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) {
return RunSpecifiedBenchmarks(display_reporter, nullptr); return RunSpecifiedBenchmarks(display_reporter, nullptr,
FLAGS_benchmark_filter);
}
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
std::string spec) {
return RunSpecifiedBenchmarks(display_reporter, nullptr, std::move(spec));
} }
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter) { BenchmarkReporter* file_reporter) {
std::string spec = FLAGS_benchmark_filter; return RunSpecifiedBenchmarks(display_reporter, file_reporter,
FLAGS_benchmark_filter);
}
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter,
std::string spec) {
if (spec.empty() || spec == "all") if (spec.empty() || spec == "all")
spec = "."; // Regexp that matches all benchmarks spec = "."; // Regexp that matches all benchmarks
// Setup the reporters // Setup the reporters
std::ofstream output_file; std::ofstream output_file;
std::unique_ptr<BenchmarkReporter> default_display_reporter; std::unique_ptr<BenchmarkReporter> default_display_reporter;
std::unique_ptr<BenchmarkReporter> default_file_reporter; std::unique_ptr<BenchmarkReporter> default_file_reporter;
if (!display_reporter) { if (!display_reporter) {
Show All 39 Lines for (auto const& benchmark : benchmarks)
Out << benchmark.name().str() << "\n"; Out << benchmark.name().str() << "\n";
} else { } else {
internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); internal::RunBenchmarks(benchmarks, display_reporter, file_reporter);
} }
return benchmarks.size(); return benchmarks.size();
} }
std::string GetBenchmarkFilter() { return FLAGS_benchmark_filter; }
void RegisterMemoryManager(MemoryManager* manager) { void RegisterMemoryManager(MemoryManager* manager) {
internal::memory_manager = manager; internal::memory_manager = manager;
} }
void AddCustomContext(const std::string& key, const std::string& value) { void AddCustomContext(const std::string& key, const std::string& value) {
if (internal::global_context == nullptr) { if (internal::global_context == nullptr) {
internal::global_context = new std::map<std::string, std::string>(); internal::global_context = new std::map<std::string, std::string>();
} }
Show All 15 Lines fprintf(stdout,
" [--benchmark_enable_random_interleaving={true|false}]\n" " [--benchmark_enable_random_interleaving={true|false}]\n"
" [--benchmark_report_aggregates_only={true|false}]\n" " [--benchmark_report_aggregates_only={true|false}]\n"
" [--benchmark_display_aggregates_only={true|false}]\n" " [--benchmark_display_aggregates_only={true|false}]\n"
" [--benchmark_format=<console|json|csv>]\n" " [--benchmark_format=<console|json|csv>]\n"
" [--benchmark_out=<filename>]\n" " [--benchmark_out=<filename>]\n"
" [--benchmark_out_format=<json|console|csv>]\n" " [--benchmark_out_format=<json|console|csv>]\n"
" [--benchmark_color={auto|true|false}]\n" " [--benchmark_color={auto|true|false}]\n"
" [--benchmark_counters_tabular={true|false}]\n" " [--benchmark_counters_tabular={true|false}]\n"
" [--benchmark_perf_counters=<counter>,...]\n"
" [--benchmark_context=<key>=<value>,...]\n" " [--benchmark_context=<key>=<value>,...]\n"
" [--v=<verbosity>]\n"); " [--v=<verbosity>]\n");
exit(0); exit(0);
} }
void ParseCommandLineFlags(int* argc, char** argv) { void ParseCommandLineFlags(int* argc, char** argv) {
using namespace benchmark; using namespace benchmark;
BenchmarkReporter::Context::executable_name = BenchmarkReporter::Context::executable_name =
Show All 12 Lines if (ParseBoolFlag(argv[i], "benchmark_list_tests",
&FLAGS_benchmark_report_aggregates_only) || &FLAGS_benchmark_report_aggregates_only) ||
ParseBoolFlag(argv[i], "benchmark_display_aggregates_only", ParseBoolFlag(argv[i], "benchmark_display_aggregates_only",
&FLAGS_benchmark_display_aggregates_only) || &FLAGS_benchmark_display_aggregates_only) ||
ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) ||
ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) || ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) ||
ParseStringFlag(argv[i], "benchmark_out_format", ParseStringFlag(argv[i], "benchmark_out_format",
&FLAGS_benchmark_out_format) || &FLAGS_benchmark_out_format) ||
ParseStringFlag(argv[i], "benchmark_color", &FLAGS_benchmark_color) || ParseStringFlag(argv[i], "benchmark_color", &FLAGS_benchmark_color) ||
// "color_print" is the deprecated name for "benchmark_color".
// TODO: Remove this.
ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) ||
ParseBoolFlag(argv[i], "benchmark_counters_tabular", ParseBoolFlag(argv[i], "benchmark_counters_tabular",
&FLAGS_benchmark_counters_tabular) || &FLAGS_benchmark_counters_tabular) ||
ParseStringFlag(argv[i], "benchmark_perf_counters", ParseStringFlag(argv[i], "benchmark_perf_counters",
&FLAGS_benchmark_perf_counters) || &FLAGS_benchmark_perf_counters) ||
ParseKeyValueFlag(argv[i], "benchmark_context", ParseKeyValueFlag(argv[i], "benchmark_context",
&FLAGS_benchmark_context) || &FLAGS_benchmark_context) ||
ParseInt32Flag(argv[i], "v", &FLAGS_v)) { ParseInt32Flag(argv[i], "v", &FLAGS_v)) {
for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1];
Show All 25 Lines
} // end namespace internal } // end namespace internal
void Initialize(int* argc, char** argv) { void Initialize(int* argc, char** argv) {
internal::ParseCommandLineFlags(argc, argv); internal::ParseCommandLineFlags(argc, argv);
internal::LogLevel() = FLAGS_v; internal::LogLevel() = FLAGS_v;
} }
void Shutdown() { void Shutdown() { delete internal::global_context; }
delete internal::global_context;
}
bool ReportUnrecognizedArguments(int argc, char** argv) { bool ReportUnrecognizedArguments(int argc, char** argv) {
for (int i = 1; i < argc; ++i) { for (int i = 1; i < argc; ++i) {
fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0], fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0],
argv[i]); argv[i]);
} }
return argc > 1; return argc > 1;
} }
} // end namespace benchmark } // end namespace benchmark

third-party/benchmark/src/benchmark_api_internal.h

Show All 26 Lines public:
AggregationReportMode aggregation_report_mode() const { AggregationReportMode aggregation_report_mode() const {
return aggregation_report_mode_; return aggregation_report_mode_;
} }
TimeUnit time_unit() const { return time_unit_; } TimeUnit time_unit() const { return time_unit_; }
bool measure_process_cpu_time() const { return measure_process_cpu_time_; } bool measure_process_cpu_time() const { return measure_process_cpu_time_; }
bool use_real_time() const { return use_real_time_; } bool use_real_time() const { return use_real_time_; }
bool use_manual_time() const { return use_manual_time_; } bool use_manual_time() const { return use_manual_time_; }
BigO complexity() const { return complexity_; } BigO complexity() const { return complexity_; }
BigOFunc& complexity_lambda() const { return *complexity_lambda_; } BigOFunc* complexity_lambda() const { return complexity_lambda_; }
const std::vector<Statistics>& statistics() const { return statistics_; } const std::vector<Statistics>& statistics() const { return statistics_; }
int repetitions() const { return repetitions_; } int repetitions() const { return repetitions_; }
double min_time() const { return min_time_; } double min_time() const { return min_time_; }
IterationCount iterations() const { return iterations_; } IterationCount iterations() const { return iterations_; }
int threads() const { return threads_; } int threads() const { return threads_; }
void Setup() const;
void Teardown() const;
State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer, State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer,
internal::ThreadManager* manager, internal::ThreadManager* manager,
internal::PerfCountersMeasurement* perf_counters_measurement) const; internal::PerfCountersMeasurement* perf_counters_measurement) const;
private: private:
BenchmarkName name_; BenchmarkName name_;
Benchmark& benchmark_; Benchmark& benchmark_;
const int family_index_; const int family_index_;
const int per_family_instance_index_; const int per_family_instance_index_;
AggregationReportMode aggregation_report_mode_; AggregationReportMode aggregation_report_mode_;
const std::vector<int64_t>& args_; const std::vector<int64_t>& args_;
TimeUnit time_unit_; TimeUnit time_unit_;
bool measure_process_cpu_time_; bool measure_process_cpu_time_;
bool use_real_time_; bool use_real_time_;
bool use_manual_time_; bool use_manual_time_;
BigO complexity_; BigO complexity_;
BigOFunc* complexity_lambda_; BigOFunc* complexity_lambda_;
UserCounters counters_; UserCounters counters_;
const std::vector<Statistics>& statistics_; const std::vector<Statistics>& statistics_;
int repetitions_; int repetitions_;
double min_time_; double min_time_;
IterationCount iterations_; IterationCount iterations_;
int threads_; // Number of concurrent threads to us int threads_; // Number of concurrent threads to us
typedef void (*callback_function)(const benchmark::State&);
callback_function setup_ = nullptr;
callback_function teardown_ = nullptr;
}; };
bool FindBenchmarksInternal(const std::string& re, bool FindBenchmarksInternal(const std::string& re,
std::vector<BenchmarkInstance>* benchmarks, std::vector<BenchmarkInstance>* benchmarks,
std::ostream* Err); std::ostream* Err);
bool IsZero(double n); bool IsZero(double n);
ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false); ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false);
} // end namespace internal } // end namespace internal
} // end namespace benchmark } // end namespace benchmark
#endif // BENCHMARK_API_INTERNAL_H #endif // BENCHMARK_API_INTERNAL_H

third-party/benchmark/src/benchmark_api_internal.cc

Show First 20 Lines • Show All 72 Lines • ▼ Show 20 Lines if (!name_.time_type.empty()) {
name_.time_type += '/'; name_.time_type += '/';
} }
name_.time_type += "real_time"; name_.time_type += "real_time";
} }
if (!benchmark_.thread_counts_.empty()) { if (!benchmark_.thread_counts_.empty()) {
name_.threads = StrFormat("threads:%d", threads_); name_.threads = StrFormat("threads:%d", threads_);
} }
setup_ = benchmark_.setup_;
teardown_ = benchmark_.teardown_;
} }
State BenchmarkInstance::Run( State BenchmarkInstance::Run(
IterationCount iters, int thread_id, internal::ThreadTimer* timer, IterationCount iters, int thread_id, internal::ThreadTimer* timer,
internal::ThreadManager* manager, internal::ThreadManager* manager,
internal::PerfCountersMeasurement* perf_counters_measurement) const { internal::PerfCountersMeasurement* perf_counters_measurement) const {
State st(iters, args_, thread_id, threads_, timer, manager, State st(iters, args_, thread_id, threads_, timer, manager,
perf_counters_measurement); perf_counters_measurement);
benchmark_.Run(st); benchmark_.Run(st);
return st; return st;
} }
void BenchmarkInstance::Setup() const {
if (setup_) {
State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr,
nullptr);
setup_(st);
}
}
void BenchmarkInstance::Teardown() const {
if (teardown_) {
State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr,
nullptr);
teardown_(st);
}
}
} // namespace internal } // namespace internal
} // namespace benchmark } // namespace benchmark

third-party/benchmark/src/benchmark_register.h

#ifndef BENCHMARK_REGISTER_H #ifndef BENCHMARK_REGISTER_H
#define BENCHMARK_REGISTER_H #define BENCHMARK_REGISTER_H
#include <limits> #include <limits>
#include <vector> #include <vector>
#include "check.h" #include "check.h"
namespace benchmark { namespace benchmark {
namespace internal { namespace internal {
// Append the powers of 'mult' in the closed interval [lo, hi]. // Append the powers of 'mult' in the closed interval [lo, hi].
// Returns iterator to the start of the inserted range. // Returns iterator to the start of the inserted range.
template <typename T> template <typename T>
typename std::vector<T>::iterator typename std::vector<T>::iterator AddPowers(std::vector<T>* dst, T lo, T hi,
AddPowers(std::vector<T>* dst, T lo, T hi, int mult) { int mult) {
CHECK_GE(lo, 0); BM_CHECK_GE(lo, 0);
CHECK_GE(hi, lo); BM_CHECK_GE(hi, lo);
CHECK_GE(mult, 2); BM_CHECK_GE(mult, 2);
const size_t start_offset = dst->size(); const size_t start_offset = dst->size();
static const T kmax = std::numeric_limits<T>::max(); static const T kmax = std::numeric_limits<T>::max();
// Space out the values in multiples of "mult" // Space out the values in multiples of "mult"
for (T i = static_cast<T>(1); i <= hi; i *= mult) { for (T i = static_cast<T>(1); i <= hi; i *= mult) {
if (i >= lo) { if (i >= lo) {
dst->push_back(i); dst->push_back(i);
} }
// Break the loop here since multiplying by // Break the loop here since multiplying by
// 'mult' would move outside of the range of T // 'mult' would move outside of the range of T
if (i > kmax / mult) break; if (i > kmax / mult) break;
} }
return dst->begin() + start_offset; return dst->begin() + start_offset;
} }
template <typename T> template <typename T>
void AddNegatedPowers(std::vector<T>* dst, T lo, T hi, int mult) { void AddNegatedPowers(std::vector<T>* dst, T lo, T hi, int mult) {
// We negate lo and hi so we require that they cannot be equal to 'min'. // We negate lo and hi so we require that they cannot be equal to 'min'.
CHECK_GT(lo, std::numeric_limits<T>::min()); BM_CHECK_GT(lo, std::numeric_limits<T>::min());
CHECK_GT(hi, std::numeric_limits<T>::min()); BM_CHECK_GT(hi, std::numeric_limits<T>::min());
CHECK_GE(hi, lo); BM_CHECK_GE(hi, lo);
CHECK_LE(hi, 0); BM_CHECK_LE(hi, 0);
// Add positive powers, then negate and reverse. // Add positive powers, then negate and reverse.
// Casts necessary since small integers get promoted // Casts necessary since small integers get promoted
// to 'int' when negating. // to 'int' when negating.
const auto lo_complement = static_cast<T>(-lo); const auto lo_complement = static_cast<T>(-lo);
const auto hi_complement = static_cast<T>(-hi); const auto hi_complement = static_cast<T>(-hi);
const auto it = AddPowers(dst, hi_complement, lo_complement, mult); const auto it = AddPowers(dst, hi_complement, lo_complement, mult);
std::for_each(it, dst->end(), [](T& t) { t *= -1; }); std::for_each(it, dst->end(), [](T& t) { t *= -1; });
std::reverse(it, dst->end()); std::reverse(it, dst->end());
} }
template <typename T> template <typename T>
void AddRange(std::vector<T>* dst, T lo, T hi, int mult) { void AddRange(std::vector<T>* dst, T lo, T hi, int mult) {
static_assert(std::is_integral<T>::value && std::is_signed<T>::value, static_assert(std::is_integral<T>::value && std::is_signed<T>::value,
"Args type must be a signed integer"); "Args type must be a signed integer");
CHECK_GE(hi, lo); BM_CHECK_GE(hi, lo);
CHECK_GE(mult, 2); BM_CHECK_GE(mult, 2);
// Add "lo" // Add "lo"
dst->push_back(lo); dst->push_back(lo);
// Handle lo == hi as a special case, so we then know // Handle lo == hi as a special case, so we then know
// lo < hi and so it is safe to add 1 to lo and subtract 1 // lo < hi and so it is safe to add 1 to lo and subtract 1
// from hi without falling outside of the range of T. // from hi without falling outside of the range of T.
if (lo == hi) return; if (lo == hi) return;
Show All 36 Lines

third-party/benchmark/src/benchmark_register.cc

Show First 20 Lines • Show All 105 Lines • ▼ Show 20 Linesvoid BenchmarkFamilies::ClearBenchmarks() {
MutexLock l(mutex_); MutexLock l(mutex_);
families_.clear(); families_.clear();
families_.shrink_to_fit(); families_.shrink_to_fit();
} }
bool BenchmarkFamilies::FindBenchmarks( bool BenchmarkFamilies::FindBenchmarks(
std::string spec, std::vector<BenchmarkInstance>* benchmarks, std::string spec, std::vector<BenchmarkInstance>* benchmarks,
std::ostream* ErrStream) { std::ostream* ErrStream) {
CHECK(ErrStream); BM_CHECK(ErrStream);
auto& Err = *ErrStream; auto& Err = *ErrStream;
// Make regular expression out of command-line flag // Make regular expression out of command-line flag
std::string error_msg; std::string error_msg;
Regex re; Regex re;
bool isNegativeFilter = false; bool isNegativeFilter = false;
if (spec[0] == '-') { if (spec[0] == '-') {
spec.replace(0, 1, ""); spec.replace(0, 1, "");
isNegativeFilter = true; isNegativeFilter = true;
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines : name_(name),
range_multiplier_(kRangeMultiplier), range_multiplier_(kRangeMultiplier),
min_time_(0), min_time_(0),
iterations_(0), iterations_(0),
repetitions_(0), repetitions_(0),
measure_process_cpu_time_(false), measure_process_cpu_time_(false),
use_real_time_(false), use_real_time_(false),
use_manual_time_(false), use_manual_time_(false),
complexity_(oNone), complexity_(oNone),
complexity_lambda_(nullptr) { complexity_lambda_(nullptr),
setup_(nullptr),
teardown_(nullptr) {
ComputeStatistics("mean", StatisticsMean); ComputeStatistics("mean", StatisticsMean);
ComputeStatistics("median", StatisticsMedian); ComputeStatistics("median", StatisticsMedian);
ComputeStatistics("stddev", StatisticsStdDev); ComputeStatistics("stddev", StatisticsStdDev);
ComputeStatistics("cv", StatisticsCV, kPercentage);
} }
Benchmark::~Benchmark() {} Benchmark::~Benchmark() {}
Benchmark* Benchmark::Name(const std::string& name) { Benchmark* Benchmark::Name(const std::string& name) {
SetName(name.c_str()); SetName(name.c_str());
return this; return this;
} }
Benchmark* Benchmark::Arg(int64_t x) { Benchmark* Benchmark::Arg(int64_t x) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
args_.push_back({x}); args_.push_back({x});
return this; return this;
} }
Benchmark* Benchmark::Unit(TimeUnit unit) { Benchmark* Benchmark::Unit(TimeUnit unit) {
time_unit_ = unit; time_unit_ = unit;
return this; return this;
} }
Benchmark* Benchmark::Range(int64_t start, int64_t limit) { Benchmark* Benchmark::Range(int64_t start, int64_t limit) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
std::vector<int64_t> arglist; std::vector<int64_t> arglist;
AddRange(&arglist, start, limit, range_multiplier_); AddRange(&arglist, start, limit, range_multiplier_);
for (int64_t i : arglist) { for (int64_t i : arglist) {
args_.push_back({i}); args_.push_back({i});
} }
return this; return this;
} }
Benchmark* Benchmark::Ranges( Benchmark* Benchmark::Ranges(
const std::vector<std::pair<int64_t, int64_t>>& ranges) { const std::vector<std::pair<int64_t, int64_t>>& ranges) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(ranges.size())); BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(ranges.size()));
std::vector<std::vector<int64_t>> arglists(ranges.size()); std::vector<std::vector<int64_t>> arglists(ranges.size());
for (std::size_t i = 0; i < ranges.size(); i++) { for (std::size_t i = 0; i < ranges.size(); i++) {
AddRange(&arglists[i], ranges[i].first, ranges[i].second, AddRange(&arglists[i], ranges[i].first, ranges[i].second,
range_multiplier_); range_multiplier_);
} }
ArgsProduct(arglists); ArgsProduct(arglists);
return this; return this;
} }
Benchmark* Benchmark::ArgsProduct( Benchmark* Benchmark::ArgsProduct(
const std::vector<std::vector<int64_t>>& arglists) { const std::vector<std::vector<int64_t>>& arglists) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(arglists.size())); BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(arglists.size()));
std::vector<std::size_t> indices(arglists.size()); std::vector<std::size_t> indices(arglists.size());
const std::size_t total = std::accumulate( const std::size_t total = std::accumulate(
std::begin(arglists), std::end(arglists), std::size_t{1}, std::begin(arglists), std::end(arglists), std::size_t{1},
[](const std::size_t res, const std::vector<int64_t>& arglist) { [](const std::size_t res, const std::vector<int64_t>& arglist) {
return res * arglist.size(); return res * arglist.size();
}); });
std::vector<int64_t> args; std::vector<int64_t> args;
Show All 10 Lines do {
indices[arg] = (indices[arg] + 1) % arglists[arg].size(); indices[arg] = (indices[arg] + 1) % arglists[arg].size();
} while (indices[arg++] == 0 && arg < arglists.size()); } while (indices[arg++] == 0 && arg < arglists.size());
} }
return this; return this;
} }
Benchmark* Benchmark::ArgName(const std::string& name) { Benchmark* Benchmark::ArgName(const std::string& name) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
arg_names_ = {name}; arg_names_ = {name};
return this; return this;
} }
Benchmark* Benchmark::ArgNames(const std::vector<std::string>& names) { Benchmark* Benchmark::ArgNames(const std::vector<std::string>& names) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(names.size())); BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(names.size()));
arg_names_ = names; arg_names_ = names;
return this; return this;
} }
Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) { Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
CHECK_LE(start, limit); BM_CHECK_LE(start, limit);
for (int64_t arg = start; arg <= limit; arg += step) { for (int64_t arg = start; arg <= limit; arg += step) {
args_.push_back({arg}); args_.push_back({arg});
} }
return this; return this;
} }
Benchmark* Benchmark::Args(const std::vector<int64_t>& args) { Benchmark* Benchmark::Args(const std::vector<int64_t>& args) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(args.size())); BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast<int>(args.size()));
args_.push_back(args); args_.push_back(args);
return this; return this;
} }
Benchmark* Benchmark::Apply(void (*custom_arguments)(Benchmark* benchmark)) { Benchmark* Benchmark::Apply(void (*custom_arguments)(Benchmark* benchmark)) {
custom_arguments(this); custom_arguments(this);
return this; return this;
} }
Benchmark* Benchmark::Setup(void (*setup)(const benchmark::State&)) {
BM_CHECK(setup != nullptr);
setup_ = setup;
return this;
}
Benchmark* Benchmark::Teardown(void (*teardown)(const benchmark::State&)) {
BM_CHECK(teardown != nullptr);
teardown_ = teardown;
return this;
}
Benchmark* Benchmark::RangeMultiplier(int multiplier) { Benchmark* Benchmark::RangeMultiplier(int multiplier) {
CHECK(multiplier > 1); BM_CHECK(multiplier > 1);
range_multiplier_ = multiplier; range_multiplier_ = multiplier;
return this; return this;
} }
Benchmark* Benchmark::MinTime(double t) { Benchmark* Benchmark::MinTime(double t) {
CHECK(t > 0.0); BM_CHECK(t > 0.0);
CHECK(iterations_ == 0); BM_CHECK(iterations_ == 0);
min_time_ = t; min_time_ = t;
return this; return this;
} }
Benchmark* Benchmark::Iterations(IterationCount n) { Benchmark* Benchmark::Iterations(IterationCount n) {
CHECK(n > 0); BM_CHECK(n > 0);
CHECK(IsZero(min_time_)); BM_CHECK(IsZero(min_time_));
iterations_ = n; iterations_ = n;
return this; return this;
} }
Benchmark* Benchmark::Repetitions(int n) { Benchmark* Benchmark::Repetitions(int n) {
CHECK(n > 0); BM_CHECK(n > 0);
repetitions_ = n; repetitions_ = n;
return this; return this;
} }
Benchmark* Benchmark::ReportAggregatesOnly(bool value) { Benchmark* Benchmark::ReportAggregatesOnly(bool value) {
aggregation_report_mode_ = value ? ARM_ReportAggregatesOnly : ARM_Default; aggregation_report_mode_ = value ? ARM_ReportAggregatesOnly : ARM_Default;
return this; return this;
} }
Show All 16 Lines
Benchmark* Benchmark::MeasureProcessCPUTime() { Benchmark* Benchmark::MeasureProcessCPUTime() {
// Can be used together with UseRealTime() / UseManualTime(). // Can be used together with UseRealTime() / UseManualTime().
measure_process_cpu_time_ = true; measure_process_cpu_time_ = true;
return this; return this;
} }
Benchmark* Benchmark::UseRealTime() { Benchmark* Benchmark::UseRealTime() {
CHECK(!use_manual_time_) BM_CHECK(!use_manual_time_)
<< "Cannot set UseRealTime and UseManualTime simultaneously."; << "Cannot set UseRealTime and UseManualTime simultaneously.";
use_real_time_ = true; use_real_time_ = true;
return this; return this;
} }
Benchmark* Benchmark::UseManualTime() { Benchmark* Benchmark::UseManualTime() {
CHECK(!use_real_time_) BM_CHECK(!use_real_time_)
<< "Cannot set UseRealTime and UseManualTime simultaneously."; << "Cannot set UseRealTime and UseManualTime simultaneously.";
use_manual_time_ = true; use_manual_time_ = true;
return this; return this;
} }
Benchmark* Benchmark::Complexity(BigO complexity) { Benchmark* Benchmark::Complexity(BigO complexity) {
complexity_ = complexity; complexity_ = complexity;
return this; return this;
} }
Benchmark* Benchmark::Complexity(BigOFunc* complexity) { Benchmark* Benchmark::Complexity(BigOFunc* complexity) {
complexity_lambda_ = complexity; complexity_lambda_ = complexity;
complexity_ = oLambda; complexity_ = oLambda;
return this; return this;
} }
Benchmark* Benchmark::ComputeStatistics(std::string name, Benchmark* Benchmark::ComputeStatistics(const std::string& name,
StatisticsFunc* statistics) { StatisticsFunc* statistics,
statistics_.emplace_back(name, statistics); StatisticUnit unit) {
statistics_.emplace_back(name, statistics, unit);
return this; return this;
} }
Benchmark* Benchmark::Threads(int t) { Benchmark* Benchmark::Threads(int t) {
CHECK_GT(t, 0); BM_CHECK_GT(t, 0);
thread_counts_.push_back(t); thread_counts_.push_back(t);
return this; return this;
} }
Benchmark* Benchmark::ThreadRange(int min_threads, int max_threads) { Benchmark* Benchmark::ThreadRange(int min_threads, int max_threads) {
CHECK_GT(min_threads, 0); BM_CHECK_GT(min_threads, 0);
CHECK_GE(max_threads, min_threads); BM_CHECK_GE(max_threads, min_threads);
AddRange(&thread_counts_, min_threads, max_threads, 2); AddRange(&thread_counts_, min_threads, max_threads, 2);
return this; return this;
} }
Benchmark* Benchmark::DenseThreadRange(int min_threads, int max_threads, Benchmark* Benchmark::DenseThreadRange(int min_threads, int max_threads,
int stride) { int stride) {
CHECK_GT(min_threads, 0); BM_CHECK_GT(min_threads, 0);
CHECK_GE(max_threads, min_threads); BM_CHECK_GE(max_threads, min_threads);
CHECK_GE(stride, 1); BM_CHECK_GE(stride, 1);
for (auto i = min_threads; i < max_threads; i += stride) { for (auto i = min_threads; i < max_threads; i += stride) {
thread_counts_.push_back(i); thread_counts_.push_back(i);
} }
thread_counts_.push_back(max_threads); thread_counts_.push_back(max_threads);
return this; return this;
} }
Show All 19 Lines
void FunctionBenchmark::Run(State& st) { func_(st); } void FunctionBenchmark::Run(State& st) { func_(st); }
} // end namespace internal } // end namespace internal
void ClearRegisteredBenchmarks() { void ClearRegisteredBenchmarks() {
internal::BenchmarkFamilies::GetInstance()->ClearBenchmarks(); internal::BenchmarkFamilies::GetInstance()->ClearBenchmarks();
} }
std::vector<int64_t> CreateRange(int64_t lo, int64_t hi, int multi) {
std::vector<int64_t> args;
internal::AddRange(&args, lo, hi, multi);
return args;
}
std::vector<int64_t> CreateDenseRange(int64_t start, int64_t limit, int step) {
BM_CHECK_LE(start, limit);
std::vector<int64_t> args;
for (int64_t arg = start; arg <= limit; arg += step) {
args.push_back(arg);
}
return args;
}
} // end namespace benchmark } // end namespace benchmark

third-party/benchmark/src/benchmark_runner.h

Show All 17 Lines
#include <thread> #include <thread>
#include <vector> #include <vector>
#include "benchmark_api_internal.h" #include "benchmark_api_internal.h"
#include "internal_macros.h" #include "internal_macros.h"
#include "perf_counters.h" #include "perf_counters.h"
#include "thread_manager.h" #include "thread_manager.h"
DECLARE_double(benchmark_min_time);
DECLARE_int32(benchmark_repetitions);
DECLARE_bool(benchmark_report_aggregates_only);
DECLARE_bool(benchmark_display_aggregates_only);
DECLARE_string(benchmark_perf_counters);
namespace benchmark { namespace benchmark {
BM_DECLARE_double(benchmark_min_time);
BM_DECLARE_int32(benchmark_repetitions);
BM_DECLARE_bool(benchmark_report_aggregates_only);
BM_DECLARE_bool(benchmark_display_aggregates_only);
BM_DECLARE_string(benchmark_perf_counters);
namespace internal { namespace internal {
extern MemoryManager* memory_manager; extern MemoryManager* memory_manager;
struct RunResults { struct RunResults {
std::vector<BenchmarkReporter::Run> non_aggregates; std::vector<BenchmarkReporter::Run> non_aggregates;
std::vector<BenchmarkReporter::Run> aggregates_only; std::vector<BenchmarkReporter::Run> aggregates_only;
Show All 13 Lines public:
} }
void DoOneRepetition(); void DoOneRepetition();
RunResults&& GetResults(); RunResults&& GetResults();
BenchmarkReporter::PerFamilyRunReports* GetReportsForFamily() const { BenchmarkReporter::PerFamilyRunReports* GetReportsForFamily() const {
return reports_for_family; return reports_for_family;
}; }
private: private:
RunResults run_results; RunResults run_results;
const benchmark::internal::BenchmarkInstance& b; const benchmark::internal::BenchmarkInstance& b;
BenchmarkReporter::PerFamilyRunReports* reports_for_family; BenchmarkReporter::PerFamilyRunReports* reports_for_family;
const double min_time; const double min_time;
const int repeats; const int repeats;
const bool has_explicit_iteration_count; const bool has_explicit_iteration_count;
int num_repetitions_done = 0; int num_repetitions_done = 0;
std::vector<std::thread> pool; std::vector<std::thread> pool;
std::vector<MemoryManager::Result> memory_results;
IterationCount iters; // preserved between repetitions! IterationCount iters; // preserved between repetitions!
// So only the first repetition has to find/calculate it, // So only the first repetition has to find/calculate it,
// the other repetitions will just use that precomputed iteration count. // the other repetitions will just use that precomputed iteration count.
PerfCountersMeasurement perf_counters_measurement; PerfCountersMeasurement perf_counters_measurement;
PerfCountersMeasurement* const perf_counters_measurement_ptr; PerfCountersMeasurement* const perf_counters_measurement_ptr;
struct IterationResults { struct IterationResults {
Show All 16 Lines

third-party/benchmark/src/benchmark_runner.cc

Show First 20 Lines • Show All 61 Lines • ▼ Show 20 Lines
namespace { namespace {
static constexpr IterationCount kMaxIterations = 1000000000; static constexpr IterationCount kMaxIterations = 1000000000;
BenchmarkReporter::Run CreateRunReport( BenchmarkReporter::Run CreateRunReport(
const benchmark::internal::BenchmarkInstance& b, const benchmark::internal::BenchmarkInstance& b,
const internal::ThreadManager::Result& results, const internal::ThreadManager::Result& results,
IterationCount memory_iterations, IterationCount memory_iterations,
const MemoryManager::Result& memory_result, double seconds, const MemoryManager::Result* memory_result, double seconds,
int64_t repetition_index, int64_t repeats) { int64_t repetition_index, int64_t repeats) {
// Create report about this benchmark run. // Create report about this benchmark run.
BenchmarkReporter::Run report; BenchmarkReporter::Run report;
report.run_name = b.name(); report.run_name = b.name();
report.family_index = b.family_index(); report.family_index = b.family_index();
report.per_family_instance_index = b.per_family_instance_index(); report.per_family_instance_index = b.per_family_instance_index();
report.error_occurred = results.has_error_; report.error_occurred = results.has_error_;
Show All 15 Lines if (!report.error_occurred) {
report.cpu_accumulated_time = results.cpu_time_used; report.cpu_accumulated_time = results.cpu_time_used;
report.complexity_n = results.complexity_n; report.complexity_n = results.complexity_n;
report.complexity = b.complexity(); report.complexity = b.complexity();
report.complexity_lambda = b.complexity_lambda(); report.complexity_lambda = b.complexity_lambda();
report.statistics = &b.statistics(); report.statistics = &b.statistics();
report.counters = results.counters; report.counters = results.counters;
if (memory_iterations > 0) { if (memory_iterations > 0) {
report.has_memory_result = true; assert(memory_result != nullptr);
report.memory_result = memory_result;
report.allocs_per_iter = report.allocs_per_iter =
memory_iterations ? static_cast<double>(memory_result.num_allocs) / memory_iterations ? static_cast<double>(memory_result->num_allocs) /
memory_iterations memory_iterations
: 0; : 0;
report.max_bytes_used = memory_result.max_bytes_used;
} }
internal::Finish(&report.counters, results.iterations, seconds, internal::Finish(&report.counters, results.iterations, seconds,
b.threads()); b.threads());
} }
return report; return report;
} }
// Execute one thread of benchmark b for the specified number of iterations. // Execute one thread of benchmark b for the specified number of iterations.
// Adds the stats collected for the thread into manager->results. // Adds the stats collected for the thread into manager->results.
void RunInThread(const BenchmarkInstance* b, IterationCount iters, void RunInThread(const BenchmarkInstance* b, IterationCount iters,
int thread_id, ThreadManager* manager, int thread_id, ThreadManager* manager,
PerfCountersMeasurement* perf_counters_measurement) { PerfCountersMeasurement* perf_counters_measurement) {
internal::ThreadTimer timer( internal::ThreadTimer timer(
b->measure_process_cpu_time() b->measure_process_cpu_time()
? internal::ThreadTimer::CreateProcessCpuTime() ? internal::ThreadTimer::CreateProcessCpuTime()
: internal::ThreadTimer::Create()); : internal::ThreadTimer::Create());
State st = State st =
b->Run(iters, thread_id, &timer, manager, perf_counters_measurement); b->Run(iters, thread_id, &timer, manager, perf_counters_measurement);
CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) BM_CHECK(st.error_occurred() || st.iterations() >= st.max_iterations)
<< "Benchmark returned before State::KeepRunning() returned false!"; << "Benchmark returned before State::KeepRunning() returned false!";
{ {
MutexLock l(manager->GetBenchmarkMutex()); MutexLock l(manager->GetBenchmarkMutex());
internal::ThreadManager::Result& results = manager->results; internal::ThreadManager::Result& results = manager->results;
results.iterations += st.iterations(); results.iterations += st.iterations();
results.cpu_time_used += timer.cpu_time_used(); results.cpu_time_used += timer.cpu_time_used();
results.real_time_used += timer.real_time_used(); results.real_time_used += timer.real_time_used();
results.manual_time_used += timer.manual_time_used(); results.manual_time_used += timer.manual_time_used();
Show All 27 LinesBenchmarkRunner::BenchmarkRunner(
run_results.file_report_aggregates_only = run_results.file_report_aggregates_only =
FLAGS_benchmark_report_aggregates_only; FLAGS_benchmark_report_aggregates_only;
if (b.aggregation_report_mode() != internal::ARM_Unspecified) { if (b.aggregation_report_mode() != internal::ARM_Unspecified) {
run_results.display_report_aggregates_only = run_results.display_report_aggregates_only =
(b.aggregation_report_mode() & (b.aggregation_report_mode() &
internal::ARM_DisplayReportAggregatesOnly); internal::ARM_DisplayReportAggregatesOnly);
run_results.file_report_aggregates_only = run_results.file_report_aggregates_only =
(b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly);
CHECK(FLAGS_benchmark_perf_counters.empty() || BM_CHECK(FLAGS_benchmark_perf_counters.empty() ||
perf_counters_measurement.IsValid()) perf_counters_measurement.IsValid())
<< "Perf counters were requested but could not be set up."; << "Perf counters were requested but could not be set up.";
} }
} }
BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() {
VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; BM_VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n";
std::unique_ptr<internal::ThreadManager> manager; std::unique_ptr<internal::ThreadManager> manager;
manager.reset(new internal::ThreadManager(b.threads())); manager.reset(new internal::ThreadManager(b.threads()));
// Run all but one thread in separate threads // Run all but one thread in separate threads
for (std::size_t ti = 0; ti < pool.size(); ++ti) { for (std::size_t ti = 0; ti < pool.size(); ++ti) {
pool[ti] = std::thread(&RunInThread, &b, iters, static_cast<int>(ti + 1), pool[ti] = std::thread(&RunInThread, &b, iters, static_cast<int>(ti + 1),
manager.get(), perf_counters_measurement_ptr); manager.get(), perf_counters_measurement_ptr);
Show All 18 LinesBenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() {
manager.reset(); manager.reset();
// Adjust real/manual time stats since they were reported per thread. // Adjust real/manual time stats since they were reported per thread.
i.results.real_time_used /= b.threads(); i.results.real_time_used /= b.threads();
i.results.manual_time_used /= b.threads(); i.results.manual_time_used /= b.threads();
// If we were measuring whole-process CPU usage, adjust the CPU time too. // If we were measuring whole-process CPU usage, adjust the CPU time too.
if (b.measure_process_cpu_time()) i.results.cpu_time_used /= b.threads(); if (b.measure_process_cpu_time()) i.results.cpu_time_used /= b.threads();
VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" BM_VLOG(2) << "Ran in " << i.results.cpu_time_used << "/"
<< i.results.real_time_used << "\n"; << i.results.real_time_used << "\n";
// By using KeepRunningBatch a benchmark can iterate more times than // By using KeepRunningBatch a benchmark can iterate more times than
// requested, so take the iteration count from i.results. // requested, so take the iteration count from i.results.
i.iters = i.results.iterations / b.threads(); i.iters = i.results.iterations / b.threads();
// Base decisions off of real time if requested by this benchmark. // Base decisions off of real time if requested by this benchmark.
i.seconds = i.results.cpu_time_used; i.seconds = i.results.cpu_time_used;
if (b.use_manual_time()) { if (b.use_manual_time()) {
Show All 11 LinesIterationCount BenchmarkRunner::PredictNumItersNeeded(
// Note: Avoid division by zero with max(seconds, 1ns). // Note: Avoid division by zero with max(seconds, 1ns).
double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9);
// If our last run was at least 10% of FLAGS_benchmark_min_time then we // If our last run was at least 10% of FLAGS_benchmark_min_time then we
// use the multiplier directly. // use the multiplier directly.
// Otherwise we use at most 10 times expansion. // Otherwise we use at most 10 times expansion.
// NOTE: When the last run was at least 10% of the min time the max // NOTE: When the last run was at least 10% of the min time the max
// expansion should be 14x. // expansion should be 14x.
bool is_significant = (i.seconds / min_time) > 0.1; bool is_significant = (i.seconds / min_time) > 0.1;
multiplier = is_significant ? multiplier : std::min(10.0, multiplier); multiplier = is_significant ? multiplier : 10.0;
if (multiplier <= 1.0) multiplier = 2.0;
// So what seems to be the sufficiently-large iteration count? Round up. // So what seems to be the sufficiently-large iteration count? Round up.
const IterationCount max_next_iters = static_cast<IterationCount>( const IterationCount max_next_iters = static_cast<IterationCount>(
std::lround(std::max(multiplier * static_cast<double>(i.iters), std::lround(std::max(multiplier * static_cast<double>(i.iters),
static_cast<double>(i.iters) + 1.0))); static_cast<double>(i.iters) + 1.0)));
// But we do have *some* sanity limits though.. // But we do have *some* sanity limits though..
const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); const IterationCount next_iters = std::min(max_next_iters, kMaxIterations);
VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; BM_VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n";
return next_iters; // round up before conversion to integer. return next_iters; // round up before conversion to integer.
} }
bool BenchmarkRunner::ShouldReportIterationResults( bool BenchmarkRunner::ShouldReportIterationResults(
const IterationResults& i) const { const IterationResults& i) const {
// Determine if this run should be reported; // Determine if this run should be reported;
// Either it has run for a sufficient amount of time // Either it has run for a sufficient amount of time
// or because an error was reported. // or because an error was reported.
Show All 14 Linesvoid BenchmarkRunner::DoOneRepetition() {
// We *may* be gradually increasing the length (iteration count) // We *may* be gradually increasing the length (iteration count)
// of the benchmark until we decide the results are significant. // of the benchmark until we decide the results are significant.
// And once we do, we report those last results and exit. // And once we do, we report those last results and exit.
// Please do note that the if there are repetitions, the iteration count // Please do note that the if there are repetitions, the iteration count
// is *only* calculated for the *first* repetition, and other repetitions // is *only* calculated for the *first* repetition, and other repetitions
// simply use that precomputed iteration count. // simply use that precomputed iteration count.
for (;;) { for (;;) {
b.Setup();
i = DoNIterations(); i = DoNIterations();
b.Teardown();
// Do we consider the results to be significant? // Do we consider the results to be significant?
// If we are doing repetitions, and the first repetition was already done, // If we are doing repetitions, and the first repetition was already done,
// it has calculated the correct iteration time, so we have run that very // it has calculated the correct iteration time, so we have run that very
// iteration count just now. No need to calculate anything. Just report. // iteration count just now. No need to calculate anything. Just report.
// Else, the normal rules apply. // Else, the normal rules apply.
const bool results_are_significant = !is_the_first_repetition || const bool results_are_significant = !is_the_first_repetition ||
has_explicit_iteration_count || has_explicit_iteration_count ||
ShouldReportIterationResults(i); ShouldReportIterationResults(i);
if (results_are_significant) break; // Good, let's report them! if (results_are_significant) break; // Good, let's report them!
// Nope, bad iteration. Let's re-estimate the hopefully-sufficient // Nope, bad iteration. Let's re-estimate the hopefully-sufficient
// iteration count, and run the benchmark again... // iteration count, and run the benchmark again...
iters = PredictNumItersNeeded(i); iters = PredictNumItersNeeded(i);
assert(iters > i.iters && assert(iters > i.iters &&
"if we did more iterations than we want to do the next time, " "if we did more iterations than we want to do the next time, "
"then we should have accepted the current iteration run."); "then we should have accepted the current iteration run.");
} }
// Oh, one last thing, we need to also produce the 'memory measurements'.. // Oh, one last thing, we need to also produce the 'memory measurements'..
MemoryManager::Result memory_result; MemoryManager::Result* memory_result = nullptr;
IterationCount memory_iterations = 0; IterationCount memory_iterations = 0;
if (memory_manager != nullptr) { if (memory_manager != nullptr) {
// TODO(vyng): Consider making BenchmarkReporter::Run::memory_result an
// optional so we don't have to own the Result here.
// Can't do it now due to cxx03.
memory_results.push_back(MemoryManager::Result());
memory_result = &memory_results.back();
// Only run a few iterations to reduce the impact of one-time // Only run a few iterations to reduce the impact of one-time
// allocations in benchmarks that are not properly managed. // allocations in benchmarks that are not properly managed.
memory_iterations = std::min<IterationCount>(16, iters); memory_iterations = std::min<IterationCount>(16, iters);
memory_manager->Start(); memory_manager->Start();
std::unique_ptr<internal::ThreadManager> manager; std::unique_ptr<internal::ThreadManager> manager;
manager.reset(new internal::ThreadManager(1)); manager.reset(new internal::ThreadManager(1));
b.Setup();
RunInThread(&b, memory_iterations, 0, manager.get(), RunInThread(&b, memory_iterations, 0, manager.get(),
perf_counters_measurement_ptr); perf_counters_measurement_ptr);
manager->WaitForAllThreads(); manager->WaitForAllThreads();
manager.reset(); manager.reset();
b.Teardown();
memory_manager->Stop(&memory_result); BENCHMARK_DISABLE_DEPRECATED_WARNING
memory_manager->Stop(memory_result);
BENCHMARK_RESTORE_DEPRECATED_WARNING
} }
// Ok, now actualy report. // Ok, now actually report.
BenchmarkReporter::Run report = BenchmarkReporter::Run report =
CreateRunReport(b, i.results, memory_iterations, memory_result, i.seconds, CreateRunReport(b, i.results, memory_iterations, memory_result, i.seconds,
num_repetitions_done, repeats); num_repetitions_done, repeats);
if (reports_for_family) { if (reports_for_family) {
++reports_for_family->num_runs_done; ++reports_for_family->num_runs_done;
if (!report.error_occurred) reports_for_family->Runs.push_back(report); if (!report.error_occurred) reports_for_family->Runs.push_back(report);
} }
Show All 18 Lines

third-party/benchmark/src/check.h

Show All 17 Linesinline AbortHandlerT*& GetAbortHandler() {
return handler; return handler;
} }
BENCHMARK_NORETURN inline void CallAbortHandler() { BENCHMARK_NORETURN inline void CallAbortHandler() {
GetAbortHandler()(); GetAbortHandler()();
std::abort(); // fallback to enforce noreturn std::abort(); // fallback to enforce noreturn
} }
// CheckHandler is the class constructed by failing CHECK macros. CheckHandler // CheckHandler is the class constructed by failing BM_CHECK macros.
// will log information about the failures and abort when it is destructed. // CheckHandler will log information about the failures and abort when it is
// destructed.
class CheckHandler { class CheckHandler {
public: public:
CheckHandler(const char* check, const char* file, const char* func, int line) CheckHandler(const char* check, const char* file, const char* func, int line)
: log_(GetErrorLogInstance()) { : log_(GetErrorLogInstance()) {
log_ << file << ":" << line << ": " << func << ": Check `" << check log_ << file << ":" << line << ": " << func << ": Check `" << check
<< "' failed. "; << "' failed. ";
} }
Show All 10 Lines
private: private:
LogType& log_; LogType& log_;
}; };
} // end namespace internal } // end namespace internal
} // end namespace benchmark } // end namespace benchmark
// The CHECK macro returns a std::ostream object that can have extra information // The BM_CHECK macro returns a std::ostream object that can have extra
// written to it. // information written to it.
#ifndef NDEBUG #ifndef NDEBUG
#define CHECK(b) \ #define BM_CHECK(b) \
(b ? ::benchmark::internal::GetNullLogInstance() \ (b ? ::benchmark::internal::GetNullLogInstance() \
: ::benchmark::internal::CheckHandler(#b, __FILE__, __func__, __LINE__) \ : ::benchmark::internal::CheckHandler(#b, __FILE__, __func__, __LINE__) \
.GetLog()) .GetLog())
#else #else
#define CHECK(b) ::benchmark::internal::GetNullLogInstance() #define BM_CHECK(b) ::benchmark::internal::GetNullLogInstance()
#endif #endif
// clang-format off // clang-format off
// preserve whitespacing between operators for alignment // preserve whitespacing between operators for alignment
#define CHECK_EQ(a, b) CHECK((a) == (b)) #define BM_CHECK_EQ(a, b) BM_CHECK((a) == (b))
#define CHECK_NE(a, b) CHECK((a) != (b)) #define BM_CHECK_NE(a, b) BM_CHECK((a) != (b))
#define CHECK_GE(a, b) CHECK((a) >= (b)) #define BM_CHECK_GE(a, b) BM_CHECK((a) >= (b))
#define CHECK_LE(a, b) CHECK((a) <= (b)) #define BM_CHECK_LE(a, b) BM_CHECK((a) <= (b))
#define CHECK_GT(a, b) CHECK((a) > (b)) #define BM_CHECK_GT(a, b) BM_CHECK((a) > (b))
#define CHECK_LT(a, b) CHECK((a) < (b)) #define BM_CHECK_LT(a, b) BM_CHECK((a) < (b))
#define CHECK_FLOAT_EQ(a, b, eps) CHECK(std::fabs((a) - (b)) < (eps)) #define BM_CHECK_FLOAT_EQ(a, b, eps) BM_CHECK(std::fabs((a) - (b)) < (eps))
#define CHECK_FLOAT_NE(a, b, eps) CHECK(std::fabs((a) - (b)) >= (eps)) #define BM_CHECK_FLOAT_NE(a, b, eps) BM_CHECK(std::fabs((a) - (b)) >= (eps))
#define CHECK_FLOAT_GE(a, b, eps) CHECK((a) - (b) > -(eps)) #define BM_CHECK_FLOAT_GE(a, b, eps) BM_CHECK((a) - (b) > -(eps))
#define CHECK_FLOAT_LE(a, b, eps) CHECK((b) - (a) > -(eps)) #define BM_CHECK_FLOAT_LE(a, b, eps) BM_CHECK((b) - (a) > -(eps))
#define CHECK_FLOAT_GT(a, b, eps) CHECK((a) - (b) > (eps)) #define BM_CHECK_FLOAT_GT(a, b, eps) BM_CHECK((a) - (b) > (eps))
#define CHECK_FLOAT_LT(a, b, eps) CHECK((b) - (a) > (eps)) #define BM_CHECK_FLOAT_LT(a, b, eps) BM_CHECK((b) - (a) > (eps))
//clang-format on //clang-format on
#endif // CHECK_H_ #endif // CHECK_H_

third-party/benchmark/src/colorprint.cc

Show All 19 Lines
#include <cstring> #include <cstring>
#include <memory> #include <memory>
#include <string> #include <string>
#include "check.h" #include "check.h"
#include "internal_macros.h" #include "internal_macros.h"
#ifdef BENCHMARK_OS_WINDOWS #ifdef BENCHMARK_OS_WINDOWS
#include <windows.h>
#include <io.h> #include <io.h>
#include <windows.h>
#else #else
#include <unistd.h> #include <unistd.h>
#endif // BENCHMARK_OS_WINDOWS #endif // BENCHMARK_OS_WINDOWS
namespace benchmark { namespace benchmark {
namespace { namespace {
#ifdef BENCHMARK_OS_WINDOWS #ifdef BENCHMARK_OS_WINDOWS
typedef WORD PlatformColorCode; typedef WORD PlatformColorCode;
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Linesstd::string FormatString(const char* msg, va_list args) {
std::size_t size = 256; std::size_t size = 256;
char local_buff[256]; char local_buff[256];
auto ret = vsnprintf(local_buff, size, msg, args_cp); auto ret = vsnprintf(local_buff, size, msg, args_cp);
va_end(args_cp); va_end(args_cp);
// currently there is no error handling for failure, so this is hack. // currently there is no error handling for failure, so this is hack.
CHECK(ret >= 0); BM_CHECK(ret >= 0);
if (ret == 0) // handle empty expansion if (ret == 0) // handle empty expansion
return {}; return {};
else if (static_cast<size_t>(ret) < size) else if (static_cast<size_t>(ret) < size)
return local_buff; return local_buff;
else { else {
// we did not provide a long enough buffer on our first attempt. // we did not provide a long enough buffer on our first attempt.
size = (size_t)ret + 1; // + 1 for the null byte size = static_cast<size_t>(ret) + 1; // + 1 for the null byte
std::unique_ptr<char[]> buff(new char[size]); std::unique_ptr<char[]> buff(new char[size]);
ret = vsnprintf(buff.get(), size, msg, args); ret = vsnprintf(buff.get(), size, msg, args);
CHECK(ret > 0 && ((size_t)ret) < size); BM_CHECK(ret > 0 && (static_cast<size_t>(ret)) < size);
return buff.get(); return buff.get();
} }
} }
std::string FormatString(const char* msg, ...) { std::string FormatString(const char* msg, ...) {
va_list args; va_list args;
va_start(args, msg); va_start(args, msg);
auto tmp = FormatString(msg, args); auto tmp = FormatString(msg, args);
▲ Show 20 Lines • Show All 72 Lines • Show Last 20 Lines

third-party/benchmark/src/commandlineflags.h

#ifndef BENCHMARK_COMMANDLINEFLAGS_H_ #ifndef BENCHMARK_COMMANDLINEFLAGS_H_
#define BENCHMARK_COMMANDLINEFLAGS_H_ #define BENCHMARK_COMMANDLINEFLAGS_H_
#include <cstdint> #include <cstdint>
#include <map> #include <map>
#include <string> #include <string>
// Macro for referencing flags. // Macro for referencing flags.
#define FLAG(name) FLAGS_##name #define FLAG(name) FLAGS_##name
// Macros for declaring flags. // Macros for declaring flags.
#define DECLARE_bool(name) extern bool FLAG(name) #define BM_DECLARE_bool(name) extern bool FLAG(name)
#define DECLARE_int32(name) extern int32_t FLAG(name) #define BM_DECLARE_int32(name) extern int32_t FLAG(name)
#define DECLARE_double(name) extern double FLAG(name) #define BM_DECLARE_double(name) extern double FLAG(name)
#define DECLARE_string(name) extern std::string FLAG(name) #define BM_DECLARE_string(name) extern std::string FLAG(name)
#define DECLARE_kvpairs(name) \ #define BM_DECLARE_kvpairs(name) \
extern std::map<std::string, std::string> FLAG(name) extern std::map<std::string, std::string> FLAG(name)
// Macros for defining flags. // Macros for defining flags.
#define DEFINE_bool(name, default_val) \ #define BM_DEFINE_bool(name, default_val) \
bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val)
#define DEFINE_int32(name, default_val) \ #define BM_DEFINE_int32(name, default_val) \
int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val) int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val)
#define DEFINE_double(name, default_val) \ #define BM_DEFINE_double(name, default_val) \
double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val) double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val)
#define DEFINE_string(name, default_val) \ #define BM_DEFINE_string(name, default_val) \
std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val) std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val)
#define DEFINE_kvpairs(name, default_val) \ #define BM_DEFINE_kvpairs(name, default_val) \
std::map<std::string, std::string> FLAG(name) = \ std::map<std::string, std::string> FLAG(name) = \
benchmark::KvPairsFromEnv(#name, default_val) benchmark::KvPairsFromEnv(#name, default_val)
namespace benchmark { namespace benchmark {
// Parses a bool from the environment variable corresponding to the given flag. // Parses a bool from the environment variable corresponding to the given flag.
// //
// If the variable exists, returns IsTruthyFlagValue() value; if not, // If the variable exists, returns IsTruthyFlagValue() value; if not,
▲ Show 20 Lines • Show All 80 Lines • Show Last 20 Lines

third-party/benchmark/src/commandlineflags.cc

Show First 20 Lines • Show All 242 Lines • ▼ Show 20 Linesbool ParseStringFlag(const char* str, const char* flag, std::string* value) {
// Aborts if the parsing failed. // Aborts if the parsing failed.
if (value_str == nullptr) return false; if (value_str == nullptr) return false;
*value = value_str; *value = value_str;
return true; return true;
} }
bool ParseKeyValueFlag( bool ParseKeyValueFlag(const char* str, const char* flag,
const char* str, const char* flag,
std::map<std::string, std::string>* value) { std::map<std::string, std::string>* value) {
const char* const value_str = ParseFlagValue(str, flag, false); const char* const value_str = ParseFlagValue(str, flag, false);
if (value_str == nullptr) return false; if (value_str == nullptr) return false;
for (const auto& kvpair : StrSplit(value_str, ',')) { for (const auto& kvpair : StrSplit(value_str, ',')) {
const auto kv = StrSplit(kvpair, '='); const auto kv = StrSplit(kvpair, '=');
if (kv.size() != 2) return false; if (kv.size() != 2) return false;
value->emplace(kv[0], kv[1]); value->emplace(kv[0], kv[1]);
Show All 25 Lines

third-party/benchmark/src/complexity.cc

Show All 9 Lines
// distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// Source project : https://github.com/ismaelJimenez/cpp.leastsq // Source project : https://github.com/ismaelJimenez/cpp.leastsq
// Adapted to be used with google benchmark // Adapted to be used with google benchmark
#include "benchmark/benchmark.h" #include "complexity.h"
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
#include "benchmark/benchmark.h"
#include "check.h" #include "check.h"
#include "complexity.h"
namespace benchmark { namespace benchmark {
// Internal function to calculate the different scalability forms // Internal function to calculate the different scalability forms
BigOFunc* FittingCurve(BigO complexity) { BigOFunc* FittingCurve(BigO complexity) {
static const double kLog2E = 1.44269504088896340736; static const double kLog2E = 1.44269504088896340736;
switch (complexity) { switch (complexity) {
case oN: case oN:
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines
// minimizing the sum of squares of relative error. // minimizing the sum of squares of relative error.
// - n : Vector containing the size of the benchmark tests. // - n : Vector containing the size of the benchmark tests.
// - time : Vector containing the times for the benchmark tests. // - time : Vector containing the times for the benchmark tests.
// - complexity : If different than oAuto, the fitting curve will stick to // - complexity : If different than oAuto, the fitting curve will stick to
// this one. If it is oAuto, it will be calculated the best // this one. If it is oAuto, it will be calculated the best
// fitting curve. // fitting curve.
LeastSq MinimalLeastSq(const std::vector<int64_t>& n, LeastSq MinimalLeastSq(const std::vector<int64_t>& n,
const std::vector<double>& time, const BigO complexity) { const std::vector<double>& time, const BigO complexity) {
CHECK_EQ(n.size(), time.size()); BM_CHECK_EQ(n.size(), time.size());
CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two BM_CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two
// benchmark runs are given // benchmark runs are given
CHECK_NE(complexity, oNone); BM_CHECK_NE(complexity, oNone);
LeastSq best_fit; LeastSq best_fit;
if (complexity == oAuto) { if (complexity == oAuto) {
std::vector<BigO> fit_curves = {oLogN, oN, oNLogN, oNSquared, oNCubed}; std::vector<BigO> fit_curves = {oLogN, oN, oNLogN, oNSquared, oNCubed};
// Take o1 as default best fitting curve // Take o1 as default best fitting curve
best_fit = MinimalLeastSq(n, time, FittingCurve(o1)); best_fit = MinimalLeastSq(n, time, FittingCurve(o1));
Show All 24 Linesstd::vector<BenchmarkReporter::Run> ComputeBigO(
// Accumulators. // Accumulators.
std::vector<int64_t> n; std::vector<int64_t> n;
std::vector<double> real_time; std::vector<double> real_time;
std::vector<double> cpu_time; std::vector<double> cpu_time;
// Populate the accumulators. // Populate the accumulators.
for (const Run& run : reports) { for (const Run& run : reports) {
CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; BM_CHECK_GT(run.complexity_n, 0)
<< "Did you forget to call SetComplexityN?";
n.push_back(run.complexity_n); n.push_back(run.complexity_n);
real_time.push_back(run.real_accumulated_time / run.iterations); real_time.push_back(run.real_accumulated_time / run.iterations);
cpu_time.push_back(run.cpu_accumulated_time / run.iterations); cpu_time.push_back(run.cpu_accumulated_time / run.iterations);
} }
LeastSq result_cpu; LeastSq result_cpu;
LeastSq result_real; LeastSq result_real;
Show All 14 Linesstd::vector<BenchmarkReporter::Run> ComputeBigO(
big_o.run_name = run_name; big_o.run_name = run_name;
big_o.family_index = reports[0].family_index; big_o.family_index = reports[0].family_index;
big_o.per_family_instance_index = reports[0].per_family_instance_index; big_o.per_family_instance_index = reports[0].per_family_instance_index;
big_o.run_type = BenchmarkReporter::Run::RT_Aggregate; big_o.run_type = BenchmarkReporter::Run::RT_Aggregate;
big_o.repetitions = reports[0].repetitions; big_o.repetitions = reports[0].repetitions;
big_o.repetition_index = Run::no_repetition_index; big_o.repetition_index = Run::no_repetition_index;
big_o.threads = reports[0].threads; big_o.threads = reports[0].threads;
big_o.aggregate_name = "BigO"; big_o.aggregate_name = "BigO";
big_o.aggregate_unit = StatisticUnit::kTime;
big_o.report_label = reports[0].report_label; big_o.report_label = reports[0].report_label;
big_o.iterations = 0; big_o.iterations = 0;
big_o.real_accumulated_time = result_real.coef; big_o.real_accumulated_time = result_real.coef;
big_o.cpu_accumulated_time = result_cpu.coef; big_o.cpu_accumulated_time = result_cpu.coef;
big_o.report_big_o = true; big_o.report_big_o = true;
big_o.complexity = result_cpu.complexity; big_o.complexity = result_cpu.complexity;
// All the time results are reported after being multiplied by the // All the time results are reported after being multiplied by the
// time unit multiplier. But since RMS is a relative quantity it // time unit multiplier. But since RMS is a relative quantity it
// should not be multiplied at all. So, here, we _divide_ it by the // should not be multiplied at all. So, here, we _divide_ it by the
// multiplier so that when it is multiplied later the result is the // multiplier so that when it is multiplied later the result is the
// correct one. // correct one.
double multiplier = GetTimeUnitMultiplier(reports[0].time_unit); double multiplier = GetTimeUnitMultiplier(reports[0].time_unit);
// Only add label to mean/stddev if it is same for all runs // Only add label to mean/stddev if it is same for all runs
Run rms; Run rms;
rms.run_name = run_name; rms.run_name = run_name;
rms.family_index = reports[0].family_index; rms.family_index = reports[0].family_index;
rms.per_family_instance_index = reports[0].per_family_instance_index; rms.per_family_instance_index = reports[0].per_family_instance_index;
rms.run_type = BenchmarkReporter::Run::RT_Aggregate; rms.run_type = BenchmarkReporter::Run::RT_Aggregate;
rms.aggregate_name = "RMS"; rms.aggregate_name = "RMS";
rms.aggregate_unit = StatisticUnit::kPercentage;
rms.report_label = big_o.report_label; rms.report_label = big_o.report_label;
rms.iterations = 0; rms.iterations = 0;
rms.repetition_index = Run::no_repetition_index; rms.repetition_index = Run::no_repetition_index;
rms.repetitions = reports[0].repetitions; rms.repetitions = reports[0].repetitions;
rms.threads = reports[0].threads; rms.threads = reports[0].threads;
rms.real_accumulated_time = result_real.rms / multiplier; rms.real_accumulated_time = result_real.rms / multiplier;
rms.cpu_accumulated_time = result_cpu.rms / multiplier; rms.cpu_accumulated_time = result_cpu.rms / multiplier;
rms.report_rms = true; rms.report_rms = true;
Show All 11 Lines

third-party/benchmark/src/console_reporter.cc

Show All 39 Linesbool ConsoleReporter::ReportContext(const Context& context) {
PrintBasicContext(&GetErrorStream(), context); PrintBasicContext(&GetErrorStream(), context);
#ifdef BENCHMARK_OS_WINDOWS #ifdef BENCHMARK_OS_WINDOWS
if ((output_options_ & OO_Color) && &std::cout != &GetOutputStream()) { if ((output_options_ & OO_Color) && &std::cout != &GetOutputStream()) {
GetErrorStream() GetErrorStream()
<< "Color printing is only supported for stdout on windows." << "Color printing is only supported for stdout on windows."
" Disabling color printing\n"; " Disabling color printing\n";
output_options_ = static_cast< OutputOptions >(output_options_ & ~OO_Color); output_options_ = static_cast<OutputOptions>(output_options_ & ~OO_Color);
} }
#endif #endif
return true; return true;
} }
void ConsoleReporter::PrintHeader(const Run& run) { void ConsoleReporter::PrintHeader(const Run& run) {
std::string str = FormatString("%-*s %13s %15s %12s", static_cast<int>(name_field_width_), std::string str =
FormatString("%-*s %13s %15s %12s", static_cast<int>(name_field_width_),
"Benchmark", "Time", "CPU", "Iterations"); "Benchmark", "Time", "CPU", "Iterations");
if(!run.counters.empty()) { if (!run.counters.empty()) {
if(output_options_ & OO_Tabular) { if (output_options_ & OO_Tabular) {
for(auto const& c : run.counters) { for (auto const& c : run.counters) {
str += FormatString(" %10s", c.first.c_str()); str += FormatString(" %10s", c.first.c_str());
} }
} else { } else {
str += " UserCounters..."; str += " UserCounters...";
} }
} }
std::string line = std::string(str.length(), '-'); std::string line = std::string(str.length(), '-');
GetOutputStream() << line << "\n" << str << "\n" << line << "\n"; GetOutputStream() << line << "\n" << str << "\n" << line << "\n";
Show All 23 Lines
static void IgnoreColorPrint(std::ostream& out, LogColor, const char* fmt, static void IgnoreColorPrint(std::ostream& out, LogColor, const char* fmt,
...) { ...) {
va_list args; va_list args;
va_start(args, fmt); va_start(args, fmt);
out << FormatString(fmt, args); out << FormatString(fmt, args);
va_end(args); va_end(args);
} }
static std::string FormatTime(double time) { static std::string FormatTime(double time) {
// Align decimal places... // Align decimal places...
if (time < 1.0) { if (time < 1.0) {
return FormatString("%10.3f", time); return FormatString("%10.3f", time);
} }
if (time < 10.0) { if (time < 10.0) {
return FormatString("%10.2f", time); return FormatString("%10.2f", time);
} }
if (time < 100.0) { if (time < 100.0) {
return FormatString("%10.1f", time); return FormatString("%10.1f", time);
} }
return FormatString("%10.0f", time); return FormatString("%10.0f", time);
} }
void ConsoleReporter::PrintRunData(const Run& result) { void ConsoleReporter::PrintRunData(const Run& result) {
typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...); typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...);
auto& Out = GetOutputStream(); auto& Out = GetOutputStream();
PrinterFn* printer = (output_options_ & OO_Color) ? PrinterFn* printer = (output_options_ & OO_Color)
(PrinterFn*)ColorPrintf : IgnoreColorPrint; ? static_cast<PrinterFn*>(ColorPrintf)
: IgnoreColorPrint;
auto name_color = auto name_color =
(result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN; (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN;
printer(Out, name_color, "%-*s ", name_field_width_, printer(Out, name_color, "%-*s ", name_field_width_,
result.benchmark_name().c_str()); result.benchmark_name().c_str());
if (result.error_occurred) { if (result.error_occurred) {
printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'",
result.error_message.c_str()); result.error_message.c_str());
printer(Out, COLOR_DEFAULT, "\n"); printer(Out, COLOR_DEFAULT, "\n");
return; return;
} }
const double real_time = result.GetAdjustedRealTime(); const double real_time = result.GetAdjustedRealTime();
const double cpu_time = result.GetAdjustedCPUTime(); const double cpu_time = result.GetAdjustedCPUTime();
const std::string real_time_str = FormatTime(real_time); const std::string real_time_str = FormatTime(real_time);
const std::string cpu_time_str = FormatTime(cpu_time); const std::string cpu_time_str = FormatTime(cpu_time);
if (result.report_big_o) { if (result.report_big_o) {
std::string big_o = GetBigOString(result.complexity); std::string big_o = GetBigOString(result.complexity);
printer(Out, COLOR_YELLOW, "%10.2f %-4s %10.2f %-4s ", real_time, big_o.c_str(), printer(Out, COLOR_YELLOW, "%10.2f %-4s %10.2f %-4s ", real_time,
cpu_time, big_o.c_str()); big_o.c_str(), cpu_time, big_o.c_str());
} else if (result.report_rms) { } else if (result.report_rms) {
printer(Out, COLOR_YELLOW, "%10.0f %-4s %10.0f %-4s ", real_time * 100, "%", printer(Out, COLOR_YELLOW, "%10.0f %-4s %10.0f %-4s ", real_time * 100, "%",
cpu_time * 100, "%"); cpu_time * 100, "%");
} else { } else if (result.run_type != Run::RT_Aggregate ||
result.aggregate_unit == StatisticUnit::kTime) {
const char* timeLabel = GetTimeUnitString(result.time_unit); const char* timeLabel = GetTimeUnitString(result.time_unit);
printer(Out, COLOR_YELLOW, "%s %-4s %s %-4s ", real_time_str.c_str(), timeLabel, printer(Out, COLOR_YELLOW, "%s %-4s %s %-4s ", real_time_str.c_str(),
cpu_time_str.c_str(), timeLabel); timeLabel, cpu_time_str.c_str(), timeLabel);
} else {
assert(result.aggregate_unit == StatisticUnit::kPercentage);
printer(Out, COLOR_YELLOW, "%10.2f %-4s %10.2f %-4s ",
(100. * result.real_accumulated_time), "%",
(100. * result.cpu_accumulated_time), "%");
} }
if (!result.report_big_o && !result.report_rms) { if (!result.report_big_o && !result.report_rms) {
printer(Out, COLOR_CYAN, "%10lld", result.iterations); printer(Out, COLOR_CYAN, "%10lld", result.iterations);
} }
for (auto& c : result.counters) { for (auto& c : result.counters) {
const std::size_t cNameLen = std::max(std::string::size_type(10), const std::size_t cNameLen =
c.first.length()); std::max(std::string::size_type(10), c.first.length());
auto const& s = HumanReadableNumber(c.second.value, c.second.oneK); std::string s;
const char* unit = ""; const char* unit = "";
if (result.run_type == Run::RT_Aggregate &&
result.aggregate_unit == StatisticUnit::kPercentage) {
s = StrFormat("%.2f", 100. * c.second.value);
unit = "%";
} else {
s = HumanReadableNumber(c.second.value, c.second.oneK);
if (c.second.flags & Counter::kIsRate) if (c.second.flags & Counter::kIsRate)
unit = (c.second.flags & Counter::kInvert) ? "s" : "/s"; unit = (c.second.flags & Counter::kInvert) ? "s" : "/s";
}
if (output_options_ & OO_Tabular) { if (output_options_ & OO_Tabular) {
printer(Out, COLOR_DEFAULT, " %*s%s", cNameLen - strlen(unit), s.c_str(), printer(Out, COLOR_DEFAULT, " %*s%s", cNameLen - strlen(unit), s.c_str(),
unit); unit);
} else { } else {
printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), unit); printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), unit);
} }
} }
if (!result.report_label.empty()) { if (!result.report_label.empty()) {
printer(Out, COLOR_DEFAULT, " %s", result.report_label.c_str()); printer(Out, COLOR_DEFAULT, " %s", result.report_label.c_str());
} }
printer(Out, COLOR_DEFAULT, "\n"); printer(Out, COLOR_DEFAULT, "\n");
} }
} // end namespace benchmark } // end namespace benchmark

third-party/benchmark/src/csv_reporter.cc

// Copyright 2015 Google Inc. All rights reserved. // Copyright 2015 Google Inc. All rights reserved.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
// You may obtain a copy of the License at // You may obtain a copy of the License at
// //
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
// //
// Unless required by applicable law or agreed to in writing, software // Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "benchmark/benchmark.h"
#include "complexity.h"
#include <algorithm> #include <algorithm>
#include <cstdint> #include <cstdint>
#include <iostream> #include <iostream>
#include <string> #include <string>
#include <tuple> #include <tuple>
#include <vector> #include <vector>
#include "benchmark/benchmark.h"
#include "check.h" #include "check.h"
#include "complexity.h"
#include "string_util.h" #include "string_util.h"
#include "timers.h" #include "timers.h"
// File format reference: http://edoceo.com/utilitas/csv-file-format. // File format reference: http://edoceo.com/utilitas/csv-file-format.
namespace benchmark { namespace benchmark {
namespace { namespace {
std::vector<std::string> elements = { std::vector<std::string> elements = {
"name", "iterations", "real_time", "cpu_time", "name", "iterations", "real_time", "cpu_time",
"time_unit", "bytes_per_second", "items_per_second", "label", "time_unit", "bytes_per_second", "items_per_second", "label",
"error_occurred", "error_message"}; "error_occurred", "error_message"};
} // namespace } // namespace
std::string CsvEscape(const std::string & s) { std::string CsvEscape(const std::string& s) {
std::string tmp; std::string tmp;
tmp.reserve(s.size() + 2); tmp.reserve(s.size() + 2);
for (char c : s) { for (char c : s) {
switch (c) { switch (c) {
case '"' : tmp += "\"\""; break; case '"':
default : tmp += c; break; tmp += "\"\"";
break;
default:
tmp += c;
break;
} }
} }
return '"' + tmp + '"'; return '"' + tmp + '"';
} }
bool CSVReporter::ReportContext(const Context& context) { bool CSVReporter::ReportContext(const Context& context) {
PrintBasicContext(&GetErrorStream(), context); PrintBasicContext(&GetErrorStream(), context);
return true; return true;
Show All 25 Lines if (!printed_header_) {
printed_header_ = true; printed_header_ = true;
} else { } else {
// check that all the current counters are saved in the name set // check that all the current counters are saved in the name set
for (const auto& run : reports) { for (const auto& run : reports) {
for (const auto& cnt : run.counters) { for (const auto& cnt : run.counters) {
if (cnt.first == "bytes_per_second" || cnt.first == "items_per_second") if (cnt.first == "bytes_per_second" || cnt.first == "items_per_second")
continue; continue;
CHECK(user_counter_names_.find(cnt.first) != user_counter_names_.end()) BM_CHECK(user_counter_names_.find(cnt.first) !=
user_counter_names_.end())
<< "All counters must be present in each run. " << "All counters must be present in each run. "
<< "Counter named \"" << cnt.first << "Counter named \"" << cnt.first
<< "\" was not in a run after being added to the header"; << "\" was not in a run after being added to the header";
} }
} }
} }
// print results for each run // print results for each run
▲ Show 20 Lines • Show All 58 Lines • Show Last 20 Lines

third-party/benchmark/src/cycleclock.h

Show First 20 Lines • Show All 109 Lines • ▼ Show 20 Lines#elif defined(__ia64__)
return itc; return itc;
#elif defined(COMPILER_MSVC) && defined(_M_IX86) #elif defined(COMPILER_MSVC) && defined(_M_IX86)
// Older MSVC compilers (like 7.x) don't seem to support the // Older MSVC compilers (like 7.x) don't seem to support the
// __rdtsc intrinsic properly, so I prefer to use _asm instead // __rdtsc intrinsic properly, so I prefer to use _asm instead
// when I know it will work. Otherwise, I'll use __rdtsc and hope // when I know it will work. Otherwise, I'll use __rdtsc and hope
// the code is being compiled with a non-ancient compiler. // the code is being compiled with a non-ancient compiler.
_asm rdtsc _asm rdtsc
#elif defined(COMPILER_MSVC) && defined(_M_ARM64) #elif defined(COMPILER_MSVC) && defined(_M_ARM64)
// See https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=vs-2019 // See // https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics
// and https://reviews.llvm.org/D53115 // and https://reviews.llvm.org/D53115
int64_t virtual_timer_value; int64_t virtual_timer_value;
virtual_timer_value = _ReadStatusReg(ARM64_CNTVCT); virtual_timer_value = _ReadStatusReg(ARM64_CNTVCT);
return virtual_timer_value; return virtual_timer_value;
#elif defined(COMPILER_MSVC) #elif defined(COMPILER_MSVC)
return __rdtsc(); return __rdtsc();
#elif defined(BENCHMARK_OS_NACL) #elif defined(BENCHMARK_OS_NACL)
// Native Client validator on x86/x86-64 allows RDTSC instructions, // Native Client validator on x86/x86-64 allows RDTSC instructions,
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines#elif defined(__s390__) // Covers both s390 and s390x.
uint64_t tsc; uint64_t tsc;
#if defined(BENCHMARK_OS_ZOS) && defined(COMPILER_IBMXL) #if defined(BENCHMARK_OS_ZOS) && defined(COMPILER_IBMXL)
// z/OS XL compiler HLASM syntax. // z/OS XL compiler HLASM syntax.
asm(" stck %0" : "=m"(tsc) : : "cc"); asm(" stck %0" : "=m"(tsc) : : "cc");
#else #else
asm("stck %0" : "=Q"(tsc) : : "cc"); asm("stck %0" : "=Q"(tsc) : : "cc");
#endif #endif
return tsc; return tsc;
#elif defined(__riscv) // RISC-V #elif defined(__riscv) // RISC-V
// Use RDCYCLE (and RDCYCLEH on riscv32) // Use RDCYCLE (and RDCYCLEH on riscv32)
#if __riscv_xlen == 32 #if __riscv_xlen == 32
uint32_t cycles_lo, cycles_hi0, cycles_hi1; uint32_t cycles_lo, cycles_hi0, cycles_hi1;
// This asm also includes the PowerPC overflow handling strategy, as above. // This asm also includes the PowerPC overflow handling strategy, as above.
// Implemented in assembly because Clang insisted on branching. // Implemented in assembly because Clang insisted on branching.
asm volatile( asm volatile(
"rdcycleh %0\n" "rdcycleh %0\n"
"rdcycle %1\n" "rdcycle %1\n"
Show All 27 Lines

third-party/benchmark/src/json_reporter.cc

// Copyright 2015 Google Inc. All rights reserved. // Copyright 2015 Google Inc. All rights reserved.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
// You may obtain a copy of the License at // You may obtain a copy of the License at
// //
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
// //
// Unless required by applicable law or agreed to in writing, software // Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "benchmark/benchmark.h"
#include "complexity.h"
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
#include <cstdint> #include <cstdint>
#include <iomanip> // for setprecision #include <iomanip> // for setprecision
#include <iostream> #include <iostream>
#include <limits> #include <limits>
#include <string> #include <string>
#include <tuple> #include <tuple>
#include <vector> #include <vector>
#include "benchmark/benchmark.h"
#include "complexity.h"
#include "string_util.h" #include "string_util.h"
#include "timers.h" #include "timers.h"
namespace benchmark { namespace benchmark {
namespace internal { namespace internal {
extern std::map<std::string, std::string>* global_context; extern std::map<std::string, std::string>* global_context;
} }
namespace { namespace {
std::string StrEscape(const std::string & s) { std::string StrEscape(const std::string& s) {
std::string tmp; std::string tmp;
tmp.reserve(s.size()); tmp.reserve(s.size());
for (char c : s) { for (char c : s) {
switch (c) { switch (c) {
case '\b': tmp += "\\b"; break; case '\b':
case '\f': tmp += "\\f"; break; tmp += "\\b";
case '\n': tmp += "\\n"; break; break;
case '\r': tmp += "\\r"; break; case '\f':
case '\t': tmp += "\\t"; break; tmp += "\\f";
case '\\': tmp += "\\\\"; break; break;
case '"' : tmp += "\\\""; break; case '\n':
default : tmp += c; break; tmp += "\\n";
break;
case '\r':
tmp += "\\r";
break;
case '\t':
tmp += "\\t";
break;
case '\\':
tmp += "\\\\";
break;
case '"':
tmp += "\\\"";
break;
default:
tmp += c;
break;
} }
} }
return tmp; return tmp;
} }
std::string FormatKV(std::string const& key, std::string const& value) { std::string FormatKV(std::string const& key, std::string const& value) {
return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str()); return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(),
StrEscape(value).c_str());
} }
std::string FormatKV(std::string const& key, const char* value) { std::string FormatKV(std::string const& key, const char* value) {
return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str()); return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(),
StrEscape(value).c_str());
} }
std::string FormatKV(std::string const& key, bool value) { std::string FormatKV(std::string const& key, bool value) {
return StrFormat("\"%s\": %s", StrEscape(key).c_str(), value ? "true" : "false"); return StrFormat("\"%s\": %s", StrEscape(key).c_str(),
value ? "true" : "false");
} }
std::string FormatKV(std::string const& key, int64_t value) { std::string FormatKV(std::string const& key, int64_t value) {
std::stringstream ss; std::stringstream ss;
ss << '"' << StrEscape(key) << "\": " << value; ss << '"' << StrEscape(key) << "\": " << value;
return ss.str(); return ss.str();
} }
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Linesbool JSONReporter::ReportContext(const Context& context) {
CPUInfo const& info = context.cpu_info; CPUInfo const& info = context.cpu_info;
out << indent << FormatKV("num_cpus", static_cast<int64_t>(info.num_cpus)) out << indent << FormatKV("num_cpus", static_cast<int64_t>(info.num_cpus))
<< ",\n"; << ",\n";
out << indent out << indent
<< FormatKV("mhz_per_cpu", << FormatKV("mhz_per_cpu",
RoundDouble(info.cycles_per_second / 1000000.0)) RoundDouble(info.cycles_per_second / 1000000.0))
<< ",\n"; << ",\n";
if (CPUInfo::Scaling::UNKNOWN != info.scaling) { if (CPUInfo::Scaling::UNKNOWN != info.scaling) {
out << indent << FormatKV("cpu_scaling_enabled", info.scaling == CPUInfo::Scaling::ENABLED ? true : false) out << indent
<< FormatKV("cpu_scaling_enabled",
info.scaling == CPUInfo::Scaling::ENABLED ? true : false)
<< ",\n"; << ",\n";
} }
out << indent << "\"caches\": [\n"; out << indent << "\"caches\": [\n";
indent = std::string(6, ' '); indent = std::string(6, ' ');
std::string cache_indent(8, ' '); std::string cache_indent(8, ' ');
for (size_t i = 0; i < info.caches.size(); ++i) { for (size_t i = 0; i < info.caches.size(); ++i) {
auto& CI = info.caches[i]; auto& CI = info.caches[i];
out << indent << "{\n"; out << indent << "{\n";
out << cache_indent << FormatKV("type", CI.type) << ",\n"; out << cache_indent << FormatKV("type", CI.type) << ",\n";
out << cache_indent << FormatKV("level", static_cast<int64_t>(CI.level)) out << cache_indent << FormatKV("level", static_cast<int64_t>(CI.level))
<< ",\n"; << ",\n";
out << cache_indent out << cache_indent << FormatKV("size", static_cast<int64_t>(CI.size))
<< FormatKV("size", static_cast<int64_t>(CI.size)) << ",\n"; << ",\n";
out << cache_indent out << cache_indent
<< FormatKV("num_sharing", static_cast<int64_t>(CI.num_sharing)) << FormatKV("num_sharing", static_cast<int64_t>(CI.num_sharing))
<< "\n"; << "\n";
out << indent << "}"; out << indent << "}";
if (i != info.caches.size() - 1) out << ","; if (i != info.caches.size() - 1) out << ",";
out << "\n"; out << "\n";
} }
indent = std::string(4, ' '); indent = std::string(4, ' ');
out << indent << "],\n"; out << indent << "],\n";
out << indent << "\"load_avg\": ["; out << indent << "\"load_avg\": [";
for (auto it = info.load_avg.begin(); it != info.load_avg.end();) { for (auto it = info.load_avg.begin(); it != info.load_avg.end();) {
out << *it++; out << *it++;
if (it != info.load_avg.end()) out << ","; if (it != info.load_avg.end()) out << ",";
} }
out << "],\n"; out << "],\n";
#if defined(NDEBUG) #if defined(NDEBUG)
const char build_type[] = "release"; const char build_type[] = "release";
#else #else
const char build_type[] = "debug"; const char build_type[] = "debug";
#endif #endif
out << indent << FormatKV("library_build_type", build_type) << "\n"; out << indent << FormatKV("library_build_type", build_type);
if (internal::global_context != nullptr) { if (internal::global_context != nullptr) {
for (const auto& kv: *internal::global_context) { for (const auto& kv : *internal::global_context) {
out << indent << FormatKV(kv.first, kv.second) << "\n"; out << ",\n";
out << indent << FormatKV(kv.first, kv.second);
} }
} }
out << "\n";
// Close context block and open the list of benchmarks. // Close context block and open the list of benchmarks.
out << inner_indent << "},\n"; out << inner_indent << "},\n";
out << inner_indent << "\"benchmarks\": [\n"; out << inner_indent << "\"benchmarks\": [\n";
return true; return true;
} }
void JSONReporter::ReportRuns(std::vector<Run> const& reports) { void JSONReporter::ReportRuns(std::vector<Run> const& reports) {
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Linesvoid JSONReporter::PrintRunData(Run const& run) {
out << indent << FormatKV("repetitions", run.repetitions) << ",\n"; out << indent << FormatKV("repetitions", run.repetitions) << ",\n";
if (run.run_type != BenchmarkReporter::Run::RT_Aggregate) { if (run.run_type != BenchmarkReporter::Run::RT_Aggregate) {
out << indent << FormatKV("repetition_index", run.repetition_index) out << indent << FormatKV("repetition_index", run.repetition_index)
<< ",\n"; << ",\n";
} }
out << indent << FormatKV("threads", run.threads) << ",\n"; out << indent << FormatKV("threads", run.threads) << ",\n";
if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) { if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) {
out << indent << FormatKV("aggregate_name", run.aggregate_name) << ",\n"; out << indent << FormatKV("aggregate_name", run.aggregate_name) << ",\n";
out << indent << FormatKV("aggregate_unit", [&run]() -> const char* {
switch (run.aggregate_unit) {
case StatisticUnit::kTime:
return "time";
case StatisticUnit::kPercentage:
return "percentage";
}
BENCHMARK_UNREACHABLE();
}()) << ",\n";
} }
if (run.error_occurred) { if (run.error_occurred) {
out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n";
out << indent << FormatKV("error_message", run.error_message) << ",\n"; out << indent << FormatKV("error_message", run.error_message) << ",\n";
} }
if (!run.report_big_o && !run.report_rms) { if (!run.report_big_o && !run.report_rms) {
out << indent << FormatKV("iterations", run.iterations) << ",\n"; out << indent << FormatKV("iterations", run.iterations) << ",\n";
out << indent << FormatKV("real_time", run.GetAdjustedRealTime()) << ",\n"; if (run.run_type != Run::RT_Aggregate ||
run.aggregate_unit == StatisticUnit::kTime) {
out << indent << FormatKV("real_time", run.GetAdjustedRealTime())
<< ",\n";
out << indent << FormatKV("cpu_time", run.GetAdjustedCPUTime()); out << indent << FormatKV("cpu_time", run.GetAdjustedCPUTime());
} else {
assert(run.aggregate_unit == StatisticUnit::kPercentage);
out << indent << FormatKV("real_time", run.real_accumulated_time)
<< ",\n";
out << indent << FormatKV("cpu_time", run.cpu_accumulated_time);
}
out << ",\n" out << ",\n"
<< indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit));
} else if (run.report_big_o) { } else if (run.report_big_o) {
out << indent << FormatKV("cpu_coefficient", run.GetAdjustedCPUTime()) out << indent << FormatKV("cpu_coefficient", run.GetAdjustedCPUTime())
<< ",\n"; << ",\n";
out << indent << FormatKV("real_coefficient", run.GetAdjustedRealTime()) out << indent << FormatKV("real_coefficient", run.GetAdjustedRealTime())
<< ",\n"; << ",\n";
out << indent << FormatKV("big_o", GetBigOString(run.complexity)) << ",\n"; out << indent << FormatKV("big_o", GetBigOString(run.complexity)) << ",\n";
out << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); out << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit));
} else if (run.report_rms) { } else if (run.report_rms) {
out << indent << FormatKV("rms", run.GetAdjustedCPUTime()); out << indent << FormatKV("rms", run.GetAdjustedCPUTime());
} }
for (auto& c : run.counters) { for (auto& c : run.counters) {
out << ",\n" << indent << FormatKV(c.first, c.second); out << ",\n" << indent << FormatKV(c.first, c.second);
} }
if (run.has_memory_result) { if (run.memory_result) {
const MemoryManager::Result memory_result = *run.memory_result;
out << ",\n" << indent << FormatKV("allocs_per_iter", run.allocs_per_iter); out << ",\n" << indent << FormatKV("allocs_per_iter", run.allocs_per_iter);
out << ",\n" << indent << FormatKV("max_bytes_used", run.max_bytes_used); out << ",\n"
<< indent << FormatKV("max_bytes_used", memory_result.max_bytes_used);
auto report_if_present = [&out, &indent](const char* label, int64_t val) {
if (val != MemoryManager::TombstoneValue)
out << ",\n" << indent << FormatKV(label, val);
};
report_if_present("total_allocated_bytes",
memory_result.total_allocated_bytes);
report_if_present("net_heap_growth", memory_result.net_heap_growth);
} }
if (!run.report_label.empty()) { if (!run.report_label.empty()) {
out << ",\n" << indent << FormatKV("label", run.report_label); out << ",\n" << indent << FormatKV("label", run.report_label);
} }
out << '\n'; out << '\n';
} }
const int64_t MemoryManager::TombstoneValue =
std::numeric_limits<int64_t>::max();
} // end namespace benchmark } // end namespace benchmark

third-party/benchmark/src/log.h

Show First 20 Lines • Show All 61 Lines • ▼ Show 20 Linesinline LogType& GetLogInstanceForLevel(int level) {
} }
return GetNullLogInstance(); return GetNullLogInstance();
} }
} // end namespace internal } // end namespace internal
} // end namespace benchmark } // end namespace benchmark
// clang-format off // clang-format off
#define VLOG(x) \ #define BM_VLOG(x) \
(::benchmark::internal::GetLogInstanceForLevel(x) << "-- LOG(" << x << "):" \ (::benchmark::internal::GetLogInstanceForLevel(x) << "-- LOG(" << x << "):" \
" ") " ")
// clang-format on // clang-format on
#endif #endif

third-party/benchmark/src/mutex.h

Show First 20 Lines • Show All 124 Lines • ▼ Show 20 Lines private:
// State for barrier management // State for barrier management
int phase_number_ = 0; int phase_number_ = 0;
int entered_ = 0; // Number of threads that have entered this barrier int entered_ = 0; // Number of threads that have entered this barrier
// Enter the barrier and wait until all other threads have also // Enter the barrier and wait until all other threads have also
// entered the barrier. Returns iff this is the last thread to // entered the barrier. Returns iff this is the last thread to
// enter the barrier. // enter the barrier.
bool createBarrier(MutexLock& ml) REQUIRES(lock_) { bool createBarrier(MutexLock& ml) REQUIRES(lock_) {
CHECK_LT(entered_, running_threads_); BM_CHECK_LT(entered_, running_threads_);
entered_++; entered_++;
if (entered_ < running_threads_) { if (entered_ < running_threads_) {
// Wait for all threads to enter // Wait for all threads to enter
int phase_number_cp = phase_number_; int phase_number_cp = phase_number_;
auto cb = [this, phase_number_cp]() { auto cb = [this, phase_number_cp]() {
return this->phase_number_ > phase_number_cp || return this->phase_number_ > phase_number_cp ||
entered_ == running_threads_; // A thread has aborted in error entered_ == running_threads_; // A thread has aborted in error
}; };
Show All 14 Lines

third-party/benchmark/src/perf_counters.h

Show All 36 Lines
// indexing into the counter values. // indexing into the counter values.
// The object is used in conjunction with a PerfCounters object, by passing it // The object is used in conjunction with a PerfCounters object, by passing it
// to Snapshot(). The values are populated such that // to Snapshot(). The values are populated such that
// perfCounters->names()[i]'s value is obtained at position i (as given by // perfCounters->names()[i]'s value is obtained at position i (as given by
// operator[]) of this object. // operator[]) of this object.
class PerfCounterValues { class PerfCounterValues {
public: public:
explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) { explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) {
CHECK_LE(nr_counters_, kMaxCounters); BM_CHECK_LE(nr_counters_, kMaxCounters);
} }
uint64_t operator[](size_t pos) const { return values_[kPadding + pos]; } uint64_t operator[](size_t pos) const { return values_[kPadding + pos]; }
static constexpr size_t kMaxCounters = 3; static constexpr size_t kMaxCounters = 3;
private: private:
friend class PerfCounters; friend class PerfCounters;
▲ Show 20 Lines • Show All 119 Lines • Show Last 20 Lines

third-party/benchmark/src/perf_counters.cc

Show First 20 Lines • Show All 43 Lines • ▼ Show 20 Lines GetErrorLogInstance()
<< PerfCounterValues::kMaxCounters << "\n"; << PerfCounterValues::kMaxCounters << "\n";
return NoCounters(); return NoCounters();
} }
std::vector<int> counter_ids(counter_names.size()); std::vector<int> counter_ids(counter_names.size());
const int mode = PFM_PLM3; // user mode only const int mode = PFM_PLM3; // user mode only
for (size_t i = 0; i < counter_names.size(); ++i) { for (size_t i = 0; i < counter_names.size(); ++i) {
const bool is_first = i == 0; const bool is_first = i == 0;
struct perf_event_attr attr{}; struct perf_event_attr attr {};
attr.size = sizeof(attr); attr.size = sizeof(attr);
const int group_id = !is_first ? counter_ids[0] : -1; const int group_id = !is_first ? counter_ids[0] : -1;
const auto& name = counter_names[i]; const auto& name = counter_names[i];
if (name.empty()) { if (name.empty()) {
GetErrorLogInstance() << "A counter name was the empty string\n"; GetErrorLogInstance() << "A counter name was the empty string\n";
return NoCounters(); return NoCounters();
} }
pfm_perf_encode_arg_t arg{}; pfm_perf_encode_arg_t arg{};
▲ Show 20 Lines • Show All 72 Lines • Show Last 20 Lines

third-party/benchmark/src/re.h

Show First 20 Lines • Show All 120 Lines • ▼ Show 20 Linesinline bool Regex::Init(const std::string& spec, std::string* error) {
if (ec != 0) { if (ec != 0) {
if (error) { if (error) {
size_t needed = regerror(ec, &re_, nullptr, 0); size_t needed = regerror(ec, &re_, nullptr, 0);
char* errbuf = new char[needed]; char* errbuf = new char[needed];
regerror(ec, &re_, errbuf, needed); regerror(ec, &re_, errbuf, needed);
// regerror returns the number of bytes necessary to null terminate // regerror returns the number of bytes necessary to null terminate
// the string, so we move that when assigning to error. // the string, so we move that when assigning to error.
CHECK_NE(needed, 0); BM_CHECK_NE(needed, 0);
error->assign(errbuf, needed - 1); error->assign(errbuf, needed - 1);
delete[] errbuf; delete[] errbuf;
} }
return false; return false;
} }
Show All 21 Lines

third-party/benchmark/src/reporter.cc

// Copyright 2015 Google Inc. All rights reserved. // Copyright 2015 Google Inc. All rights reserved.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
// You may obtain a copy of the License at // You may obtain a copy of the License at
// //
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
// //
// Unless required by applicable law or agreed to in writing, software // Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "benchmark/benchmark.h"
#include "timers.h"
#include <cstdlib> #include <cstdlib>
#include <iostream> #include <iostream>
#include <map> #include <map>
#include <string> #include <string>
#include <tuple> #include <tuple>
#include <vector> #include <vector>
#include "benchmark/benchmark.h"
#include "check.h" #include "check.h"
#include "string_util.h" #include "string_util.h"
#include "timers.h"
namespace benchmark { namespace benchmark {
namespace internal { namespace internal {
extern std::map<std::string, std::string>* global_context; extern std::map<std::string, std::string> *global_context;
} }
BenchmarkReporter::BenchmarkReporter() BenchmarkReporter::BenchmarkReporter()
: output_stream_(&std::cout), error_stream_(&std::cerr) {} : output_stream_(&std::cout), error_stream_(&std::cerr) {}
BenchmarkReporter::~BenchmarkReporter() {} BenchmarkReporter::~BenchmarkReporter() {}
void BenchmarkReporter::PrintBasicContext(std::ostream *out, void BenchmarkReporter::PrintBasicContext(std::ostream *out,
Context const &context) { Context const &context) {
CHECK(out) << "cannot be null"; BM_CHECK(out) << "cannot be null";
auto &Out = *out; auto &Out = *out;
Out << LocalDateTimeString() << "\n"; Out << LocalDateTimeString() << "\n";
if (context.executable_name) if (context.executable_name)
Out << "Running " << context.executable_name << "\n"; Out << "Running " << context.executable_name << "\n";
const CPUInfo &info = context.cpu_info; const CPUInfo &info = context.cpu_info;
Show All 15 Lines if (!info.load_avg.empty()) {
for (auto It = info.load_avg.begin(); It != info.load_avg.end();) { for (auto It = info.load_avg.begin(); It != info.load_avg.end();) {
Out << StrFormat("%.2f", *It++); Out << StrFormat("%.2f", *It++);
if (It != info.load_avg.end()) Out << ", "; if (It != info.load_avg.end()) Out << ", ";
} }
Out << "\n"; Out << "\n";
} }
if (internal::global_context != nullptr) { if (internal::global_context != nullptr) {
for (const auto& kv: *internal::global_context) { for (const auto &kv : *internal::global_context) {
Out << kv.first << ": " << kv.second << "\n"; Out << kv.first << ": " << kv.second << "\n";
} }
} }
if (CPUInfo::Scaling::ENABLED == info.scaling) { if (CPUInfo::Scaling::ENABLED == info.scaling) {
Out << "***WARNING*** CPU scaling is enabled, the benchmark " Out << "***WARNING*** CPU scaling is enabled, the benchmark "
"real time measurements may be noisy and will incur extra " "real time measurements may be noisy and will incur extra "
"overhead.\n"; "overhead.\n";
Show All 35 Lines

third-party/benchmark/src/sleep.cc

Show All 29 Lines
namespace benchmark { namespace benchmark {
#ifdef BENCHMARK_OS_WINDOWS #ifdef BENCHMARK_OS_WINDOWS
// Window's Sleep takes milliseconds argument. // Window's Sleep takes milliseconds argument.
void SleepForMilliseconds(int milliseconds) { Sleep(milliseconds); } void SleepForMilliseconds(int milliseconds) { Sleep(milliseconds); }
void SleepForSeconds(double seconds) { void SleepForSeconds(double seconds) {
SleepForMilliseconds(static_cast<int>(kNumMillisPerSecond * seconds)); SleepForMilliseconds(static_cast<int>(kNumMillisPerSecond * seconds));
} }
#else // BENCHMARK_OS_WINDOWS #else // BENCHMARK_OS_WINDOWS
void SleepForMicroseconds(int microseconds) { void SleepForMicroseconds(int microseconds) {
#ifdef BENCHMARK_OS_ZOS #ifdef BENCHMARK_OS_ZOS
// z/OS does not support nanosleep. Instead call sleep() and then usleep() to // z/OS does not support nanosleep. Instead call sleep() and then usleep() to
// sleep for the remaining microseconds because usleep() will fail if its // sleep for the remaining microseconds because usleep() will fail if its
// argument is greater than 1000000. // argument is greater than 1000000.
div_t sleepTime = div(microseconds, kNumMicrosPerSecond); div_t sleepTime = div(microseconds, kNumMicrosPerSecond);
int seconds = sleepTime.quot; int seconds = sleepTime.quot;
while (seconds != 0) while (seconds != 0) seconds = sleep(seconds);
seconds = sleep(seconds);
while (usleep(sleepTime.rem) == -1 && errno == EINTR) while (usleep(sleepTime.rem) == -1 && errno == EINTR)
; ;
#else #else
struct timespec sleep_time; struct timespec sleep_time;
sleep_time.tv_sec = microseconds / kNumMicrosPerSecond; sleep_time.tv_sec = microseconds / kNumMicrosPerSecond;
sleep_time.tv_nsec = (microseconds % kNumMicrosPerSecond) * kNumNanosPerMicro; sleep_time.tv_nsec = (microseconds % kNumMicrosPerSecond) * kNumNanosPerMicro;
while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR)
; // Ignore signals and wait for the full interval to elapse. ; // Ignore signals and wait for the full interval to elapse.
Show All 12 Lines

third-party/benchmark/src/statistics.h

Show All 25 Lines
// (and any user-specified info) for the specified list of reports. If 'reports' // (and any user-specified info) for the specified list of reports. If 'reports'
// contains less than two non-errored runs an empty vector is returned // contains less than two non-errored runs an empty vector is returned
std::vector<BenchmarkReporter::Run> ComputeStats( std::vector<BenchmarkReporter::Run> ComputeStats(
const std::vector<BenchmarkReporter::Run>& reports); const std::vector<BenchmarkReporter::Run>& reports);
double StatisticsMean(const std::vector<double>& v); double StatisticsMean(const std::vector<double>& v);
double StatisticsMedian(const std::vector<double>& v); double StatisticsMedian(const std::vector<double>& v);
double StatisticsStdDev(const std::vector<double>& v); double StatisticsStdDev(const std::vector<double>& v);
double StatisticsCV(const std::vector<double>& v);
} // end namespace benchmark } // end namespace benchmark
#endif // STATISTICS_H_ #endif // STATISTICS_H_

third-party/benchmark/src/statistics.cc

// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. // Copyright 2016 Ismael Jimenez Martinez. All rights reserved.
// Copyright 2017 Roman Lebedev. All rights reserved. // Copyright 2017 Roman Lebedev. All rights reserved.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
// You may obtain a copy of the License at // You may obtain a copy of the License at
// //
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
// //
// Unless required by applicable law or agreed to in writing, software // Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "benchmark/benchmark.h" #include "statistics.h"
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
#include <numeric> #include <numeric>
#include <string> #include <string>
#include <vector> #include <vector>
#include "benchmark/benchmark.h"
#include "check.h" #include "check.h"
#include "statistics.h"
namespace benchmark { namespace benchmark {
auto StatisticsSum = [](const std::vector<double>& v) { auto StatisticsSum = [](const std::vector<double>& v) {
return std::accumulate(v.begin(), v.end(), 0.0); return std::accumulate(v.begin(), v.end(), 0.0);
}; };
double StatisticsMean(const std::vector<double>& v) { double StatisticsMean(const std::vector<double>& v) {
Show All 36 Linesdouble StatisticsStdDev(const std::vector<double>& v) {
// Sample standard deviation is undefined for n = 1 // Sample standard deviation is undefined for n = 1
if (v.size() == 1) return 0.0; if (v.size() == 1) return 0.0;
const double avg_squares = SumSquares(v) * (1.0 / v.size()); const double avg_squares = SumSquares(v) * (1.0 / v.size());
return Sqrt(v.size() / (v.size() - 1.0) * (avg_squares - Sqr(mean))); return Sqrt(v.size() / (v.size() - 1.0) * (avg_squares - Sqr(mean)));
} }
double StatisticsCV(const std::vector<double>& v) {
if (v.size() < 2) return 0.0;
const auto stddev = StatisticsStdDev(v);
const auto mean = StatisticsMean(v);
return stddev / mean;
}
std::vector<BenchmarkReporter::Run> ComputeStats( std::vector<BenchmarkReporter::Run> ComputeStats(
const std::vector<BenchmarkReporter::Run>& reports) { const std::vector<BenchmarkReporter::Run>& reports) {
typedef BenchmarkReporter::Run Run; typedef BenchmarkReporter::Run Run;
std::vector<Run> results; std::vector<Run> results;
auto error_count = auto error_count =
std::count_if(reports.begin(), reports.end(), std::count_if(reports.begin(), reports.end(),
[](Run const& run) { return run.error_occurred; }); [](Run const& run) { return run.error_occurred; });
Show All 22 Linesstd::vector<BenchmarkReporter::Run> ComputeStats(
for (Run const& r : reports) { for (Run const& r : reports) {
for (auto const& cnt : r.counters) { for (auto const& cnt : r.counters) {
auto it = counter_stats.find(cnt.first); auto it = counter_stats.find(cnt.first);
if (it == counter_stats.end()) { if (it == counter_stats.end()) {
counter_stats.insert({cnt.first, {cnt.second, std::vector<double>{}}}); counter_stats.insert({cnt.first, {cnt.second, std::vector<double>{}}});
it = counter_stats.find(cnt.first); it = counter_stats.find(cnt.first);
it->second.s.reserve(reports.size()); it->second.s.reserve(reports.size());
} else { } else {
CHECK_EQ(counter_stats[cnt.first].c.flags, cnt.second.flags); BM_CHECK_EQ(counter_stats[cnt.first].c.flags, cnt.second.flags);
} }
} }
} }
// Populate the accumulators. // Populate the accumulators.
for (Run const& run : reports) { for (Run const& run : reports) {
CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name()); BM_CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name());
CHECK_EQ(run_iterations, run.iterations); BM_CHECK_EQ(run_iterations, run.iterations);
if (run.error_occurred) continue; if (run.error_occurred) continue;
real_accumulated_time_stat.emplace_back(run.real_accumulated_time); real_accumulated_time_stat.emplace_back(run.real_accumulated_time);
cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time); cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time);
// user counters // user counters
for (auto const& cnt : run.counters) { for (auto const& cnt : run.counters) {
auto it = counter_stats.find(cnt.first); auto it = counter_stats.find(cnt.first);
CHECK_NE(it, counter_stats.end()); BM_CHECK_NE(it, counter_stats.end());
it->second.s.emplace_back(cnt.second); it->second.s.emplace_back(cnt.second);
} }
} }
// Only add label if it is same for all runs // Only add label if it is same for all runs
std::string report_label = reports[0].report_label; std::string report_label = reports[0].report_label;
for (std::size_t i = 1; i < reports.size(); i++) { for (std::size_t i = 1; i < reports.size(); i++) {
if (reports[i].report_label != report_label) { if (reports[i].report_label != report_label) {
Show All 11 Lines for (const auto& Stat : *reports[0].statistics) {
data.run_name = reports[0].run_name; data.run_name = reports[0].run_name;
data.family_index = reports[0].family_index; data.family_index = reports[0].family_index;
data.per_family_instance_index = reports[0].per_family_instance_index; data.per_family_instance_index = reports[0].per_family_instance_index;
data.run_type = BenchmarkReporter::Run::RT_Aggregate; data.run_type = BenchmarkReporter::Run::RT_Aggregate;
data.threads = reports[0].threads; data.threads = reports[0].threads;
data.repetitions = reports[0].repetitions; data.repetitions = reports[0].repetitions;
data.repetition_index = Run::no_repetition_index; data.repetition_index = Run::no_repetition_index;
data.aggregate_name = Stat.name_; data.aggregate_name = Stat.name_;
data.aggregate_unit = Stat.unit_;
data.report_label = report_label; data.report_label = report_label;
// It is incorrect to say that an aggregate is computed over // It is incorrect to say that an aggregate is computed over
// run's iterations, because those iterations already got averaged. // run's iterations, because those iterations already got averaged.
// Similarly, if there are N repetitions with 1 iterations each, // Similarly, if there are N repetitions with 1 iterations each,
// an aggregate will be computed over N measurements, not 1. // an aggregate will be computed over N measurements, not 1.
// Thus it is best to simply use the count of separate reports. // Thus it is best to simply use the count of separate reports.
data.iterations = reports.size(); data.iterations = reports.size();
data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat); data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat);
data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat); data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat);
if (data.aggregate_unit == StatisticUnit::kTime) {
// We will divide these times by data.iterations when reporting, but the // We will divide these times by data.iterations when reporting, but the
// data.iterations is not nessesairly the scale of these measurements, // data.iterations is not necessarily the scale of these measurements,
// because in each repetition, these timers are sum over all the iterations. // because in each repetition, these timers are sum over all the iters.
// And if we want to say that the stats are over N repetitions and not // And if we want to say that the stats are over N repetitions and not
// M iterations, we need to multiply these by (N/M). // M iterations, we need to multiply these by (N/M).
data.real_accumulated_time *= iteration_rescale_factor; data.real_accumulated_time *= iteration_rescale_factor;
data.cpu_accumulated_time *= iteration_rescale_factor; data.cpu_accumulated_time *= iteration_rescale_factor;
}
data.time_unit = reports[0].time_unit; data.time_unit = reports[0].time_unit;
// user counters // user counters
for (auto const& kv : counter_stats) { for (auto const& kv : counter_stats) {
// Do *NOT* rescale the custom counters. They are already properly scaled. // Do *NOT* rescale the custom counters. They are already properly scaled.
const auto uc_stat = Stat.compute_(kv.second.s); const auto uc_stat = Stat.compute_(kv.second.s);
auto c = Counter(uc_stat, counter_stats[kv.first].c.flags, auto c = Counter(uc_stat, counter_stats[kv.first].c.flags,
Show All 11 Lines

third-party/benchmark/src/string_util.h

#ifndef BENCHMARK_STRING_UTIL_H_ #ifndef BENCHMARK_STRING_UTIL_H_
#define BENCHMARK_STRING_UTIL_H_ #define BENCHMARK_STRING_UTIL_H_
#include <sstream> #include <sstream>
#include <string> #include <string>
#include <utility> #include <utility>
#include "internal_macros.h" #include "internal_macros.h"
namespace benchmark { namespace benchmark {
void AppendHumanReadable(int n, std::string* str); void AppendHumanReadable(int n, std::string* str);
std::string HumanReadableNumber(double n, double one_k = 1024.0); std::string HumanReadableNumber(double n, double one_k = 1024.0);
Show All 19 Lines
inline std::string StrCat(Args&&... args) { inline std::string StrCat(Args&&... args) {
std::ostringstream ss; std::ostringstream ss;
StrCatImp(ss, std::forward<Args>(args)...); StrCatImp(ss, std::forward<Args>(args)...);
return ss.str(); return ss.str();
} }
std::vector<std::string> StrSplit(const std::string& str, char delim); std::vector<std::string> StrSplit(const std::string& str, char delim);
// Disable lint checking for this block since it re-implements C functions.
// NOLINTBEGIN
#ifdef BENCHMARK_STL_ANDROID_GNUSTL #ifdef BENCHMARK_STL_ANDROID_GNUSTL
/* /*
* GNU STL in Android NDK lacks support for some C++11 functions, including * GNU STL in Android NDK lacks support for some C++11 functions, including
* stoul, stoi, stod. We reimplement them here using C functions strtoul, * stoul, stoi, stod. We reimplement them here using C functions strtoul,
* strtol, strtod. Note that reimplemented functions are in benchmark:: * strtol, strtod. Note that reimplemented functions are in benchmark::
* namespace, not std:: namespace. * namespace, not std:: namespace.
*/ */
unsigned long stoul(const std::string& str, size_t* pos = nullptr, unsigned long stoul(const std::string& str, size_t* pos = nullptr,
int base = 10); int base = 10);
int stoi(const std::string& str, size_t* pos = nullptr, int base = 10); int stoi(const std::string& str, size_t* pos = nullptr, int base = 10);
double stod(const std::string& str, size_t* pos = nullptr); double stod(const std::string& str, size_t* pos = nullptr);
#else #else
using std::stoul; using std::stod; // NOLINT(misc-unused-using-decls)
using std::stoi; using std::stoi; // NOLINT(misc-unused-using-decls)
using std::stod; using std::stoul; // NOLINT(misc-unused-using-decls)
#endif #endif
// NOLINTEND
} // end namespace benchmark } // end namespace benchmark
#endif // BENCHMARK_STRING_UTIL_H_ #endif // BENCHMARK_STRING_UTIL_H_

third-party/benchmark/src/string_util.cc

Show First 20 Lines • Show All 145 Lines • ▼ Show 20 Lines if (static_cast<std::size_t>(ret) < size)
return std::string(local_buff.data()); return std::string(local_buff.data());
// we did not provide a long enough buffer on our first attempt. // we did not provide a long enough buffer on our first attempt.
// add 1 to size to account for null-byte in size cast to prevent overflow // add 1 to size to account for null-byte in size cast to prevent overflow
size = static_cast<std::size_t>(ret) + 1; size = static_cast<std::size_t>(ret) + 1;
auto buff_ptr = std::unique_ptr<char[]>(new char[size]); auto buff_ptr = std::unique_ptr<char[]>(new char[size]);
// 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation
// in the android-ndk // in the android-ndk
ret = vsnprintf(buff_ptr.get(), size, msg, args); vsnprintf(buff_ptr.get(), size, msg, args);
return std::string(buff_ptr.get()); return std::string(buff_ptr.get());
} }
std::string StrFormat(const char* format, ...) { std::string StrFormat(const char* format, ...) {
va_list args; va_list args;
va_start(args, format); va_start(args, format);
std::string tmp = StrFormatImp(format, args); std::string tmp = StrFormatImp(format, args);
va_end(args); va_end(args);
Show All 30 Linesunsigned long stoul(const std::string& str, size_t* pos, int base) {
const unsigned long result = strtoul(strStart, &strEnd, base); const unsigned long result = strtoul(strStart, &strEnd, base);
const int strtoulErrno = errno; const int strtoulErrno = errno;
/* Restore previous errno */ /* Restore previous errno */
errno = oldErrno; errno = oldErrno;
/* Check for errors and return */ /* Check for errors and return */
if (strtoulErrno == ERANGE) { if (strtoulErrno == ERANGE) {
throw std::out_of_range( throw std::out_of_range("stoul failed: " + str +
"stoul failed: " + str + " is outside of range of unsigned long"); " is outside of range of unsigned long");
} else if (strEnd == strStart || strtoulErrno != 0) { } else if (strEnd == strStart || strtoulErrno != 0) {
throw std::invalid_argument( throw std::invalid_argument("stoul failed: " + str + " is not an integer");
"stoul failed: " + str + " is not an integer");
} }
if (pos != nullptr) { if (pos != nullptr) {
*pos = static_cast<size_t>(strEnd - strStart); *pos = static_cast<size_t>(strEnd - strStart);
} }
return result; return result;
} }
int stoi(const std::string& str, size_t* pos, int base) { int stoi(const std::string& str, size_t* pos, int base) {
/* Record previous errno */ /* Record previous errno */
const int oldErrno = errno; const int oldErrno = errno;
errno = 0; errno = 0;
const char* strStart = str.c_str(); const char* strStart = str.c_str();
char* strEnd = const_cast<char*>(strStart); char* strEnd = const_cast<char*>(strStart);
const long result = strtol(strStart, &strEnd, base); const long result = strtol(strStart, &strEnd, base);
const int strtolErrno = errno; const int strtolErrno = errno;
/* Restore previous errno */ /* Restore previous errno */
errno = oldErrno; errno = oldErrno;
/* Check for errors and return */ /* Check for errors and return */
if (strtolErrno == ERANGE || long(int(result)) != result) { if (strtolErrno == ERANGE || long(int(result)) != result) {
throw std::out_of_range( throw std::out_of_range("stoul failed: " + str +
"stoul failed: " + str + " is outside of range of int"); " is outside of range of int");
} else if (strEnd == strStart || strtolErrno != 0) { } else if (strEnd == strStart || strtolErrno != 0) {
throw std::invalid_argument( throw std::invalid_argument("stoul failed: " + str + " is not an integer");
"stoul failed: " + str + " is not an integer");
} }
if (pos != nullptr) { if (pos != nullptr) {
*pos = static_cast<size_t>(strEnd - strStart); *pos = static_cast<size_t>(strEnd - strStart);
} }
return int(result); return int(result);
} }
double stod(const std::string& str, size_t* pos) { double stod(const std::string& str, size_t* pos) {
/* Record previous errno */ /* Record previous errno */
const int oldErrno = errno; const int oldErrno = errno;
errno = 0; errno = 0;
const char* strStart = str.c_str(); const char* strStart = str.c_str();
char* strEnd = const_cast<char*>(strStart); char* strEnd = const_cast<char*>(strStart);
const double result = strtod(strStart, &strEnd); const double result = strtod(strStart, &strEnd);
/* Restore previous errno */ /* Restore previous errno */
const int strtodErrno = errno; const int strtodErrno = errno;
errno = oldErrno; errno = oldErrno;
/* Check for errors and return */ /* Check for errors and return */
if (strtodErrno == ERANGE) { if (strtodErrno == ERANGE) {
throw std::out_of_range( throw std::out_of_range("stoul failed: " + str +
"stoul failed: " + str + " is outside of range of int"); " is outside of range of int");
} else if (strEnd == strStart || strtodErrno != 0) { } else if (strEnd == strStart || strtodErrno != 0) {
throw std::invalid_argument( throw std::invalid_argument("stoul failed: " + str + " is not an integer");
"stoul failed: " + str + " is not an integer");
} }
if (pos != nullptr) { if (pos != nullptr) {
*pos = static_cast<size_t>(strEnd - strStart); *pos = static_cast<size_t>(strEnd - strStart);
} }
return result; return result;
} }
#endif #endif
} // end namespace benchmark } // end namespace benchmark

third-party/benchmark/src/sysinfo.cc

Show All 13 Lines
#include "internal_macros.h" #include "internal_macros.h"
#ifdef BENCHMARK_OS_WINDOWS #ifdef BENCHMARK_OS_WINDOWS
#include <shlwapi.h> #include <shlwapi.h>
#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA
#include <versionhelpers.h> #include <versionhelpers.h>
#include <windows.h> #include <windows.h>
#include <codecvt> #include <codecvt>
#else #else
#include <fcntl.h> #include <fcntl.h>
#ifndef BENCHMARK_OS_FUCHSIA #ifndef BENCHMARK_OS_FUCHSIA
#include <sys/resource.h> #include <sys/resource.h>
#endif #endif
#include <sys/time.h> #include <sys/time.h>
#include <sys/types.h> // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD #include <sys/types.h> // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD
Show All 20 Lines
#include <cstdint> #include <cstdint>
#include <cstdio> #include <cstdio>
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include <fstream> #include <fstream>
#include <iostream> #include <iostream>
#include <iterator> #include <iterator>
#include <limits> #include <limits>
#include <locale>
#include <memory> #include <memory>
#include <sstream> #include <sstream>
#include <locale>
#include <utility> #include <utility>
#include "check.h" #include "check.h"
#include "cycleclock.h" #include "cycleclock.h"
#include "internal_macros.h" #include "internal_macros.h"
#include "log.h" #include "log.h"
#include "sleep.h" #include "sleep.h"
#include "string_util.h" #include "string_util.h"
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines uint64_t GetAsUnsigned() const {
else if (Size == sizeof(Buff->uint64_value)) else if (Size == sizeof(Buff->uint64_value))
return Buff->uint64_value; return Buff->uint64_value;
BENCHMARK_UNREACHABLE(); BENCHMARK_UNREACHABLE();
} }
template <class T, int N> template <class T, int N>
std::array<T, N> GetAsArray() { std::array<T, N> GetAsArray() {
const int ArrSize = sizeof(T) * N; const int ArrSize = sizeof(T) * N;
CHECK_LE(ArrSize, Size); BM_CHECK_LE(ArrSize, Size);
std::array<T, N> Arr; std::array<T, N> Arr;
std::memcpy(Arr.data(), data(), ArrSize); std::memcpy(Arr.data(), data(), ArrSize);
return Arr; return Arr;
} }
}; };
ValueUnion GetSysctlImp(std::string const& Name) { ValueUnion GetSysctlImp(std::string const& Name) {
#if defined BENCHMARK_OS_OPENBSD #if defined BENCHMARK_OS_OPENBSD
int mib[2]; int mib[2];
mib[0] = CTL_HW; mib[0] = CTL_HW;
if ((Name == "hw.ncpu") || (Name == "hw.cpuspeed")){ if ((Name == "hw.ncpu") || (Name == "hw.cpuspeed")) {
ValueUnion buff(sizeof(int)); ValueUnion buff(sizeof(int));
if (Name == "hw.ncpu") { if (Name == "hw.ncpu") {
mib[1] = HW_NCPU; mib[1] = HW_NCPU;
} else { } else {
mib[1] = HW_CPUSPEED; mib[1] = HW_CPUSPEED;
} }
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Linesbool ReadFromFile(std::string const& fname, ArgT* arg) {
if (!f.is_open()) return false; if (!f.is_open()) return false;
f >> *arg; f >> *arg;
return f.good(); return f.good();
} }
CPUInfo::Scaling CpuScaling(int num_cpus) { CPUInfo::Scaling CpuScaling(int num_cpus) {
// We don't have a valid CPU count, so don't even bother. // We don't have a valid CPU count, so don't even bother.
if (num_cpus <= 0) return CPUInfo::Scaling::UNKNOWN; if (num_cpus <= 0) return CPUInfo::Scaling::UNKNOWN;
#ifdef BENCHMARK_OS_QNX #if defined(BENCHMARK_OS_QNX)
return CPUInfo::Scaling::UNKNOWN; return CPUInfo::Scaling::UNKNOWN;
#endif #elif !defined(BENCHMARK_OS_WINDOWS)
#ifndef BENCHMARK_OS_WINDOWS
// On Linux, the CPUfreq subsystem exposes CPU information as files on the // On Linux, the CPUfreq subsystem exposes CPU information as files on the
// local file system. If reading the exported files fails, then we may not be // local file system. If reading the exported files fails, then we may not be
// running on Linux, so we silently ignore all the read errors. // running on Linux, so we silently ignore all the read errors.
std::string res; std::string res;
for (int cpu = 0; cpu < num_cpus; ++cpu) { for (int cpu = 0; cpu < num_cpus; ++cpu) {
std::string governor_file = std::string governor_file =
StrCat("/sys/devices/system/cpu/cpu", cpu, "/cpufreq/scaling_governor"); StrCat("/sys/devices/system/cpu/cpu", cpu, "/cpufreq/scaling_governor");
if (ReadFromFile(governor_file, &res) && res != "performance") return CPUInfo::Scaling::ENABLED; if (ReadFromFile(governor_file, &res) && res != "performance")
return CPUInfo::Scaling::ENABLED;
} }
return CPUInfo::Scaling::DISABLED; return CPUInfo::Scaling::DISABLED;
#endif #else
return CPUInfo::Scaling::UNKNOWN; return CPUInfo::Scaling::UNKNOWN;
#endif
} }
int CountSetBitsInCPUMap(std::string Val) { int CountSetBitsInCPUMap(std::string Val) {
auto CountBits = [](std::string Part) { auto CountBits = [](std::string Part) {
using CPUMask = std::bitset<sizeof(std::uintptr_t) * CHAR_BIT>; using CPUMask = std::bitset<sizeof(std::uintptr_t) * CHAR_BIT>;
Part = "0x" + Part; Part = "0x" + Part;
CPUMask Mask(benchmark::stoul(Part, nullptr, 16)); CPUMask Mask(benchmark::stoul(Part, nullptr, 16));
return static_cast<int>(Mask.count()); return static_cast<int>(Mask.count());
▲ Show 20 Lines • Show All 120 Lines • ▼ Show 20 Lines for (; it != end; ++it) {
} }
res.push_back(C); res.push_back(C);
} }
return res; return res;
} }
#elif BENCHMARK_OS_QNX #elif BENCHMARK_OS_QNX
std::vector<CPUInfo::CacheInfo> GetCacheSizesQNX() { std::vector<CPUInfo::CacheInfo> GetCacheSizesQNX() {
std::vector<CPUInfo::CacheInfo> res; std::vector<CPUInfo::CacheInfo> res;
struct cacheattr_entry *cache = SYSPAGE_ENTRY(cacheattr); struct cacheattr_entry* cache = SYSPAGE_ENTRY(cacheattr);
uint32_t const elsize = SYSPAGE_ELEMENT_SIZE(cacheattr); uint32_t const elsize = SYSPAGE_ELEMENT_SIZE(cacheattr);
int num = SYSPAGE_ENTRY_SIZE(cacheattr) / elsize ; int num = SYSPAGE_ENTRY_SIZE(cacheattr) / elsize;
for(int i = 0; i < num; ++i ) { for (int i = 0; i < num; ++i) {
CPUInfo::CacheInfo info; CPUInfo::CacheInfo info;
switch (cache->flags){ switch (cache->flags) {
case CACHE_FLAG_INSTR : case CACHE_FLAG_INSTR:
info.type = "Instruction"; info.type = "Instruction";
info.level = 1; info.level = 1;
break; break;
case CACHE_FLAG_DATA : case CACHE_FLAG_DATA:
info.type = "Data"; info.type = "Data";
info.level = 1; info.level = 1;
break; break;
case CACHE_FLAG_UNIFIED : case CACHE_FLAG_UNIFIED:
info.type = "Unified"; info.type = "Unified";
info.level = 2; info.level = 2;
break; break;
case CACHE_FLAG_SHARED : case CACHE_FLAG_SHARED:
info.type = "Shared"; info.type = "Shared";
info.level = 3; info.level = 3;
break; break;
default : default:
continue; continue;
break; break;
} }
info.size = cache->line_size * cache->num_lines; info.size = cache->line_size * cache->num_lines;
info.num_sharing = 0; info.num_sharing = 0;
res.push_back(std::move(info)); res.push_back(std::move(info));
cache = SYSPAGE_ARRAY_ADJ_OFFSET(cacheattr, cache, elsize); cache = SYSPAGE_ARRAY_ADJ_OFFSET(cacheattr, cache, elsize);
} }
Show All 11 Lines
#else #else
return GetCacheSizesFromKVFS(); return GetCacheSizesFromKVFS();
#endif #endif
} }
std::string GetSystemName() { std::string GetSystemName() {
#if defined(BENCHMARK_OS_WINDOWS) #if defined(BENCHMARK_OS_WINDOWS)
std::string str; std::string str;
const unsigned COUNT = MAX_COMPUTERNAME_LENGTH+1; const unsigned COUNT = MAX_COMPUTERNAME_LENGTH + 1;
TCHAR hostname[COUNT] = {'\0'}; TCHAR hostname[COUNT] = {'\0'};
DWORD DWCOUNT = COUNT; DWORD DWCOUNT = COUNT;
if (!GetComputerName(hostname, &DWCOUNT)) if (!GetComputerName(hostname, &DWCOUNT)) return std::string("");
return std::string("");
#ifndef UNICODE #ifndef UNICODE
str = std::string(hostname, DWCOUNT); str = std::string(hostname, DWCOUNT);
#else #else
//Using wstring_convert, Is deprecated in C++17 // Using wstring_convert, Is deprecated in C++17
using convert_type = std::codecvt_utf8<wchar_t>; using convert_type = std::codecvt_utf8<wchar_t>;
std::wstring_convert<convert_type, wchar_t> converter; std::wstring_convert<convert_type, wchar_t> converter;
std::wstring wStr(hostname, DWCOUNT); std::wstring wStr(hostname, DWCOUNT);
str = converter.to_bytes(wStr); str = converter.to_bytes(wStr);
#endif #endif
return str; return str;
#else // defined(BENCHMARK_OS_WINDOWS) #else // defined(BENCHMARK_OS_WINDOWS)
#ifndef HOST_NAME_MAX #ifndef HOST_NAME_MAX
#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined #ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined
#define HOST_NAME_MAX 64 #define HOST_NAME_MAX 64
#elif defined(BENCHMARK_OS_NACL) #elif defined(BENCHMARK_OS_NACL)
#define HOST_NAME_MAX 64 #define HOST_NAME_MAX 64
#elif defined(BENCHMARK_OS_QNX) #elif defined(BENCHMARK_OS_QNX)
#define HOST_NAME_MAX 154 #define HOST_NAME_MAX 154
#elif defined(BENCHMARK_OS_RTEMS) #elif defined(BENCHMARK_OS_RTEMS)
#define HOST_NAME_MAX 256 #define HOST_NAME_MAX 256
#else #else
#warning "HOST_NAME_MAX not defined. using 64" #pragma message("HOST_NAME_MAX not defined. using 64")
#define HOST_NAME_MAX 64 #define HOST_NAME_MAX 64
#endif #endif
#endif // def HOST_NAME_MAX #endif // def HOST_NAME_MAX
char hostname[HOST_NAME_MAX]; char hostname[HOST_NAME_MAX];
int retVal = gethostname(hostname, HOST_NAME_MAX); int retVal = gethostname(hostname, HOST_NAME_MAX);
if (retVal != 0) return std::string(""); if (retVal != 0) return std::string("");
return std::string(hostname); return std::string(hostname);
#endif // Catch-all POSIX block. #endif // Catch-all POSIX block.
} }
int GetNumCPUs() { int GetNumCPUs() {
#ifdef BENCHMARK_HAS_SYSCTL #ifdef BENCHMARK_HAS_SYSCTL
int NumCPU = -1; int NumCPU = -1;
if (GetSysctl("hw.ncpu", &NumCPU)) return NumCPU; if (GetSysctl("hw.ncpu", &NumCPU)) return NumCPU;
fprintf(stderr, "Err: %s\n", strerror(errno)); fprintf(stderr, "Err: %s\n", strerror(errno));
std::exit(EXIT_FAILURE); std::exit(EXIT_FAILURE);
#elif defined(BENCHMARK_OS_WINDOWS) #elif defined(BENCHMARK_OS_WINDOWS)
SYSTEM_INFO sysinfo; SYSTEM_INFO sysinfo;
// Use memset as opposed to = {} to avoid GCC missing initializer false // Use memset as opposed to = {} to avoid GCC missing initializer false
// positives. // positives.
std::memset(&sysinfo, 0, sizeof(SYSTEM_INFO)); std::memset(&sysinfo, 0, sizeof(SYSTEM_INFO));
GetSystemInfo(&sysinfo); GetSystemInfo(&sysinfo);
return sysinfo.dwNumberOfProcessors; // number of logical return sysinfo.dwNumberOfProcessors; // number of logical
// processors in the current // processors in the current
// group // group
#elif defined(BENCHMARK_OS_SOLARIS) #elif defined(BENCHMARK_OS_SOLARIS)
// Returns -1 in case of a failure. // Returns -1 in case of a failure.
int NumCPU = sysconf(_SC_NPROCESSORS_ONLN); int NumCPU = sysconf(_SC_NPROCESSORS_ONLN);
if (NumCPU < 0) { if (NumCPU < 0) {
fprintf(stderr, fprintf(stderr, "sysconf(_SC_NPROCESSORS_ONLN) failed with error: %s\n",
"sysconf(_SC_NPROCESSORS_ONLN) failed with error: %s\n",
strerror(errno)); strerror(errno));
} }
return NumCPU; return NumCPU;
#elif defined(BENCHMARK_OS_QNX) #elif defined(BENCHMARK_OS_QNX)
return static_cast<int>(_syspage_ptr->num_cpu); return static_cast<int>(_syspage_ptr->num_cpu);
#else #else
int NumCPUs = 0; int NumCPUs = 0;
int MaxID = -1; int MaxID = -1;
std::ifstream f("/proc/cpuinfo"); std::ifstream f("/proc/cpuinfo");
if (!f.is_open()) { if (!f.is_open()) {
std::cerr << "failed to open /proc/cpuinfo\n"; std::cerr << "failed to open /proc/cpuinfo\n";
return -1; return -1;
} }
const std::string Key = "processor"; const std::string Key = "processor";
std::string ln; std::string ln;
while (std::getline(f, ln)) { while (std::getline(f, ln)) {
if (ln.empty()) continue; if (ln.empty()) continue;
size_t SplitIdx = ln.find(':'); size_t SplitIdx = ln.find(':');
std::string value; std::string value;
#if defined(__s390__) #if defined(__s390__)
// s390 has another format in /proc/cpuinfo // s390 has another format in /proc/cpuinfo
// it needs to be parsed differently // it needs to be parsed differently
if (SplitIdx != std::string::npos) value = ln.substr(Key.size()+1,SplitIdx-Key.size()-1); if (SplitIdx != std::string::npos)
value = ln.substr(Key.size() + 1, SplitIdx - Key.size() - 1);
#else #else
if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1);
#endif #endif
if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) { if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) {
NumCPUs++; NumCPUs++;
if (!value.empty()) { if (!value.empty()) {
int CurID = benchmark::stoi(value); int CurID = benchmark::stoi(value);
MaxID = std::max(CurID, MaxID); MaxID = std::max(CurID, MaxID);
Show All 30 Lines#if defined BENCHMARK_OS_LINUX || defined BENCHMARK_OS_CYGWIN
// If the kernel is exporting the tsc frequency use that. There are issues // If the kernel is exporting the tsc frequency use that. There are issues
// where cpuinfo_max_freq cannot be relied on because the BIOS may be // where cpuinfo_max_freq cannot be relied on because the BIOS may be
// exporintg an invalid p-state (on x86) or p-states may be used to put the // exporintg an invalid p-state (on x86) or p-states may be used to put the
// processor in a new mode (turbo mode). Essentially, those frequencies // processor in a new mode (turbo mode). Essentially, those frequencies
// cannot always be relied upon. The same reasons apply to /proc/cpuinfo as // cannot always be relied upon. The same reasons apply to /proc/cpuinfo as
// well. // well.
if (ReadFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz", &freq) if (ReadFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz", &freq)
// If CPU scaling is disabled, use the the *current* frequency. // If CPU scaling is disabled, use the *current* frequency.
// Note that we specifically don't want to read cpuinfo_cur_freq, // Note that we specifically don't want to read cpuinfo_cur_freq,
// because it is only readable by root. // because it is only readable by root.
|| (scaling == CPUInfo::Scaling::DISABLED && || (scaling == CPUInfo::Scaling::DISABLED &&
ReadFromFile("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq", ReadFromFile("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq",
&freq)) &freq))
// Otherwise, if CPU scaling may be in effect, we want to use // Otherwise, if CPU scaling may be in effect, we want to use
// the *maximum* frequency, not whatever CPU speed some random processor // the *maximum* frequency, not whatever CPU speed some random processor
// happens to be using now. // happens to be using now.
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines#elif defined BENCHMARK_OS_WINDOWS
DWORD data, data_size = sizeof(data); DWORD data, data_size = sizeof(data);
if (IsWindowsXPOrGreater() && if (IsWindowsXPOrGreater() &&
SUCCEEDED( SUCCEEDED(
SHGetValueA(HKEY_LOCAL_MACHINE, SHGetValueA(HKEY_LOCAL_MACHINE,
"HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0",
"~MHz", nullptr, &data, &data_size))) "~MHz", nullptr, &data, &data_size)))
return static_cast<double>((int64_t)data * return static_cast<double>((int64_t)data *
(int64_t)(1000 * 1000)); // was mhz (int64_t)(1000 * 1000)); // was mhz
#elif defined (BENCHMARK_OS_SOLARIS) #elif defined(BENCHMARK_OS_SOLARIS)
kstat_ctl_t *kc = kstat_open(); kstat_ctl_t* kc = kstat_open();
if (!kc) { if (!kc) {
std::cerr << "failed to open /dev/kstat\n"; std::cerr << "failed to open /dev/kstat\n";
return -1; return -1;
} }
kstat_t *ksp = kstat_lookup(kc, (char*)"cpu_info", -1, (char*)"cpu_info0"); kstat_t* ksp = kstat_lookup(kc, (char*)"cpu_info", -1, (char*)"cpu_info0");
if (!ksp) { if (!ksp) {
std::cerr << "failed to lookup in /dev/kstat\n"; std::cerr << "failed to lookup in /dev/kstat\n";
return -1; return -1;
} }
if (kstat_read(kc, ksp, NULL) < 0) { if (kstat_read(kc, ksp, NULL) < 0) {
std::cerr << "failed to read from /dev/kstat\n"; std::cerr << "failed to read from /dev/kstat\n";
return -1; return -1;
} }
kstat_named_t *knp = kstat_named_t* knp =
(kstat_named_t*)kstat_data_lookup(ksp, (char*)"current_clock_Hz"); (kstat_named_t*)kstat_data_lookup(ksp, (char*)"current_clock_Hz");
if (!knp) { if (!knp) {
std::cerr << "failed to lookup data in /dev/kstat\n"; std::cerr << "failed to lookup data in /dev/kstat\n";
return -1; return -1;
} }
if (knp->data_type != KSTAT_DATA_UINT64) { if (knp->data_type != KSTAT_DATA_UINT64) {
std::cerr << "current_clock_Hz is of unexpected data type: " std::cerr << "current_clock_Hz is of unexpected data type: "
<< knp->data_type << "\n"; << knp->data_type << "\n";
return -1; return -1;
} }
double clock_hz = knp->value.ui64; double clock_hz = knp->value.ui64;
kstat_close(kc); kstat_close(kc);
return clock_hz; return clock_hz;
#elif defined (BENCHMARK_OS_QNX) #elif defined(BENCHMARK_OS_QNX)
return static_cast<double>((int64_t)(SYSPAGE_ENTRY(cpuinfo)->speed) * return static_cast<double>((int64_t)(SYSPAGE_ENTRY(cpuinfo)->speed) *
(int64_t)(1000 * 1000)); (int64_t)(1000 * 1000));
#endif #endif
// If we've fallen through, attempt to roughly estimate the CPU clock rate. // If we've fallen through, attempt to roughly estimate the CPU clock rate.
const int estimate_time_ms = 1000; const int estimate_time_ms = 1000;
const auto start_ticks = cycleclock::Now(); const auto start_ticks = cycleclock::Now();
SleepForMilliseconds(estimate_time_ms); SleepForMilliseconds(estimate_time_ms);
return static_cast<double>(cycleclock::Now() - start_ticks); return static_cast<double>(cycleclock::Now() - start_ticks);
▲ Show 20 Lines • Show All 42 Lines • Show Last 20 Lines

third-party/benchmark/src/thread_manager.h

Show All 30 Lines public:
} }
void WaitForAllThreads() EXCLUDES(end_cond_mutex_) { void WaitForAllThreads() EXCLUDES(end_cond_mutex_) {
MutexLock lock(end_cond_mutex_); MutexLock lock(end_cond_mutex_);
end_condition_.wait(lock.native_handle(), end_condition_.wait(lock.native_handle(),
[this]() { return alive_threads_ == 0; }); [this]() { return alive_threads_ == 0; });
} }
public:
struct Result { struct Result {
IterationCount iterations = 0; IterationCount iterations = 0;
double real_time_used = 0; double real_time_used = 0;
double cpu_time_used = 0; double cpu_time_used = 0;
double manual_time_used = 0; double manual_time_used = 0;
int64_t complexity_n = 0; int64_t complexity_n = 0;
std::string report_label_; std::string report_label_;
std::string error_message_; std::string error_message_;
Show All 17 Lines

third-party/benchmark/src/thread_timer.h

Show All 22 Lines public:
void StartTimer() { void StartTimer() {
running_ = true; running_ = true;
start_real_time_ = ChronoClockNow(); start_real_time_ = ChronoClockNow();
start_cpu_time_ = ReadCpuTimerOfChoice(); start_cpu_time_ = ReadCpuTimerOfChoice();
} }
// Called by each thread // Called by each thread
void StopTimer() { void StopTimer() {
CHECK(running_); BM_CHECK(running_);
running_ = false; running_ = false;
real_time_used_ += ChronoClockNow() - start_real_time_; real_time_used_ += ChronoClockNow() - start_real_time_;
// Floating point error can result in the subtraction producing a negative // Floating point error can result in the subtraction producing a negative
// time. Guard against that. // time. Guard against that.
cpu_time_used_ += cpu_time_used_ +=
std::max<double>(ReadCpuTimerOfChoice() - start_cpu_time_, 0); std::max<double>(ReadCpuTimerOfChoice() - start_cpu_time_, 0);
} }
// Called by each thread // Called by each thread
void SetIterationTime(double seconds) { manual_time_used_ += seconds; } void SetIterationTime(double seconds) { manual_time_used_ += seconds; }
bool running() const { return running_; } bool running() const { return running_; }
// REQUIRES: timer is not running // REQUIRES: timer is not running
double real_time_used() const { double real_time_used() const {
CHECK(!running_); BM_CHECK(!running_);
return real_time_used_; return real_time_used_;
} }
// REQUIRES: timer is not running // REQUIRES: timer is not running
double cpu_time_used() const { double cpu_time_used() const {
CHECK(!running_); BM_CHECK(!running_);
return cpu_time_used_; return cpu_time_used_;
} }
// REQUIRES: timer is not running // REQUIRES: timer is not running
double manual_time_used() const { double manual_time_used() const {
CHECK(!running_); BM_CHECK(!running_);
return manual_time_used_; return manual_time_used_;
} }
private: private:
double ReadCpuTimerOfChoice() const { double ReadCpuTimerOfChoice() const {
if (measure_process_cpu_time) return ProcessCPUUsage(); if (measure_process_cpu_time) return ProcessCPUUsage();
return ThreadCPUUsage(); return ThreadCPUUsage();
} }
Show All 19 Lines

third-party/benchmark/src/timers.cc

// Copyright 2015 Google Inc. All rights reserved. // Copyright 2015 Google Inc. All rights reserved.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
// You may obtain a copy of the License at // You may obtain a copy of the License at
// //
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
// //
// Unless required by applicable law or agreed to in writing, software // Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "timers.h" #include "timers.h"
#include "internal_macros.h" #include "internal_macros.h"
#ifdef BENCHMARK_OS_WINDOWS #ifdef BENCHMARK_OS_WINDOWS
#include <shlwapi.h> #include <shlwapi.h>
#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA
#include <versionhelpers.h> #include <versionhelpers.h>
#include <windows.h> #include <windows.h>
#else #else
▲ Show 20 Lines • Show All 96 Lines • ▼ Show 20 Lines#if defined(BENCHMARK_OS_WINDOWS)
DiagnoseAndExit("GetProccessTimes() failed"); DiagnoseAndExit("GetProccessTimes() failed");
#elif defined(BENCHMARK_OS_EMSCRIPTEN) #elif defined(BENCHMARK_OS_EMSCRIPTEN)
// clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) returns 0 on Emscripten. // clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) returns 0 on Emscripten.
// Use Emscripten-specific API. Reported CPU time would be exactly the // Use Emscripten-specific API. Reported CPU time would be exactly the
// same as total time, but this is ok because there aren't long-latency // same as total time, but this is ok because there aren't long-latency
// syncronous system calls in Emscripten. // syncronous system calls in Emscripten.
return emscripten_get_now() * 1e-3; return emscripten_get_now() * 1e-3;
#elif defined(CLOCK_PROCESS_CPUTIME_ID) && !defined(BENCHMARK_OS_MACOSX) #elif defined(CLOCK_PROCESS_CPUTIME_ID) && !defined(BENCHMARK_OS_MACOSX)
// FIXME We want to use clock_gettime, but its not available in MacOS 10.11. See // FIXME We want to use clock_gettime, but its not available in MacOS 10.11.
// https://github.com/google/benchmark/pull/292 // See https://github.com/google/benchmark/pull/292
struct timespec spec; struct timespec spec;
if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &spec) == 0) if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &spec) == 0)
return MakeTime(spec); return MakeTime(spec);
DiagnoseAndExit("clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) failed"); DiagnoseAndExit("clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) failed");
#else #else
struct rusage ru; struct rusage ru;
if (getrusage(RUSAGE_SELF, &ru) == 0) return MakeTime(ru); if (getrusage(RUSAGE_SELF, &ru) == 0) return MakeTime(ru);
DiagnoseAndExit("getrusage(RUSAGE_SELF, ...) failed"); DiagnoseAndExit("getrusage(RUSAGE_SELF, ...) failed");
#endif #endif
} }
double ThreadCPUUsage() { double ThreadCPUUsage() {
#if defined(BENCHMARK_OS_WINDOWS) #if defined(BENCHMARK_OS_WINDOWS)
HANDLE this_thread = GetCurrentThread(); HANDLE this_thread = GetCurrentThread();
FILETIME creation_time; FILETIME creation_time;
FILETIME exit_time; FILETIME exit_time;
FILETIME kernel_time; FILETIME kernel_time;
FILETIME user_time; FILETIME user_time;
GetThreadTimes(this_thread, &creation_time, &exit_time, &kernel_time, GetThreadTimes(this_thread, &creation_time, &exit_time, &kernel_time,
&user_time); &user_time);
return MakeTime(kernel_time, user_time); return MakeTime(kernel_time, user_time);
#elif defined(BENCHMARK_OS_MACOSX) #elif defined(BENCHMARK_OS_MACOSX)
// FIXME We want to use clock_gettime, but its not available in MacOS 10.11. See // FIXME We want to use clock_gettime, but its not available in MacOS 10.11.
// https://github.com/google/benchmark/pull/292 // See https://github.com/google/benchmark/pull/292
mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT; mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT;
thread_basic_info_data_t info; thread_basic_info_data_t info;
mach_port_t thread = pthread_mach_thread_np(pthread_self()); mach_port_t thread = pthread_mach_thread_np(pthread_self());
if (thread_info(thread, THREAD_BASIC_INFO, (thread_info_t)&info, &count) == if (thread_info(thread, THREAD_BASIC_INFO,
KERN_SUCCESS) { reinterpret_cast<thread_info_t>(&info),
&count) == KERN_SUCCESS) {
return MakeTime(info); return MakeTime(info);
} }
DiagnoseAndExit("ThreadCPUUsage() failed when evaluating thread_info"); DiagnoseAndExit("ThreadCPUUsage() failed when evaluating thread_info");
#elif defined(BENCHMARK_OS_EMSCRIPTEN) #elif defined(BENCHMARK_OS_EMSCRIPTEN)
// Emscripten doesn't support traditional threads // Emscripten doesn't support traditional threads
return ProcessCPUUsage(); return ProcessCPUUsage();
#elif defined(BENCHMARK_OS_RTEMS) #elif defined(BENCHMARK_OS_RTEMS)
// RTEMS doesn't support CLOCK_THREAD_CPUTIME_ID. See // RTEMS doesn't support CLOCK_THREAD_CPUTIME_ID. See
Show All 19 Linesstd::string LocalDateTimeString() {
const std::size_t kTzOffsetLen = 6; const std::size_t kTzOffsetLen = 6;
const std::size_t kTimestampLen = 19; const std::size_t kTimestampLen = 19;
std::size_t tz_len; std::size_t tz_len;
std::size_t timestamp_len; std::size_t timestamp_len;
long int offset_minutes; long int offset_minutes;
char tz_offset_sign = '+'; char tz_offset_sign = '+';
// tz_offset is set in one of three ways: // tz_offset is set in one of three ways:
// * strftime with %z - This either returns empty or the ISO 8601 time. The maximum length an // * strftime with %z - This either returns empty or the ISO 8601 time. The
// maximum length an
// ISO 8601 string can be is 7 (e.g. -03:30, plus trailing zero). // ISO 8601 string can be is 7 (e.g. -03:30, plus trailing zero).
// * snprintf with %c%02li:%02li - The maximum length is 41 (one for %c, up to 19 for %02li, // * snprintf with %c%02li:%02li - The maximum length is 41 (one for %c, up to
// 19 for %02li,
// one for :, up to 19 %02li, plus trailing zero). // one for :, up to 19 %02li, plus trailing zero).
// * A fixed string of "-00:00". The maximum length is 7 (-00:00, plus trailing zero). // * A fixed string of "-00:00". The maximum length is 7 (-00:00, plus
// trailing zero).
// //
// Thus, the maximum size this needs to be is 41. // Thus, the maximum size this needs to be is 41.
char tz_offset[41]; char tz_offset[41];
// Long enough buffer to avoid format-overflow warnings // Long enough buffer to avoid format-overflow warnings
char storage[128]; char storage[128];
#if defined(BENCHMARK_OS_WINDOWS) #if defined(BENCHMARK_OS_WINDOWS)
std::tm *timeinfo_p = ::localtime(&now); std::tm* timeinfo_p = ::localtime(&now);
#else #else
std::tm timeinfo; std::tm timeinfo;
std::tm *timeinfo_p = &timeinfo; std::tm* timeinfo_p = &timeinfo;
::localtime_r(&now, &timeinfo); ::localtime_r(&now, &timeinfo);
#endif #endif
tz_len = std::strftime(tz_offset, sizeof(tz_offset), "%z", timeinfo_p); tz_len = std::strftime(tz_offset, sizeof(tz_offset), "%z", timeinfo_p);
if (tz_len < kTzOffsetLen && tz_len > 1) { if (tz_len < kTzOffsetLen && tz_len > 1) {
// Timezone offset was written. strftime writes offset as +HHMM or -HHMM, // Timezone offset was written. strftime writes offset as +HHMM or -HHMM,
// RFC3339 specifies an offset as +HH:MM or -HH:MM. To convert, we parse // RFC3339 specifies an offset as +HH:MM or -HH:MM. To convert, we parse
// the offset as an integer, then reprint it to a string. // the offset as an integer, then reprint it to a string.
offset_minutes = ::strtol(tz_offset, NULL, 10); offset_minutes = ::strtol(tz_offset, NULL, 10);
if (offset_minutes < 0) { if (offset_minutes < 0) {
offset_minutes *= -1; offset_minutes *= -1;
tz_offset_sign = '-'; tz_offset_sign = '-';
} }
tz_len = ::snprintf(tz_offset, sizeof(tz_offset), "%c%02li:%02li", tz_len =
::snprintf(tz_offset, sizeof(tz_offset), "%c%02li:%02li",
tz_offset_sign, offset_minutes / 100, offset_minutes % 100); tz_offset_sign, offset_minutes / 100, offset_minutes % 100);
CHECK(tz_len == kTzOffsetLen); BM_CHECK(tz_len == kTzOffsetLen);
((void)tz_len); // Prevent unused variable warning in optimized build. ((void)tz_len); // Prevent unused variable warning in optimized build.
} else { } else {
// Unknown offset. RFC3339 specifies that unknown local offsets should be // Unknown offset. RFC3339 specifies that unknown local offsets should be
// written as UTC time with -00:00 timezone. // written as UTC time with -00:00 timezone.
#if defined(BENCHMARK_OS_WINDOWS) #if defined(BENCHMARK_OS_WINDOWS)
// Potential race condition if another thread calls localtime or gmtime. // Potential race condition if another thread calls localtime or gmtime.
timeinfo_p = ::gmtime(&now); timeinfo_p = ::gmtime(&now);
#else #else
::gmtime_r(&now, &timeinfo); ::gmtime_r(&now, &timeinfo);
#endif #endif
strncpy(tz_offset, "-00:00", kTzOffsetLen + 1); strncpy(tz_offset, "-00:00", kTzOffsetLen + 1);
} }
timestamp_len = std::strftime(storage, sizeof(storage), "%Y-%m-%dT%H:%M:%S", timestamp_len =
timeinfo_p); std::strftime(storage, sizeof(storage), "%Y-%m-%dT%H:%M:%S", timeinfo_p);
CHECK(timestamp_len == kTimestampLen); BM_CHECK(timestamp_len == kTimestampLen);
// Prevent unused variable warning in optimized build. // Prevent unused variable warning in optimized build.
((void)kTimestampLen); ((void)kTimestampLen);
std::strncat(storage, tz_offset, sizeof(storage) - timestamp_len - 1); std::strncat(storage, tz_offset, sizeof(storage) - timestamp_len - 1);
return std::string(storage); return std::string(storage);
} }
} // end namespace benchmark } // end namespace benchmark

third-party/benchmark/test/BUILD

Show All 15 LinesPER_SRC_COPTS = ({
"donotoptimize_test.cc": ["-O3"], "donotoptimize_test.cc": ["-O3"],
}) })
TEST_ARGS = ["--benchmark_min_time=0.01"] TEST_ARGS = ["--benchmark_min_time=0.01"]
PER_SRC_TEST_ARGS = ({ PER_SRC_TEST_ARGS = ({
"user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"],
"repetitions_test.cc": [" --benchmark_repetitions=3"], "repetitions_test.cc": [" --benchmark_repetitions=3"],
"spec_arg_test.cc" : ["--benchmark_filter=BM_NotChosen"],
}) })
load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
cc_library( cc_library(
name = "output_test_helper", name = "output_test_helper",
testonly = 1, testonly = 1,
srcs = ["output_test_helper.cc"], srcs = ["output_test_helper.cc"],
▲ Show 20 Lines • Show All 43 Lines • Show Last 20 Lines

third-party/benchmark/test/CMakeLists.txt

Show First 20 Lines • Show All 50 Lines • ▼ Show 20 Linesmacro(compile_output_test name)
target_link_libraries(${name} output_test_helper benchmark::benchmark target_link_libraries(${name} output_test_helper benchmark::benchmark
${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
endmacro(compile_output_test) endmacro(compile_output_test)
# Demonstration executable # Demonstration executable
compile_benchmark_test(benchmark_test) compile_benchmark_test(benchmark_test)
add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01) add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01)
compile_benchmark_test(spec_arg_test)
add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen)
compile_benchmark_test(benchmark_setup_teardown_test)
add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test)
compile_benchmark_test(filter_test) compile_benchmark_test(filter_test)
macro(add_filter_test name filter expect) macro(add_filter_test name filter expect)
add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect})
add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect})
endmacro(add_filter_test) endmacro(add_filter_test)
add_filter_test(filter_simple "Foo" 3) add_filter_test(filter_simple "Foo" 3)
add_filter_test(filter_simple_negative "-Foo" 2) add_filter_test(filter_simple_negative "-Foo" 2)
▲ Show 20 Lines • Show All 205 Lines • Show Last 20 Lines

third-party/benchmark/test/args_product_test.cc

#include "benchmark/benchmark.h"
#include <cassert> #include <cassert>
#include <iostream> #include <iostream>
#include <set> #include <set>
#include <vector> #include <vector>
#include "benchmark/benchmark.h"
class ArgsProductFixture : public ::benchmark::Fixture { class ArgsProductFixture : public ::benchmark::Fixture {
public: public:
ArgsProductFixture() ArgsProductFixture()
: expectedValues({{0, 100, 2000, 30000}, : expectedValues({{0, 100, 2000, 30000},
{1, 15, 3, 8}, {1, 15, 3, 8},
{1, 15, 3, 9}, {1, 15, 3, 9},
{1, 15, 7, 8}, {1, 15, 7, 8},
{1, 15, 7, 9}, {1, 15, 7, 9},
Show All 16 Lines void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE {
actualValues.insert(ranges); actualValues.insert(ranges);
} }
// NOTE: This is not TearDown as we want to check after _all_ runs are // NOTE: This is not TearDown as we want to check after _all_ runs are
// complete. // complete.
virtual ~ArgsProductFixture() { virtual ~ArgsProductFixture() {
if (actualValues != expectedValues) { if (actualValues != expectedValues) {
std::cout << "EXPECTED\n"; std::cout << "EXPECTED\n";
for (auto v : expectedValues) { for (const auto& v : expectedValues) {
std::cout << "{"; std::cout << "{";
for (int64_t iv : v) { for (int64_t iv : v) {
std::cout << iv << ", "; std::cout << iv << ", ";
} }
std::cout << "}\n"; std::cout << "}\n";
} }
std::cout << "ACTUAL\n"; std::cout << "ACTUAL\n";
for (auto v : actualValues) { for (const auto& v : actualValues) {
std::cout << "{"; std::cout << "{";
for (int64_t iv : v) { for (int64_t iv : v) {
std::cout << iv << ", "; std::cout << iv << ", ";
} }
std::cout << "}\n"; std::cout << "}\n";
} }
} }
} }
Show All 21 Lines

third-party/benchmark/test/basic_test.cc

#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#define BASIC_BENCHMARK_TEST(x) BENCHMARK(x)->Arg(8)->Arg(512)->Arg(8192) #define BASIC_BENCHMARK_TEST(x) BENCHMARK(x)->Arg(8)->Arg(512)->Arg(8192)
void BM_empty(benchmark::State& state) { void BM_empty(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
benchmark::DoNotOptimize(state.iterations()); benchmark::DoNotOptimize(state.iterations());
} }
} }
BENCHMARK(BM_empty); BENCHMARK(BM_empty);
BENCHMARK(BM_empty)->ThreadPerCpu(); BENCHMARK(BM_empty)->ThreadPerCpu();
void BM_spin_empty(benchmark::State& state) { void BM_spin_empty(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
for (int x = 0; x < state.range(0); ++x) { for (auto x = 0; x < state.range(0); ++x) {
benchmark::DoNotOptimize(x); benchmark::DoNotOptimize(x);
} }
} }
} }
BASIC_BENCHMARK_TEST(BM_spin_empty); BASIC_BENCHMARK_TEST(BM_spin_empty);
BASIC_BENCHMARK_TEST(BM_spin_empty)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_empty)->ThreadPerCpu();
void BM_spin_pause_before(benchmark::State& state) { void BM_spin_pause_before(benchmark::State& state) {
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
for (auto _ : state) { for (auto _ : state) {
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
} }
} }
BASIC_BENCHMARK_TEST(BM_spin_pause_before); BASIC_BENCHMARK_TEST(BM_spin_pause_before);
BASIC_BENCHMARK_TEST(BM_spin_pause_before)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_pause_before)->ThreadPerCpu();
void BM_spin_pause_during(benchmark::State& state) { void BM_spin_pause_during(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
state.PauseTiming(); state.PauseTiming();
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
state.ResumeTiming(); state.ResumeTiming();
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
} }
} }
BASIC_BENCHMARK_TEST(BM_spin_pause_during); BASIC_BENCHMARK_TEST(BM_spin_pause_during);
BASIC_BENCHMARK_TEST(BM_spin_pause_during)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_pause_during)->ThreadPerCpu();
void BM_pause_during(benchmark::State& state) { void BM_pause_during(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
state.PauseTiming(); state.PauseTiming();
state.ResumeTiming(); state.ResumeTiming();
} }
} }
BENCHMARK(BM_pause_during); BENCHMARK(BM_pause_during);
BENCHMARK(BM_pause_during)->ThreadPerCpu(); BENCHMARK(BM_pause_during)->ThreadPerCpu();
BENCHMARK(BM_pause_during)->UseRealTime(); BENCHMARK(BM_pause_during)->UseRealTime();
BENCHMARK(BM_pause_during)->UseRealTime()->ThreadPerCpu(); BENCHMARK(BM_pause_during)->UseRealTime()->ThreadPerCpu();
void BM_spin_pause_after(benchmark::State& state) { void BM_spin_pause_after(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
} }
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
} }
BASIC_BENCHMARK_TEST(BM_spin_pause_after); BASIC_BENCHMARK_TEST(BM_spin_pause_after);
BASIC_BENCHMARK_TEST(BM_spin_pause_after)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_pause_after)->ThreadPerCpu();
void BM_spin_pause_before_and_after(benchmark::State& state) { void BM_spin_pause_before_and_after(benchmark::State& state) {
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
for (auto _ : state) { for (auto _ : state) {
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
} }
for (int i = 0; i < state.range(0); ++i) { for (auto i = 0; i < state.range(0); ++i) {
benchmark::DoNotOptimize(i); benchmark::DoNotOptimize(i);
} }
} }
BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after); BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after);
BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after)->ThreadPerCpu(); BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after)->ThreadPerCpu();
void BM_empty_stop_start(benchmark::State& state) { void BM_empty_stop_start(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
} }
} }
BENCHMARK(BM_empty_stop_start); BENCHMARK(BM_empty_stop_start);
BENCHMARK(BM_empty_stop_start)->ThreadPerCpu(); BENCHMARK(BM_empty_stop_start)->ThreadPerCpu();
void BM_KeepRunning(benchmark::State& state) { void BM_KeepRunning(benchmark::State& state) {
benchmark::IterationCount iter_count = 0; benchmark::IterationCount iter_count = 0;
assert(iter_count == state.iterations()); assert(iter_count == state.iterations());
while (state.KeepRunning()) { while (state.KeepRunning()) {
++iter_count; ++iter_count;
} }
assert(iter_count == state.iterations()); assert(iter_count == state.iterations());
} }
Show All 29 Linesvoid BM_RangedFor(benchmark::State& state) {
benchmark::IterationCount iter_count = 0; benchmark::IterationCount iter_count = 0;
for (auto _ : state) { for (auto _ : state) {
++iter_count; ++iter_count;
} }
assert(iter_count == state.max_iterations); assert(iter_count == state.max_iterations);
} }
BENCHMARK(BM_RangedFor); BENCHMARK(BM_RangedFor);
#ifdef BENCHMARK_HAS_CXX11
template <typename T>
void BM_OneTemplateFunc(benchmark::State& state) {
auto arg = state.range(0);
T sum = 0;
for (auto _ : state) {
sum += arg;
}
}
BENCHMARK(BM_OneTemplateFunc<int>)->Arg(1);
BENCHMARK(BM_OneTemplateFunc<double>)->Arg(1);
template <typename A, typename B>
void BM_TwoTemplateFunc(benchmark::State& state) {
auto arg = state.range(0);
A sum = 0;
B prod = 1;
for (auto _ : state) {
sum += arg;
prod *= arg;
}
}
BENCHMARK(BM_TwoTemplateFunc<int, double>)->Arg(1);
BENCHMARK(BM_TwoTemplateFunc<double, int>)->Arg(1);
#endif // BENCHMARK_HAS_CXX11
// Ensure that StateIterator provides all the necessary typedefs required to // Ensure that StateIterator provides all the necessary typedefs required to
// instantiate std::iterator_traits. // instantiate std::iterator_traits.
static_assert(std::is_same< static_assert(
typename std::iterator_traits<benchmark::State::StateIterator>::value_type, std::is_same<typename std::iterator_traits<
typename benchmark::State::StateIterator::value_type>::value, ""); benchmark::State::StateIterator>::value_type,
typename benchmark::State::StateIterator::value_type>::value,
"");
BENCHMARK_MAIN(); BENCHMARK_MAIN();

third-party/benchmark/test/benchmark_random_interleaving_gtest.cc

#include <queue> #include <queue>
#include <string> #include <string>
#include <vector> #include <vector>
#include "../src/commandlineflags.h" #include "../src/commandlineflags.h"
#include "../src/string_util.h" #include "../src/string_util.h"
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "gmock/gmock.h" #include "gmock/gmock.h"
#include "gtest/gtest.h" #include "gtest/gtest.h"
DECLARE_bool(benchmark_enable_random_interleaving);
DECLARE_string(benchmark_filter);
DECLARE_int32(benchmark_repetitions);
namespace benchmark { namespace benchmark {
BM_DECLARE_bool(benchmark_enable_random_interleaving);
BM_DECLARE_string(benchmark_filter);
BM_DECLARE_int32(benchmark_repetitions);
namespace internal { namespace internal {
namespace { namespace {
class EventQueue : public std::queue<std::string> { class EventQueue : public std::queue<std::string> {
public: public:
void Put(const std::string& event) { push(event); } void Put(const std::string& event) { push(event); }
void Clear() { void Clear() {
while (!empty()) { while (!empty()) {
pop(); pop();
} }
} }
std::string Get() { std::string Get() {
std::string event = front(); std::string event = front();
pop(); pop();
return event; return event;
} }
}; };
static EventQueue* queue = new EventQueue; EventQueue* queue = new EventQueue();
class NullReporter : public BenchmarkReporter { class NullReporter : public BenchmarkReporter {
public: public:
bool ReportContext(const Context& /*context*/) override { return true; } bool ReportContext(const Context& /*context*/) override { return true; }
void ReportRuns(const std::vector<Run>& /* report */) override {} void ReportRuns(const std::vector<Run>& /* report */) override {}
}; };
class BenchmarkTest : public testing::Test { class BenchmarkTest : public testing::Test {
Show All 9 Lines void Execute(const std::string& pattern) {
FLAGS_benchmark_filter = pattern; FLAGS_benchmark_filter = pattern;
RunSpecifiedBenchmarks(reporter); RunSpecifiedBenchmarks(reporter);
delete reporter; delete reporter;
queue->Put("DONE"); // End marker queue->Put("DONE"); // End marker
} }
}; };
static void BM_Match1(benchmark::State& state) { void BM_Match1(benchmark::State& state) {
const int64_t arg = state.range(0); const int64_t arg = state.range(0);
for (auto _ : state) { for (auto _ : state) {
} }
queue->Put(StrFormat("BM_Match1/%d", static_cast<int>(arg))); queue->Put(StrFormat("BM_Match1/%d", static_cast<int>(arg)));
} }
BENCHMARK(BM_Match1) BENCHMARK(BM_Match1)
->Iterations(100) ->Iterations(100)
Show All 34 LinesTEST_F(BenchmarkTest, Match1WithRandomInterleaving) {
std::map<std::string, int> element_count; std::map<std::string, int> element_count;
std::map<std::string, int> interleaving_count; std::map<std::string, int> interleaving_count;
Execute("BM_Match1/(64|80)"); Execute("BM_Match1/(64|80)");
for (int i = 0; i < 100; ++i) { for (int i = 0; i < 100; ++i) {
std::vector<std::string> interleaving; std::vector<std::string> interleaving;
interleaving.push_back(queue->Get()); interleaving.push_back(queue->Get());
interleaving.push_back(queue->Get()); interleaving.push_back(queue->Get());
element_count[interleaving[0].c_str()]++; element_count[interleaving[0]]++;
element_count[interleaving[1].c_str()]++; element_count[interleaving[1]]++;
interleaving_count[StrFormat("%s,%s", interleaving[0].c_str(), interleaving_count[StrFormat("%s,%s", interleaving[0].c_str(),
interleaving[1].c_str())]++; interleaving[1].c_str())]++;
} }
EXPECT_EQ(element_count["BM_Match1/64"], 100) << "Unexpected repetitions."; EXPECT_EQ(element_count["BM_Match1/64"], 100) << "Unexpected repetitions.";
EXPECT_EQ(element_count["BM_Match1/80"], 100) << "Unexpected repetitions."; EXPECT_EQ(element_count["BM_Match1/80"], 100) << "Unexpected repetitions.";
EXPECT_GE(interleaving_count.size(), 2) << "Interleaving was not randomized."; EXPECT_GE(interleaving_count.size(), 2) << "Interleaving was not randomized.";
ASSERT_EQ("DONE", queue->Get()); ASSERT_EQ("DONE", queue->Get());
} }
} // namespace } // namespace
} // namespace internal } // namespace internal
} // namespace benchmark } // namespace benchmark

third-party/benchmark/test/benchmark_setup_teardown_test.cc

  • This file was added.
#include <atomic>
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <limits>
#include <string>
#include "benchmark/benchmark.h"
// Test that Setup() and Teardown() are called exactly once
// for each benchmark run (single-threaded).
namespace single {
static int setup_call = 0;
static int teardown_call = 0;
} // namespace single
static void DoSetup1(const benchmark::State& state) {
++single::setup_call;
// Setup/Teardown should never be called with any thread_idx != 0.
assert(state.thread_index() == 0);
}
static void DoTeardown1(const benchmark::State& state) {
++single::teardown_call;
assert(state.thread_index() == 0);
}
static void BM_with_setup(benchmark::State& state) {
for (auto s : state) {
}
}
BENCHMARK(BM_with_setup)
->Arg(1)
->Arg(3)
->Arg(5)
->Arg(7)
->Iterations(100)
->Setup(DoSetup1)
->Teardown(DoTeardown1);
// Test that Setup() and Teardown() are called once for each group of threads.
namespace concurrent {
static std::atomic<int> setup_call(0);
static std::atomic<int> teardown_call(0);
static std::atomic<int> func_call(0);
} // namespace concurrent
static void DoSetup2(const benchmark::State& state) {
concurrent::setup_call.fetch_add(1, std::memory_order_acquire);
assert(state.thread_index() == 0);
}
static void DoTeardown2(const benchmark::State& state) {
concurrent::teardown_call.fetch_add(1, std::memory_order_acquire);
assert(state.thread_index() == 0);
}
static void BM_concurrent(benchmark::State& state) {
for (auto s : state) {
}
concurrent::func_call.fetch_add(1, std::memory_order_acquire);
}
BENCHMARK(BM_concurrent)
->Setup(DoSetup2)
->Teardown(DoTeardown2)
->Iterations(100)
->Threads(5)
->Threads(10)
->Threads(15);
// Testing interaction with Fixture::Setup/Teardown
namespace fixture_interaction {
int setup = 0;
int fixture_setup = 0;
} // namespace fixture_interaction
#define FIXTURE_BECHMARK_NAME MyFixture
class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture {
public:
void SetUp(const ::benchmark::State&) BENCHMARK_OVERRIDE {
fixture_interaction::fixture_setup++;
}
~FIXTURE_BECHMARK_NAME() {}
};
BENCHMARK_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)(benchmark::State& st) {
for (auto _ : st) {
}
}
static void DoSetupWithFixture(const benchmark::State&) {
fixture_interaction::setup++;
}
BENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)
->Arg(1)
->Arg(3)
->Arg(5)
->Arg(7)
->Setup(DoSetupWithFixture)
->Repetitions(1)
->Iterations(100);
// Testing repetitions.
namespace repetitions {
int setup = 0;
}
static void DoSetupWithRepetitions(const benchmark::State&) {
repetitions::setup++;
}
static void BM_WithRep(benchmark::State& state) {
for (auto _ : state) {
}
}
BENCHMARK(BM_WithRep)
->Arg(1)
->Arg(3)
->Arg(5)
->Arg(7)
->Setup(DoSetupWithRepetitions)
->Iterations(100)
->Repetitions(4);
int main(int argc, char** argv) {
benchmark::Initialize(&argc, argv);
size_t ret = benchmark::RunSpecifiedBenchmarks(".");
assert(ret > 0);
// Setup/Teardown is called once for each arg group (1,3,5,7).
assert(single::setup_call == 4);
assert(single::teardown_call == 4);
// 3 group of threads calling this function (3,5,10).
assert(concurrent::setup_call.load(std::memory_order_relaxed) == 3);
assert(concurrent::teardown_call.load(std::memory_order_relaxed) == 3);
assert((5 + 10 + 15) ==
concurrent::func_call.load(std::memory_order_relaxed));
// Setup is called 4 times, once for each arg group (1,3,5,7)
assert(fixture_interaction::setup == 4);
// Fixture::Setup is called everytime the bm routine is run.
// The exact number is indeterministic, so we just assert that
// it's more than setup.
assert(fixture_interaction::fixture_setup > fixture_interaction::setup);
// Setup is call once for each repetition * num_arg = 4 * 4 = 16.
assert(repetitions::setup == 16);
return 0;
}

third-party/benchmark/test/benchmark_test.cc

Show First 20 Lines • Show All 87 Lines • ▼ Show 20 Lines for (auto _ : state) {
data = ConstructRandomSet(state.range(0)); data = ConstructRandomSet(state.range(0));
state.ResumeTiming(); state.ResumeTiming();
for (int j = 0; j < state.range(1); ++j) data.insert(rand()); for (int j = 0; j < state.range(1); ++j) data.insert(rand());
} }
state.SetItemsProcessed(state.iterations() * state.range(1)); state.SetItemsProcessed(state.iterations() * state.range(1));
state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int)); state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int));
} }
// Test many inserts at once to reduce the total iterations needed. Otherwise, the slower, // Test many inserts at once to reduce the total iterations needed. Otherwise,
// non-timed part of each iteration will make the benchmark take forever. // the slower, non-timed part of each iteration will make the benchmark take
// forever.
BENCHMARK(BM_SetInsert)->Ranges({{1 << 10, 8 << 10}, {128, 512}}); BENCHMARK(BM_SetInsert)->Ranges({{1 << 10, 8 << 10}, {128, 512}});
template <typename Container, template <typename Container,
typename ValueType = typename Container::value_type> typename ValueType = typename Container::value_type>
static void BM_Sequential(benchmark::State& state) { static void BM_Sequential(benchmark::State& state) {
ValueType v = 42; ValueType v = 42;
for (auto _ : state) { for (auto _ : state) {
Container c; Container c;
Show All 15 Linesstatic void BM_StringCompare(benchmark::State& state) {
size_t len = static_cast<size_t>(state.range(0)); size_t len = static_cast<size_t>(state.range(0));
std::string s1(len, '-'); std::string s1(len, '-');
std::string s2(len, '-'); std::string s2(len, '-');
for (auto _ : state) benchmark::DoNotOptimize(s1.compare(s2)); for (auto _ : state) benchmark::DoNotOptimize(s1.compare(s2));
} }
BENCHMARK(BM_StringCompare)->Range(1, 1 << 20); BENCHMARK(BM_StringCompare)->Range(1, 1 << 20);
static void BM_SetupTeardown(benchmark::State& state) { static void BM_SetupTeardown(benchmark::State& state) {
if (state.thread_index == 0) { if (state.thread_index() == 0) {
// No need to lock test_vector_mu here as this is running single-threaded. // No need to lock test_vector_mu here as this is running single-threaded.
test_vector = new std::vector<int>(); test_vector = new std::vector<int>();
} }
int i = 0; int i = 0;
for (auto _ : state) { for (auto _ : state) {
std::lock_guard<std::mutex> l(test_vector_mu); std::lock_guard<std::mutex> l(test_vector_mu);
if (i % 2 == 0) if (i % 2 == 0)
test_vector->push_back(i); test_vector->push_back(i);
else else
test_vector->pop_back(); test_vector->pop_back();
++i; ++i;
} }
if (state.thread_index == 0) { if (state.thread_index() == 0) {
delete test_vector; delete test_vector;
} }
} }
BENCHMARK(BM_SetupTeardown)->ThreadPerCpu(); BENCHMARK(BM_SetupTeardown)->ThreadPerCpu();
static void BM_LongTest(benchmark::State& state) { static void BM_LongTest(benchmark::State& state) {
double tracker = 0.0; double tracker = 0.0;
for (auto _ : state) { for (auto _ : state) {
for (int i = 0; i < state.range(0); ++i) for (int i = 0; i < state.range(0); ++i)
benchmark::DoNotOptimize(tracker += i); benchmark::DoNotOptimize(tracker += i);
} }
} }
BENCHMARK(BM_LongTest)->Range(1 << 16, 1 << 28); BENCHMARK(BM_LongTest)->Range(1 << 16, 1 << 28);
static void BM_ParallelMemset(benchmark::State& state) { static void BM_ParallelMemset(benchmark::State& state) {
int64_t size = state.range(0) / static_cast<int64_t>(sizeof(int)); int64_t size = state.range(0) / static_cast<int64_t>(sizeof(int));
int thread_size = static_cast<int>(size) / state.threads; int thread_size = static_cast<int>(size) / state.threads();
int from = thread_size * state.thread_index; int from = thread_size * state.thread_index();
int to = from + thread_size; int to = from + thread_size;
if (state.thread_index == 0) { if (state.thread_index() == 0) {
test_vector = new std::vector<int>(static_cast<size_t>(size)); test_vector = new std::vector<int>(static_cast<size_t>(size));
} }
for (auto _ : state) { for (auto _ : state) {
for (int i = from; i < to; i++) { for (int i = from; i < to; i++) {
// No need to lock test_vector_mu as ranges // No need to lock test_vector_mu as ranges
// do not overlap between threads. // do not overlap between threads.
benchmark::DoNotOptimize(test_vector->at(i) = 1); benchmark::DoNotOptimize(test_vector->at(i) = 1);
} }
} }
if (state.thread_index == 0) { if (state.thread_index() == 0) {
delete test_vector; delete test_vector;
} }
} }
BENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4); BENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4);
static void BM_ManualTiming(benchmark::State& state) { static void BM_ManualTiming(benchmark::State& state) {
int64_t slept_for = 0; int64_t slept_for = 0;
int64_t microseconds = state.range(0); int64_t microseconds = state.range(0);
Show All 25 Linesvoid BM_with_args(benchmark::State& state, Args&&...) {
for (auto _ : state) { for (auto _ : state) {
} }
} }
BENCHMARK_CAPTURE(BM_with_args, int_test, 42, 43, 44); BENCHMARK_CAPTURE(BM_with_args, int_test, 42, 43, 44);
BENCHMARK_CAPTURE(BM_with_args, string_and_pair_test, std::string("abc"), BENCHMARK_CAPTURE(BM_with_args, string_and_pair_test, std::string("abc"),
std::pair<int, double>(42, 3.8)); std::pair<int, double>(42, 3.8));
void BM_non_template_args(benchmark::State& state, int, double) { void BM_non_template_args(benchmark::State& state, int, double) {
while(state.KeepRunning()) {} while (state.KeepRunning()) {
}
} }
BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0);
#endif // BENCHMARK_HAS_CXX11 #endif // BENCHMARK_HAS_CXX11
static void BM_DenseThreadRanges(benchmark::State& st) { static void BM_DenseThreadRanges(benchmark::State& st) {
switch (st.range(0)) { switch (st.range(0)) {
case 1: case 1:
assert(st.threads == 1 || st.threads == 2 || st.threads == 3); assert(st.threads() == 1 || st.threads() == 2 || st.threads() == 3);
break; break;
case 2: case 2:
assert(st.threads == 1 || st.threads == 3 || st.threads == 4); assert(st.threads() == 1 || st.threads() == 3 || st.threads() == 4);
break; break;
case 3: case 3:
assert(st.threads == 5 || st.threads == 8 || st.threads == 11 || assert(st.threads() == 5 || st.threads() == 8 || st.threads() == 11 ||
st.threads == 14); st.threads() == 14);
break; break;
default: default:
assert(false && "Invalid test case number"); assert(false && "Invalid test case number");
} }
while (st.KeepRunning()) { while (st.KeepRunning()) {
} }
} }
BENCHMARK(BM_DenseThreadRanges)->Arg(1)->DenseThreadRange(1, 3); BENCHMARK(BM_DenseThreadRanges)->Arg(1)->DenseThreadRange(1, 3);
BENCHMARK(BM_DenseThreadRanges)->Arg(2)->DenseThreadRange(1, 4, 2); BENCHMARK(BM_DenseThreadRanges)->Arg(2)->DenseThreadRange(1, 4, 2);
BENCHMARK(BM_DenseThreadRanges)->Arg(3)->DenseThreadRange(5, 14, 3); BENCHMARK(BM_DenseThreadRanges)->Arg(3)->DenseThreadRange(5, 14, 3);
BENCHMARK_MAIN(); BENCHMARK_MAIN();

third-party/benchmark/test/clobber_memory_assembly_test.cc

#include <benchmark/benchmark.h> #include <benchmark/benchmark.h>
#ifdef __clang__ #ifdef __clang__
#pragma clang diagnostic ignored "-Wreturn-type" #pragma clang diagnostic ignored "-Wreturn-type"
#endif #endif
extern "C" { extern "C" {
extern int ExternInt; extern int ExternInt;
extern int ExternInt2; extern int ExternInt2;
extern int ExternInt3; extern int ExternInt3;
} }
// CHECK-LABEL: test_basic: // CHECK-LABEL: test_basic:
extern "C" void test_basic() { extern "C" void test_basic() {
int x; int x;
benchmark::DoNotOptimize(&x); benchmark::DoNotOptimize(&x);
x = 101; x = 101;
benchmark::ClobberMemory(); benchmark::ClobberMemory();
▲ Show 20 Lines • Show All 44 Lines • Show Last 20 Lines

third-party/benchmark/test/complexity_test.cc

#undef NDEBUG #undef NDEBUG
#include <algorithm> #include <algorithm>
#include <cassert> #include <cassert>
#include <cmath> #include <cmath>
#include <cstdlib> #include <cstdlib>
#include <vector> #include <vector>
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "output_test.h" #include "output_test.h"
namespace { namespace {
#define ADD_COMPLEXITY_CASES(...) \ #define ADD_COMPLEXITY_CASES(...) \
int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__)
int AddComplexityTest(std::string test_name, std::string big_o_test_name, int AddComplexityTest(const std::string &test_name,
std::string rms_test_name, std::string big_o, const std::string &big_o_test_name,
int family_index) { const std::string &rms_test_name,
const std::string &big_o, int family_index) {
SetSubstitutions({{"%name", test_name}, SetSubstitutions({{"%name", test_name},
{"%bigo_name", big_o_test_name}, {"%bigo_name", big_o_test_name},
{"%rms_name", rms_test_name}, {"%rms_name", rms_test_name},
{"%bigo_str", "[ ]* %float " + big_o}, {"%bigo_str", "[ ]* %float " + big_o},
{"%bigo", big_o}, {"%bigo", big_o},
{"%rms", "[ ]*[0-9]+ %"}}); {"%rms", "[ ]*[0-9]+ %"}});
AddCases( AddCases(
TC_ConsoleOut, TC_ConsoleOut,
{{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"},
{"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name.
{"^%rms_name %rms %rms[ ]*$", MR_Next}}); {"^%rms_name %rms %rms[ ]*$", MR_Next}});
AddCases( AddCases(
TC_JSONOut, TC_JSONOut,
{{"\"name\": \"%bigo_name\",$"}, {{"\"name\": \"%bigo_name\",$"},
{"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"%name\",$", MR_Next}, {"\"run_name\": \"%name\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": %int,$", MR_Next}, {"\"repetitions\": %int,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"BigO\",$", MR_Next}, {"\"aggregate_name\": \"BigO\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"cpu_coefficient\": %float,$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next},
{"\"real_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next},
{"\"big_o\": \"%bigo\",$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}, {"}", MR_Next},
{"\"name\": \"%rms_name\",$"}, {"\"name\": \"%rms_name\",$"},
{"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"%name\",$", MR_Next}, {"\"run_name\": \"%name\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": %int,$", MR_Next}, {"\"repetitions\": %int,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"RMS\",$", MR_Next}, {"\"aggregate_name\": \"RMS\",$", MR_Next},
{"\"aggregate_unit\": \"percentage\",$", MR_Next},
{"\"rms\": %float$", MR_Next}, {"\"rms\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},
{"^\"%bigo_name\"", MR_Not}, {"^\"%bigo_name\"", MR_Not},
{"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}});
return 0; return 0;
} }
} // end namespace } // end namespace
// ========================================================================= // // ========================================================================= //
// --------------------------- Testing BigO O(1) --------------------------- // // --------------------------- Testing BigO O(1) --------------------------- //
// ========================================================================= // // ========================================================================= //
void BM_Complexity_O1(benchmark::State& state) { void BM_Complexity_O1(benchmark::State &state) {
for (auto _ : state) { for (auto _ : state) {
for (int i = 0; i < 1024; ++i) { for (int i = 0; i < 1024; ++i) {
benchmark::DoNotOptimize(&i); benchmark::DoNotOptimize(&i);
} }
} }
state.SetComplexityN(state.range(0)); state.SetComplexityN(state.range(0));
} }
BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
Show All 32 Linesstd::vector<int> ConstructRandomVector(int64_t size) {
std::vector<int> v; std::vector<int> v;
v.reserve(static_cast<int>(size)); v.reserve(static_cast<int>(size));
for (int i = 0; i < size; ++i) { for (int i = 0; i < size; ++i) {
v.push_back(static_cast<int>(std::rand() % size)); v.push_back(static_cast<int>(std::rand() % size));
} }
return v; return v;
} }
void BM_Complexity_O_N(benchmark::State& state) { void BM_Complexity_O_N(benchmark::State &state) {
auto v = ConstructRandomVector(state.range(0)); auto v = ConstructRandomVector(state.range(0));
// Test worst case scenario (item not in vector) // Test worst case scenario (item not in vector)
const int64_t item_not_in_vector = state.range(0) * 2; const int64_t item_not_in_vector = state.range(0) * 2;
for (auto _ : state) { for (auto _ : state) {
benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector));
} }
state.SetComplexityN(state.range(0)); state.SetComplexityN(state.range(0));
} }
Show All 25 Lines
// Add lambda tests // Add lambda tests
ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
lambda_big_o_n, /*family_index=*/4); lambda_big_o_n, /*family_index=*/4);
// ========================================================================= // // ========================================================================= //
// ------------------------- Testing BigO O(N*lgN) ------------------------- // // ------------------------- Testing BigO O(N*lgN) ------------------------- //
// ========================================================================= // // ========================================================================= //
static void BM_Complexity_O_N_log_N(benchmark::State& state) { static void BM_Complexity_O_N_log_N(benchmark::State &state) {
auto v = ConstructRandomVector(state.range(0)); auto v = ConstructRandomVector(state.range(0));
for (auto _ : state) { for (auto _ : state) {
std::sort(v.begin(), v.end()); std::sort(v.begin(), v.end());
} }
state.SetComplexityN(state.range(0)); state.SetComplexityN(state.range(0));
} }
static const double kLog2E = 1.44269504088896340736; static const double kLog2E = 1.44269504088896340736;
BENCHMARK(BM_Complexity_O_N_log_N) BENCHMARK(BM_Complexity_O_N_log_N)
Show All 26 Lines
ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n,
/*family_index=*/7); /*family_index=*/7);
// ========================================================================= // // ========================================================================= //
// -------- Testing formatting of Complexity with captured args ------------ // // -------- Testing formatting of Complexity with captured args ------------ //
// ========================================================================= // // ========================================================================= //
void BM_ComplexityCaptureArgs(benchmark::State& state, int n) { void BM_ComplexityCaptureArgs(benchmark::State &state, int n) {
for (auto _ : state) { for (auto _ : state) {
// This test requires a non-zero CPU time to avoid divide-by-zero // This test requires a non-zero CPU time to avoid divide-by-zero
benchmark::DoNotOptimize(state.iterations()); benchmark::DoNotOptimize(state.iterations());
} }
state.SetComplexityN(n); state.SetComplexityN(n);
} }
BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100) BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100)
Show All 14 Lines

third-party/benchmark/test/cxx03_test.cc

Show All 38 Lines
template <class T> template <class T>
void BM_template1(benchmark::State& state) { void BM_template1(benchmark::State& state) {
BM_empty(state); BM_empty(state);
} }
BENCHMARK_TEMPLATE(BM_template1, long); BENCHMARK_TEMPLATE(BM_template1, long);
BENCHMARK_TEMPLATE1(BM_template1, int); BENCHMARK_TEMPLATE1(BM_template1, int);
template <class T> template <class T>
struct BM_Fixture : public ::benchmark::Fixture { struct BM_Fixture : public ::benchmark::Fixture {};
};
BENCHMARK_TEMPLATE_F(BM_Fixture, BM_template1, long)(benchmark::State& state) { BENCHMARK_TEMPLATE_F(BM_Fixture, BM_template1, long)(benchmark::State& state) {
BM_empty(state); BM_empty(state);
} }
BENCHMARK_TEMPLATE1_F(BM_Fixture, BM_template2, int)(benchmark::State& state) { BENCHMARK_TEMPLATE1_F(BM_Fixture, BM_template2, int)(benchmark::State& state) {
BM_empty(state); BM_empty(state);
} }
void BM_counters(benchmark::State& state) { void BM_counters(benchmark::State& state) {
BM_empty(state); BM_empty(state);
state.counters["Foo"] = 2; state.counters["Foo"] = 2;
} }
BENCHMARK(BM_counters); BENCHMARK(BM_counters);
BENCHMARK_MAIN(); BENCHMARK_MAIN();

third-party/benchmark/test/diagnostics_test.cc

Show All 20 Lines
#ifndef TEST_HAS_NO_EXCEPTIONS #ifndef TEST_HAS_NO_EXCEPTIONS
throw std::logic_error(""); throw std::logic_error("");
#else #else
std::abort(); std::abort();
#endif #endif
} }
void try_invalid_pause_resume(benchmark::State& state) { void try_invalid_pause_resume(benchmark::State& state) {
#if !defined(TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS) && !defined(TEST_HAS_NO_EXCEPTIONS) #if !defined(TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS) && \
!defined(TEST_HAS_NO_EXCEPTIONS)
try { try {
state.PauseTiming(); state.PauseTiming();
std::abort(); std::abort();
} catch (std::logic_error const&) { } catch (std::logic_error const&) {
} }
try { try {
state.ResumeTiming(); state.ResumeTiming();
std::abort(); std::abort();
Show All 14 Linesvoid BM_diagnostic_test(benchmark::State& state) {
} }
if (called_once == false) try_invalid_pause_resume(state); if (called_once == false) try_invalid_pause_resume(state);
called_once = true; called_once = true;
} }
BENCHMARK(BM_diagnostic_test); BENCHMARK(BM_diagnostic_test);
void BM_diagnostic_test_keep_running(benchmark::State& state) { void BM_diagnostic_test_keep_running(benchmark::State& state) {
static bool called_once = false; static bool called_once = false;
if (called_once == false) try_invalid_pause_resume(state); if (called_once == false) try_invalid_pause_resume(state);
while(state.KeepRunning()) { while (state.KeepRunning()) {
benchmark::DoNotOptimize(state.iterations()); benchmark::DoNotOptimize(state.iterations());
} }
if (called_once == false) try_invalid_pause_resume(state); if (called_once == false) try_invalid_pause_resume(state);
called_once = true; called_once = true;
} }
BENCHMARK(BM_diagnostic_test_keep_running); BENCHMARK(BM_diagnostic_test_keep_running);
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
benchmark::internal::GetAbortHandler() = &TestHandler; benchmark::internal::GetAbortHandler() = &TestHandler;
benchmark::Initialize(&argc, argv); benchmark::Initialize(&argc, argv);
benchmark::RunSpecifiedBenchmarks(); benchmark::RunSpecifiedBenchmarks();
} }

third-party/benchmark/test/display_aggregates_only_test.cc

Show All 13 Linesvoid BM_SummaryRepeat(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
} }
} }
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->DisplayAggregatesOnly(); BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->DisplayAggregatesOnly();
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
const std::string output = GetFileReporterOutput(argc, argv); const std::string output = GetFileReporterOutput(argc, argv);
if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 6 || if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 7 ||
SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3\"") != 3 || SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3\"") != 3 ||
SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 || SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 ||
SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") != SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") !=
1 || 1 ||
SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") != SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") !=
1) { 1 ||
std::cout << "Precondition mismatch. Expected to only find 6 " SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_cv\"") != 1) {
std::cout << "Precondition mismatch. Expected to only find 8 "
"occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n" "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n"
"\"name\": \"BM_SummaryRepeat/repeats:3\", " "\"name\": \"BM_SummaryRepeat/repeats:3\", "
"\"name\": \"BM_SummaryRepeat/repeats:3\", " "\"name\": \"BM_SummaryRepeat/repeats:3\", "
"\"name\": \"BM_SummaryRepeat/repeats:3\", " "\"name\": \"BM_SummaryRepeat/repeats:3\", "
"\"name\": \"BM_SummaryRepeat/repeats:3_mean\", " "\"name\": \"BM_SummaryRepeat/repeats:3_mean\", "
"\"name\": \"BM_SummaryRepeat/repeats:3_median\", " "\"name\": \"BM_SummaryRepeat/repeats:3_median\", "
"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire " "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\", "
"\"name\": \"BM_SummaryRepeat/repeats:3_cv\"\nThe entire "
"output:\n"; "output:\n";
std::cout << output; std::cout << output;
return 1; return 1;
} }
return 0; return 0;
} }

third-party/benchmark/test/donotoptimize_assembly_test.cc

Show All 9 Lines
extern int ExternInt2; extern int ExternInt2;
extern int ExternInt3; extern int ExternInt3;
inline int Add42(int x) { return x + 42; } inline int Add42(int x) { return x + 42; }
struct NotTriviallyCopyable { struct NotTriviallyCopyable {
NotTriviallyCopyable(); NotTriviallyCopyable();
explicit NotTriviallyCopyable(int x) : value(x) {} explicit NotTriviallyCopyable(int x) : value(x) {}
NotTriviallyCopyable(NotTriviallyCopyable const&); NotTriviallyCopyable(NotTriviallyCopyable const &);
int value; int value;
}; };
struct Large { struct Large {
int value; int value;
int data[2]; int data[2];
}; };
} }
// CHECK-LABEL: test_with_rvalue: // CHECK-LABEL: test_with_rvalue:
extern "C" void test_with_rvalue() { extern "C" void test_with_rvalue() {
benchmark::DoNotOptimize(Add42(0)); benchmark::DoNotOptimize(Add42(0));
// CHECK: movl $42, %eax // CHECK: movl $42, %eax
// CHECK: ret // CHECK: ret
} }
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Linesextern "C" int test_div_by_two(int input) {
// CHECK: movl $2, [[DEST:.*]] // CHECK: movl $2, [[DEST:.*]]
// CHECK: idivl [[DEST]] // CHECK: idivl [[DEST]]
// CHECK: ret // CHECK: ret
} }
// CHECK-LABEL: test_inc_integer: // CHECK-LABEL: test_inc_integer:
extern "C" int test_inc_integer() { extern "C" int test_inc_integer() {
int x = 0; int x = 0;
for (int i=0; i < 5; ++i) for (int i = 0; i < 5; ++i) benchmark::DoNotOptimize(++x);
benchmark::DoNotOptimize(++x);
// CHECK: movl $1, [[DEST:.*]] // CHECK: movl $1, [[DEST:.*]]
// CHECK: {{(addl \$1,|incl)}} [[DEST]] // CHECK: {{(addl \$1,|incl)}} [[DEST]]
// CHECK: {{(addl \$1,|incl)}} [[DEST]] // CHECK: {{(addl \$1,|incl)}} [[DEST]]
// CHECK: {{(addl \$1,|incl)}} [[DEST]] // CHECK: {{(addl \$1,|incl)}} [[DEST]]
// CHECK: {{(addl \$1,|incl)}} [[DEST]] // CHECK: {{(addl \$1,|incl)}} [[DEST]]
// CHECK-CLANG: movl [[DEST]], %eax // CHECK-CLANG: movl [[DEST]], %eax
// CHECK: ret // CHECK: ret
return x; return x;
Show All 11 Lines
// CHECK-LABEL: test_pointer_const_lvalue: // CHECK-LABEL: test_pointer_const_lvalue:
extern "C" void test_pointer_const_lvalue() { extern "C" void test_pointer_const_lvalue() {
// CHECK: movl $42, [[DEST:.*]] // CHECK: movl $42, [[DEST:.*]]
// CHECK: leaq [[DEST]], %rax // CHECK: leaq [[DEST]], %rax
// CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z]+]]) // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z]+]])
// CHECK: ret // CHECK: ret
int x = 42; int x = 42;
int * const xp = &x; int *const xp = &x;
benchmark::DoNotOptimize(xp); benchmark::DoNotOptimize(xp);
} }
// CHECK-LABEL: test_pointer_lvalue: // CHECK-LABEL: test_pointer_lvalue:
extern "C" void test_pointer_lvalue() { extern "C" void test_pointer_lvalue() {
// CHECK: movl $42, [[DEST:.*]] // CHECK: movl $42, [[DEST:.*]]
// CHECK: leaq [[DEST]], %rax // CHECK: leaq [[DEST]], %rax
// CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z+]+]]) // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z+]+]])
// CHECK: ret // CHECK: ret
int x = 42; int x = 42;
int *xp = &x; int *xp = &x;
benchmark::DoNotOptimize(xp); benchmark::DoNotOptimize(xp);
} }

third-party/benchmark/test/donotoptimize_test.cc

#include "benchmark/benchmark.h"
#include <cstdint> #include <cstdint>
#include "benchmark/benchmark.h"
namespace { namespace {
#if defined(__GNUC__) #if defined(__GNUC__)
std::uint64_t double_up(const std::uint64_t x) __attribute__((const)); std::uint64_t double_up(const std::uint64_t x) __attribute__((const));
#endif #endif
std::uint64_t double_up(const std::uint64_t x) { return x * 2; } std::uint64_t double_up(const std::uint64_t x) { return x * 2; }
} } // namespace
// Using DoNotOptimize on types like BitRef seem to cause a lot of problems // Using DoNotOptimize on types like BitRef seem to cause a lot of problems
// with the inline assembly on both GCC and Clang. // with the inline assembly on both GCC and Clang.
struct BitRef { struct BitRef {
int index; int index;
unsigned char &byte; unsigned char& byte;
public: public:
static BitRef Make() { static BitRef Make() {
static unsigned char arr[2] = {}; static unsigned char arr[2] = {};
BitRef b(1, arr[0]); BitRef b(1, arr[0]);
return b; return b;
} }
private: private:
BitRef(int i, unsigned char& b) : index(i), byte(b) {} BitRef(int i, unsigned char& b) : index(i), byte(b) {}
}; };
int main(int, char*[]) { int main(int, char*[]) {
// this test verifies compilation of DoNotOptimize() for some types // this test verifies compilation of DoNotOptimize() for some types
char buffer8[8] = ""; char buffer8[8] = "";
benchmark::DoNotOptimize(buffer8); benchmark::DoNotOptimize(buffer8);
Show All 20 Lines

third-party/benchmark/test/filter_test.cc

Show First 20 Lines • Show All 64 Lines • ▼ Show 20 Lines
BENCHMARK(BM_FooBar); BENCHMARK(BM_FooBar);
static void BM_FooBa(benchmark::State& state) { static void BM_FooBa(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
} }
} }
BENCHMARK(BM_FooBa); BENCHMARK(BM_FooBa);
int main(int argc, char **argv) { int main(int argc, char** argv) {
bool list_only = false; bool list_only = false;
for (int i = 0; i < argc; ++i) for (int i = 0; i < argc; ++i)
list_only |= std::string(argv[i]).find("--benchmark_list_tests") != list_only |= std::string(argv[i]).find("--benchmark_list_tests") !=
std::string::npos; std::string::npos;
benchmark::Initialize(&argc, argv); benchmark::Initialize(&argc, argv);
TestReporter test_reporter; TestReporter test_reporter;
Show All 37 Lines

third-party/benchmark/test/fixture_test.cc

#include "benchmark/benchmark.h"
#include <cassert> #include <cassert>
#include <memory> #include <memory>
#include "benchmark/benchmark.h"
#define FIXTURE_BECHMARK_NAME MyFixture #define FIXTURE_BECHMARK_NAME MyFixture
class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture {
public: public:
void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE {
if (state.thread_index == 0) { if (state.thread_index() == 0) {
assert(data.get() == nullptr); assert(data.get() == nullptr);
data.reset(new int(42)); data.reset(new int(42));
} }
} }
void TearDown(const ::benchmark::State& state) BENCHMARK_OVERRIDE { void TearDown(const ::benchmark::State& state) BENCHMARK_OVERRIDE {
if (state.thread_index == 0) { if (state.thread_index() == 0) {
assert(data.get() != nullptr); assert(data.get() != nullptr);
data.reset(); data.reset();
} }
} }
~FIXTURE_BECHMARK_NAME() { assert(data == nullptr); } ~FIXTURE_BECHMARK_NAME() { assert(data == nullptr); }
std::unique_ptr<int> data; std::unique_ptr<int> data;
}; };
BENCHMARK_F(FIXTURE_BECHMARK_NAME, Foo)(benchmark::State &st) { BENCHMARK_F(FIXTURE_BECHMARK_NAME, Foo)(benchmark::State& st) {
assert(data.get() != nullptr); assert(data.get() != nullptr);
assert(*data == 42); assert(*data == 42);
for (auto _ : st) { for (auto _ : st) {
} }
} }
BENCHMARK_DEFINE_F(FIXTURE_BECHMARK_NAME, Bar)(benchmark::State& st) { BENCHMARK_DEFINE_F(FIXTURE_BECHMARK_NAME, Bar)(benchmark::State& st) {
if (st.thread_index == 0) { if (st.thread_index() == 0) {
assert(data.get() != nullptr); assert(data.get() != nullptr);
assert(*data == 42); assert(*data == 42);
} }
for (auto _ : st) { for (auto _ : st) {
assert(data.get() != nullptr); assert(data.get() != nullptr);
assert(*data == 42); assert(*data == 42);
} }
st.SetItemsProcessed(st.range(0)); st.SetItemsProcessed(st.range(0));
} }
BENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, Bar)->Arg(42); BENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, Bar)->Arg(42);
BENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, Bar)->Arg(42)->ThreadPerCpu(); BENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, Bar)->Arg(42)->ThreadPerCpu();
BENCHMARK_MAIN(); BENCHMARK_MAIN();

third-party/benchmark/test/internal_threading_test.cc

#undef NDEBUG #undef NDEBUG
#include <chrono> #include <chrono>
#include <thread> #include <thread>
#include "../src/timers.h" #include "../src/timers.h"
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "output_test.h" #include "output_test.h"
static const std::chrono::duration<double, std::milli> time_frame(50); static const std::chrono::duration<double, std::milli> time_frame(50);
static const double time_frame_in_sec( static const double time_frame_in_sec(
std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1, 1>>>( std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1, 1>>>(
time_frame) time_frame)
▲ Show 20 Lines • Show All 171 Lines • Show Last 20 Lines

third-party/benchmark/test/map_test.cc

#include "benchmark/benchmark.h"
#include <cstdlib> #include <cstdlib>
#include <map> #include <map>
#include "benchmark/benchmark.h"
namespace { namespace {
std::map<int, int> ConstructRandomMap(int size) { std::map<int, int> ConstructRandomMap(int size) {
std::map<int, int> m; std::map<int, int> m;
for (int i = 0; i < size; ++i) { for (int i = 0; i < size; ++i) {
m.insert(std::make_pair(std::rand() % size, std::rand() % size)); m.insert(std::make_pair(std::rand() % size, std::rand() % size));
} }
return m; return m;
▲ Show 20 Lines • Show All 44 Lines • Show Last 20 Lines

third-party/benchmark/test/multiple_ranges_test.cc

#include "benchmark/benchmark.h"
#include <cassert> #include <cassert>
#include <iostream> #include <iostream>
#include <set> #include <set>
#include <vector> #include <vector>
#include "benchmark/benchmark.h"
class MultipleRangesFixture : public ::benchmark::Fixture { class MultipleRangesFixture : public ::benchmark::Fixture {
public: public:
MultipleRangesFixture() MultipleRangesFixture()
: expectedValues({{1, 3, 5}, : expectedValues({{1, 3, 5},
{1, 3, 8}, {1, 3, 8},
{1, 3, 15}, {1, 3, 15},
{2, 3, 5}, {2, 3, 5},
{2, 3, 8}, {2, 3, 8},
Show All 21 Lines void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE {
actualValues.insert(ranges); actualValues.insert(ranges);
} }
// NOTE: This is not TearDown as we want to check after _all_ runs are // NOTE: This is not TearDown as we want to check after _all_ runs are
// complete. // complete.
virtual ~MultipleRangesFixture() { virtual ~MultipleRangesFixture() {
if (actualValues != expectedValues) { if (actualValues != expectedValues) {
std::cout << "EXPECTED\n"; std::cout << "EXPECTED\n";
for (auto v : expectedValues) { for (const auto& v : expectedValues) {
std::cout << "{"; std::cout << "{";
for (int64_t iv : v) { for (int64_t iv : v) {
std::cout << iv << ", "; std::cout << iv << ", ";
} }
std::cout << "}\n"; std::cout << "}\n";
} }
std::cout << "ACTUAL\n"; std::cout << "ACTUAL\n";
for (auto v : actualValues) { for (const auto& v : actualValues) {
std::cout << "{"; std::cout << "{";
for (int64_t iv : v) { for (int64_t iv : v) {
std::cout << iv << ", "; std::cout << iv << ", ";
} }
std::cout << "}\n"; std::cout << "}\n";
} }
} }
} }
Show All 35 Lines

third-party/benchmark/test/options_test.cc

#include "benchmark/benchmark.h"
#include <chrono> #include <chrono>
#include <thread> #include <thread>
#include "benchmark/benchmark.h"
#if defined(NDEBUG) #if defined(NDEBUG)
#undef NDEBUG #undef NDEBUG
#endif #endif
#include <cassert> #include <cassert>
void BM_basic(benchmark::State& state) { void BM_basic(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
} }
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Linesvoid BM_explicit_iteration_count(benchmark::State& state) {
// only run once. // only run once.
static bool invoked_before = false; static bool invoked_before = false;
assert(!invoked_before); assert(!invoked_before);
invoked_before = true; invoked_before = true;
// Test that the requested iteration count is respected. // Test that the requested iteration count is respected.
assert(state.max_iterations == 42); assert(state.max_iterations == 42);
size_t actual_iterations = 0; size_t actual_iterations = 0;
for (auto _ : state) for (auto _ : state) ++actual_iterations;
++actual_iterations;
assert(state.iterations() == state.max_iterations); assert(state.iterations() == state.max_iterations);
assert(state.iterations() == 42); assert(state.iterations() == 42);
} }
BENCHMARK(BM_explicit_iteration_count)->Iterations(42); BENCHMARK(BM_explicit_iteration_count)->Iterations(42);
BENCHMARK_MAIN(); BENCHMARK_MAIN();

third-party/benchmark/test/output_test.h

Show First 20 Lines • Show All 79 Lines • ▼ Show 20 Lines
// will be the subject of a call to checker_function // will be the subject of a call to checker_function
// checker_function: should be of type ResultsCheckFn (see below) // checker_function: should be of type ResultsCheckFn (see below)
#define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \ #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \
size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function) size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function)
struct Results; struct Results;
typedef std::function<void(Results const&)> ResultsCheckFn; typedef std::function<void(Results const&)> ResultsCheckFn;
size_t AddChecker(const char* bm_name_pattern, ResultsCheckFn fn); size_t AddChecker(const char* bm_name_pattern, const ResultsCheckFn& fn);
// Class holding the results of a benchmark. // Class holding the results of a benchmark.
// It is passed in calls to checker functions. // It is passed in calls to checker functions.
struct Results { struct Results {
// the benchmark name // the benchmark name
std::string name; std::string name;
// the benchmark fields // the benchmark fields
std::map<std::string, std::string> values; std::map<std::string, std::string> values;
Show All 11 Linesstruct Results {
// get the real_time duration of the benchmark in seconds. // get the real_time duration of the benchmark in seconds.
// it is better to use fuzzy float checks for this, as the float // it is better to use fuzzy float checks for this, as the float
// ASCII formatting is lossy. // ASCII formatting is lossy.
double DurationRealTime() const { double DurationRealTime() const {
return NumIterations() * GetTime(kRealTime); return NumIterations() * GetTime(kRealTime);
} }
// get the cpu_time duration of the benchmark in seconds // get the cpu_time duration of the benchmark in seconds
double DurationCPUTime() const { double DurationCPUTime() const { return NumIterations() * GetTime(kCpuTime); }
return NumIterations() * GetTime(kCpuTime);
}
// get the string for a result by name, or nullptr if the name // get the string for a result by name, or nullptr if the name
// is not found // is not found
const std::string* Get(const char* entry_name) const { const std::string* Get(const char* entry_name) const {
auto it = values.find(entry_name); auto it = values.find(entry_name);
if (it == values.end()) return nullptr; if (it == values.end()) return nullptr;
return &it->second; return &it->second;
} }
Show All 11 Lines T GetCounterAs(const char* entry_name) const {
T tval = static_cast<T>(dval); T tval = static_cast<T>(dval);
return tval; return tval;
} }
}; };
template <class T> template <class T>
T Results::GetAs(const char* entry_name) const { T Results::GetAs(const char* entry_name) const {
auto* sv = Get(entry_name); auto* sv = Get(entry_name);
CHECK(sv != nullptr && !sv->empty()); BM_CHECK(sv != nullptr && !sv->empty());
std::stringstream ss; std::stringstream ss;
ss << *sv; ss << *sv;
T out; T out;
ss >> out; ss >> out;
CHECK(!ss.fail()); BM_CHECK(!ss.fail());
return out; return out;
} }
//---------------------------------- //----------------------------------
// Macros to help in result checking. Do not use them with arguments causing // Macros to help in result checking. Do not use them with arguments causing
// side-effects. // side-effects.
// clang-format off // clang-format off
#define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \ #define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \
CONCAT(CHECK_, relationship) \ CONCAT(BM_CHECK_, relationship) \
(entry.getfn< var_type >(var_name), (value)) << "\n" \ (entry.getfn< var_type >(var_name), (value)) << "\n" \
<< __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
<< __FILE__ << ":" << __LINE__ << ": " \ << __FILE__ << ":" << __LINE__ << ": " \
<< "expected (" << #var_type << ")" << (var_name) \ << "expected (" << #var_type << ")" << (var_name) \
<< "=" << (entry).getfn< var_type >(var_name) \ << "=" << (entry).getfn< var_type >(var_name) \
<< " to be " #relationship " to " << (value) << "\n" << " to be " #relationship " to " << (value) << "\n"
// check with tolerance. eps_factor is the tolerance window, which is // check with tolerance. eps_factor is the tolerance window, which is
// interpreted relative to value (eg, 0.1 means 10% of value). // interpreted relative to value (eg, 0.1 means 10% of value).
#define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ #define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \
CONCAT(CHECK_FLOAT_, relationship) \ CONCAT(BM_CHECK_FLOAT_, relationship) \
(entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \ (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \
<< __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
<< __FILE__ << ":" << __LINE__ << ": " \ << __FILE__ << ":" << __LINE__ << ": " \
<< "expected (" << #var_type << ")" << (var_name) \ << "expected (" << #var_type << ")" << (var_name) \
<< "=" << (entry).getfn< var_type >(var_name) \ << "=" << (entry).getfn< var_type >(var_name) \
<< " to be " #relationship " to " << (value) << "\n" \ << " to be " #relationship " to " << (value) << "\n" \
<< __FILE__ << ":" << __LINE__ << ": " \ << __FILE__ << ":" << __LINE__ << ": " \
<< "with tolerance of " << (eps_factor) * (value) \ << "with tolerance of " << (eps_factor) * (value) \
Show All 32 Lines

third-party/benchmark/test/output_test_helper.cc

#include <cstdio> #include <cstdio>
#include <cstring> #include <cstring>
#include <fstream> #include <fstream>
#include <iostream> #include <iostream>
#include <map> #include <map>
#include <memory> #include <memory>
#include <random> #include <random>
#include <sstream> #include <sstream>
#include <streambuf> #include <streambuf>
#include "../src/benchmark_api_internal.h" #include "../src/benchmark_api_internal.h"
#include "../src/check.h" // NOTE: check.h is for internal use only! #include "../src/check.h" // NOTE: check.h is for internal use only!
#include "../src/log.h" // NOTE: log.h is for internal use only
#include "../src/re.h" // NOTE: re.h is for internal use only #include "../src/re.h" // NOTE: re.h is for internal use only
#include "output_test.h" #include "output_test.h"
// ========================================================================= // // ========================================================================= //
// ------------------------------ Internals -------------------------------- // // ------------------------------ Internals -------------------------------- //
// ========================================================================= // // ========================================================================= //
namespace internal { namespace internal {
namespace { namespace {
Show All 14 LinesTestCaseList& GetTestCaseList(TestCaseID ID) {
return lists[ID]; return lists[ID];
} }
SubMap& GetSubstitutions() { SubMap& GetSubstitutions() {
// Don't use 'dec_re' from header because it may not yet be initialized. // Don't use 'dec_re' from header because it may not yet be initialized.
// clang-format off // clang-format off
static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
static std::string time_re = "([0-9]+[.])?[0-9]+"; static std::string time_re = "([0-9]+[.])?[0-9]+";
static std::string percentage_re = "[0-9]+[.][0-9]{2}";
static SubMap map = { static SubMap map = {
{"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
// human-readable float // human-readable float
{"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
{"%percentage", percentage_re},
{"%int", "[ ]*[0-9]+"}, {"%int", "[ ]*[0-9]+"},
{" %s ", "[ ]+"}, {" %s ", "[ ]+"},
{"%time", "[ ]*" + time_re + "[ ]+ns"}, {"%time", "[ ]*" + time_re + "[ ]+ns"},
{"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"}, {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"},
{"%console_percentage_report", "[ ]*" + percentage_re + "[ ]+% [ ]*" + percentage_re + "[ ]+% [ ]*[0-9]+"},
{"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
{"%console_ms_report", "[ ]*" + time_re + "[ ]+ms [ ]*" + time_re + "[ ]+ms [ ]*[0-9]+"}, {"%console_ms_report", "[ ]*" + time_re + "[ ]+ms [ ]*" + time_re + "[ ]+ms [ ]*[0-9]+"},
{"%console_s_report", "[ ]*" + time_re + "[ ]+s [ ]*" + time_re + "[ ]+s [ ]*[0-9]+"}, {"%console_s_report", "[ ]*" + time_re + "[ ]+s [ ]*" + time_re + "[ ]+s [ ]*[0-9]+"},
{"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"}, {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"},
{"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
{"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"}, {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"},
{"%csv_header", {"%csv_header",
"name,iterations,real_time,cpu_time,time_unit,bytes_per_second," "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
Show All 30 Lines
} }
void CheckCase(std::stringstream& remaining_output, TestCase const& TC, void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
TestCaseList const& not_checks) { TestCaseList const& not_checks) {
std::string first_line; std::string first_line;
bool on_first = true; bool on_first = true;
std::string line; std::string line;
while (remaining_output.eof() == false) { while (remaining_output.eof() == false) {
CHECK(remaining_output.good()); BM_CHECK(remaining_output.good());
std::getline(remaining_output, line); std::getline(remaining_output, line);
if (on_first) { if (on_first) {
first_line = line; first_line = line;
on_first = false; on_first = false;
} }
for (const auto& NC : not_checks) { for (const auto& NC : not_checks) {
CHECK(!NC.regex->Match(line)) BM_CHECK(!NC.regex->Match(line))
<< "Unexpected match for line \"" << line << "\" for MR_Not regex \"" << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
<< NC.regex_str << "\"" << NC.regex_str << "\""
<< "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n actual regex string \"" << TC.substituted_regex << "\""
<< "\n started matching near: " << first_line; << "\n started matching near: " << first_line;
} }
if (TC.regex->Match(line)) return; if (TC.regex->Match(line)) return;
CHECK(TC.match_rule != MR_Next) BM_CHECK(TC.match_rule != MR_Next)
<< "Expected line \"" << line << "\" to match regex \"" << TC.regex_str << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
<< "\"" << "\""
<< "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n actual regex string \"" << TC.substituted_regex << "\""
<< "\n started matching near: " << first_line; << "\n started matching near: " << first_line;
} }
CHECK(remaining_output.eof() == false) BM_CHECK(remaining_output.eof() == false)
<< "End of output reached before match for regex \"" << TC.regex_str << "End of output reached before match for regex \"" << TC.regex_str
<< "\" was found" << "\" was found"
<< "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n actual regex string \"" << TC.substituted_regex << "\""
<< "\n started matching near: " << first_line; << "\n started matching near: " << first_line;
} }
void CheckCases(TestCaseList const& checks, std::stringstream& output) { void CheckCases(TestCaseList const& checks, std::stringstream& output) {
std::vector<TestCase> not_checks; std::vector<TestCase> not_checks;
for (size_t i = 0; i < checks.size(); ++i) { for (size_t i = 0; i < checks.size(); ++i) {
const auto& TC = checks[i]; const auto& TC = checks[i];
if (TC.match_rule == MR_Not) { if (TC.match_rule == MR_Not) {
not_checks.push_back(TC); not_checks.push_back(TC);
continue; continue;
} }
CheckCase(output, TC, not_checks); CheckCase(output, TC, not_checks);
not_checks.clear(); not_checks.clear();
} }
} }
class TestReporter : public benchmark::BenchmarkReporter { class TestReporter : public benchmark::BenchmarkReporter {
public: public:
TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
: reporters_(reps) {} : reporters_(std::move(reps)) {}
virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE {
bool last_ret = false; bool last_ret = false;
bool first = true; bool first = true;
for (auto rep : reporters_) { for (auto rep : reporters_) {
bool new_ret = rep->ReportContext(context); bool new_ret = rep->ReportContext(context);
CHECK(first || new_ret == last_ret) BM_CHECK(first || new_ret == last_ret)
<< "Reports return different values for ReportContext"; << "Reports return different values for ReportContext";
first = false; first = false;
last_ret = new_ret; last_ret = new_ret;
} }
(void)first; (void)first;
return last_ret; return last_ret;
} }
Show All 18 Lines
namespace internal { namespace internal {
// Utility class to manage subscribers for checking benchmark results. // Utility class to manage subscribers for checking benchmark results.
// It works by parsing the CSV output to read the results. // It works by parsing the CSV output to read the results.
class ResultsChecker { class ResultsChecker {
public: public:
struct PatternAndFn : public TestCase { // reusing TestCase for its regexes struct PatternAndFn : public TestCase { // reusing TestCase for its regexes
PatternAndFn(const std::string& rx, ResultsCheckFn fn_) PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
: TestCase(rx), fn(fn_) {} : TestCase(rx), fn(std::move(fn_)) {}
ResultsCheckFn fn; ResultsCheckFn fn;
}; };
std::vector<PatternAndFn> check_patterns; std::vector<PatternAndFn> check_patterns;
std::vector<Results> results; std::vector<Results> results;
std::vector<std::string> field_names; std::vector<std::string> field_names;
void Add(const std::string& entry_pattern, ResultsCheckFn fn); void Add(const std::string& entry_pattern, const ResultsCheckFn& fn);
void CheckResults(std::stringstream& output); void CheckResults(std::stringstream& output);
private: private:
void SetHeader_(const std::string& csv_header); void SetHeader_(const std::string& csv_header);
void SetValues_(const std::string& entry_csv_line); void SetValues_(const std::string& entry_csv_line);
std::vector<std::string> SplitCsv_(const std::string& line); std::vector<std::string> SplitCsv_(const std::string& line);
}; };
// store the static ResultsChecker in a function to prevent initialization // store the static ResultsChecker in a function to prevent initialization
// order problems // order problems
ResultsChecker& GetResultsChecker() { ResultsChecker& GetResultsChecker() {
static ResultsChecker rc; static ResultsChecker rc;
return rc; return rc;
} }
// add a results checker for a benchmark // add a results checker for a benchmark
void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { void ResultsChecker::Add(const std::string& entry_pattern,
const ResultsCheckFn& fn) {
check_patterns.emplace_back(entry_pattern, fn); check_patterns.emplace_back(entry_pattern, fn);
} }
// check the results of all subscribed benchmarks // check the results of all subscribed benchmarks
void ResultsChecker::CheckResults(std::stringstream& output) { void ResultsChecker::CheckResults(std::stringstream& output) {
// first reset the stream to the start // first reset the stream to the start
{ {
auto start = std::stringstream::pos_type(0); auto start = std::stringstream::pos_type(0);
// clear before calling tellg() // clear before calling tellg()
output.clear(); output.clear();
// seek to zero only when needed // seek to zero only when needed
if (output.tellg() > start) output.seekg(start); if (output.tellg() > start) output.seekg(start);
// and just in case // and just in case
output.clear(); output.clear();
} }
// now go over every line and publish it to the ResultsChecker // now go over every line and publish it to the ResultsChecker
std::string line; std::string line;
bool on_first = true; bool on_first = true;
while (output.eof() == false) { while (output.eof() == false) {
CHECK(output.good()); BM_CHECK(output.good());
std::getline(output, line); std::getline(output, line);
if (on_first) { if (on_first) {
SetHeader_(line); // this is important SetHeader_(line); // this is important
on_first = false; on_first = false;
continue; continue;
} }
SetValues_(line); SetValues_(line);
} }
// finally we can call the subscribed check functions // finally we can call the subscribed check functions
for (const auto& p : check_patterns) { for (const auto& p : check_patterns) {
VLOG(2) << "--------------------------------\n"; BM_VLOG(2) << "--------------------------------\n";
VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; BM_VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
for (const auto& r : results) { for (const auto& r : results) {
if (!p.regex->Match(r.name)) { if (!p.regex->Match(r.name)) {
VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
continue; continue;
} else { } else {
VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
} }
VLOG(1) << "Checking results of " << r.name << ": ... \n"; BM_VLOG(1) << "Checking results of " << r.name << ": ... \n";
p.fn(r); p.fn(r);
VLOG(1) << "Checking results of " << r.name << ": OK.\n"; BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n";
} }
} }
} }
// prepare for the names in this header // prepare for the names in this header
void ResultsChecker::SetHeader_(const std::string& csv_header) { void ResultsChecker::SetHeader_(const std::string& csv_header) {
field_names = SplitCsv_(csv_header); field_names = SplitCsv_(csv_header);
} }
// set the values for a benchmark // set the values for a benchmark
void ResultsChecker::SetValues_(const std::string& entry_csv_line) { void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
if (entry_csv_line.empty()) return; // some lines are empty if (entry_csv_line.empty()) return; // some lines are empty
CHECK(!field_names.empty()); BM_CHECK(!field_names.empty());
auto vals = SplitCsv_(entry_csv_line); auto vals = SplitCsv_(entry_csv_line);
CHECK_EQ(vals.size(), field_names.size()); BM_CHECK_EQ(vals.size(), field_names.size());
results.emplace_back(vals[0]); // vals[0] is the benchmark name results.emplace_back(vals[0]); // vals[0] is the benchmark name
auto& entry = results.back(); auto& entry = results.back();
for (size_t i = 1, e = vals.size(); i < e; ++i) { for (size_t i = 1, e = vals.size(); i < e; ++i) {
entry.values[field_names[i]] = vals[i]; entry.values[field_names[i]] = vals[i];
} }
} }
// a quick'n'dirty csv splitter (eliminating quotes) // a quick'n'dirty csv splitter (eliminating quotes)
std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) { std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) {
std::vector<std::string> out; std::vector<std::string> out;
if (line.empty()) return out; if (line.empty()) return out;
if (!field_names.empty()) out.reserve(field_names.size()); if (!field_names.empty()) out.reserve(field_names.size());
size_t prev = 0, pos = line.find_first_of(','), curr = pos; size_t prev = 0, pos = line.find_first_of(','), curr = pos;
while (pos != line.npos) { while (pos != line.npos) {
CHECK(curr > 0); BM_CHECK(curr > 0);
if (line[prev] == '"') ++prev; if (line[prev] == '"') ++prev;
if (line[curr - 1] == '"') --curr; if (line[curr - 1] == '"') --curr;
out.push_back(line.substr(prev, curr - prev)); out.push_back(line.substr(prev, curr - prev));
prev = pos + 1; prev = pos + 1;
pos = line.find_first_of(',', pos + 1); pos = line.find_first_of(',', pos + 1);
curr = pos; curr = pos;
} }
curr = line.size(); curr = line.size();
if (line[prev] == '"') ++prev; if (line[prev] == '"') ++prev;
if (line[curr - 1] == '"') --curr; if (line[curr - 1] == '"') --curr;
out.push_back(line.substr(prev, curr - prev)); out.push_back(line.substr(prev, curr - prev));
return out; return out;
} }
} // end namespace internal } // end namespace internal
size_t AddChecker(const char* bm_name, ResultsCheckFn fn) { size_t AddChecker(const char* bm_name, const ResultsCheckFn& fn) {
auto& rc = internal::GetResultsChecker(); auto& rc = internal::GetResultsChecker();
rc.Add(bm_name, fn); rc.Add(bm_name, fn);
return rc.results.size(); return rc.results.size();
} }
int Results::NumThreads() const { int Results::NumThreads() const {
auto pos = name.find("/threads:"); auto pos = name.find("/threads:");
if (pos == name.npos) return 1; if (pos == name.npos) return 1;
auto end = name.find('/', pos + 9); auto end = name.find('/', pos + 9);
std::stringstream ss; std::stringstream ss;
ss << name.substr(pos + 9, end); ss << name.substr(pos + 9, end);
int num = 1; int num = 1;
ss >> num; ss >> num;
CHECK(!ss.fail()); BM_CHECK(!ss.fail());
return num; return num;
} }
double Results::NumIterations() const { double Results::NumIterations() const { return GetAs<double>("iterations"); }
return GetAs<double>("iterations");
}
double Results::GetTime(BenchmarkTime which) const { double Results::GetTime(BenchmarkTime which) const {
CHECK(which == kCpuTime || which == kRealTime); BM_CHECK(which == kCpuTime || which == kRealTime);
const char* which_str = which == kCpuTime ? "cpu_time" : "real_time"; const char* which_str = which == kCpuTime ? "cpu_time" : "real_time";
double val = GetAs<double>(which_str); double val = GetAs<double>(which_str);
auto unit = Get("time_unit"); auto unit = Get("time_unit");
CHECK(unit); BM_CHECK(unit);
if (*unit == "ns") { if (*unit == "ns") {
return val * 1.e-9; return val * 1.e-9;
} else if (*unit == "us") { } else if (*unit == "us") {
return val * 1.e-6; return val * 1.e-6;
} else if (*unit == "ms") { } else if (*unit == "ms") {
return val * 1.e-3; return val * 1.e-3;
} else if (*unit == "s") { } else if (*unit == "s") {
return val; return val;
} else { } else {
CHECK(1 == 0) << "unknown time unit: " << *unit; BM_CHECK(1 == 0) << "unknown time unit: " << *unit;
return 0; return 0;
} }
} }
// ========================================================================= // // ========================================================================= //
// -------------------------- Public API Definitions------------------------ // // -------------------------- Public API Definitions------------------------ //
// ========================================================================= // // ========================================================================= //
TestCase::TestCase(std::string re, int rule) TestCase::TestCase(std::string re, int rule)
: regex_str(std::move(re)), : regex_str(std::move(re)),
match_rule(rule), match_rule(rule),
substituted_regex(internal::PerformSubstitutions(regex_str)), substituted_regex(internal::PerformSubstitutions(regex_str)),
regex(std::make_shared<benchmark::Regex>()) { regex(std::make_shared<benchmark::Regex>()) {
std::string err_str; std::string err_str;
regex->Init(substituted_regex, &err_str); regex->Init(substituted_regex, &err_str);
CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex BM_CHECK(err_str.empty())
<< "\"" << "Could not construct regex \"" << substituted_regex << "\""
<< "\n originally \"" << regex_str << "\"" << "\n originally \"" << regex_str << "\""
<< "\n got error: " << err_str; << "\n got error: " << err_str;
} }
int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) { int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
auto& L = internal::GetTestCaseList(ID); auto& L = internal::GetTestCaseList(ID);
L.insert(L.end(), il); L.insert(L.end(), il);
return 0; return 0;
} }
Show All 12 Lines for (auto KV : il) {
} }
if (!exists) subs.push_back(std::move(KV)); if (!exists) subs.push_back(std::move(KV));
} }
return 0; return 0;
} }
// Disable deprecated warnings temporarily because we need to reference // Disable deprecated warnings temporarily because we need to reference
// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations // CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations
#ifdef __GNUC__ BENCHMARK_DISABLE_DEPRECATED_WARNING
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
void RunOutputTests(int argc, char* argv[]) { void RunOutputTests(int argc, char* argv[]) {
using internal::GetTestCaseList; using internal::GetTestCaseList;
benchmark::Initialize(&argc, argv); benchmark::Initialize(&argc, argv);
auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true); auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true);
benchmark::ConsoleReporter CR(options); benchmark::ConsoleReporter CR(options);
benchmark::JSONReporter JR; benchmark::JSONReporter JR;
benchmark::CSVReporter CSVR; benchmark::CSVReporter CSVR;
struct ReporterTest { struct ReporterTest {
Show All 38 Lines for (auto& rep_test : TestCases) {
std::cout << "\n"; std::cout << "\n";
} }
// now that we know the output is as expected, we can dispatch // now that we know the output is as expected, we can dispatch
// the checks to subscribees. // the checks to subscribees.
auto& csv = TestCases[2]; auto& csv = TestCases[2];
// would use == but gcc spits a warning // would use == but gcc spits a warning
CHECK(std::strcmp(csv.name, "CSVReporter") == 0); BM_CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
internal::GetResultsChecker().CheckResults(csv.out_stream); internal::GetResultsChecker().CheckResults(csv.out_stream);
} }
#ifdef __GNUC__ BENCHMARK_RESTORE_DEPRECATED_WARNING
#pragma GCC diagnostic pop
#endif
int SubstrCnt(const std::string& haystack, const std::string& pat) { int SubstrCnt(const std::string& haystack, const std::string& pat) {
if (pat.length() == 0) return 0; if (pat.length() == 0) return 0;
int count = 0; int count = 0;
for (size_t offset = haystack.find(pat); offset != std::string::npos; for (size_t offset = haystack.find(pat); offset != std::string::npos;
offset = haystack.find(pat, offset + pat.length())) offset = haystack.find(pat, offset + pat.length()))
++count; ++count;
return count; return count;
} }
static char ToHex(int ch) { static char ToHex(int ch) {
return ch < 10 ? static_cast<char>('0' + ch) return ch < 10 ? static_cast<char>('0' + ch)
: static_cast<char>('a' + (ch - 10)); : static_cast<char>('a' + (ch - 10));
} }
static char RandomHexChar() { static char RandomHexChar() {
static std::mt19937 rd{std::random_device{}()}; static std::mt19937 rd{std::random_device{}()};
static std::uniform_int_distribution<int> mrand{0, 15}; static std::uniform_int_distribution<int> mrand{0, 15};
return ToHex(mrand(rd)); return ToHex(mrand(rd));
} }
static std::string GetRandomFileName() { static std::string GetRandomFileName() {
std::string model = "test.%%%%%%"; std::string model = "test.%%%%%%";
for (auto & ch : model) { for (auto& ch : model) {
if (ch == '%') if (ch == '%') ch = RandomHexChar();
ch = RandomHexChar();
} }
return model; return model;
} }
static bool FileExists(std::string const& name) { static bool FileExists(std::string const& name) {
std::ifstream in(name.c_str()); std::ifstream in(name.c_str());
return in.good(); return in.good();
} }
static std::string GetTempFileName() { static std::string GetTempFileName() {
// This function attempts to avoid race conditions where two tests // This function attempts to avoid race conditions where two tests
// create the same file at the same time. However, it still introduces races // create the same file at the same time. However, it still introduces races
// similar to tmpnam. // similar to tmpnam.
int retries = 3; int retries = 3;
while (--retries) { while (--retries) {
std::string name = GetRandomFileName(); std::string name = GetRandomFileName();
if (!FileExists(name)) if (!FileExists(name)) return name;
return name;
} }
std::cerr << "Failed to create unique temporary file name" << std::endl; std::cerr << "Failed to create unique temporary file name" << std::endl;
std::abort(); std::abort();
} }
std::string GetFileReporterOutput(int argc, char* argv[]) { std::string GetFileReporterOutput(int argc, char* argv[]) {
std::vector<char*> new_argv(argv, argv + argc); std::vector<char*> new_argv(argv, argv + argc);
assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size()); assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size());
Show All 21 Lines

third-party/benchmark/test/perf_counters_gtest.cc

#include <thread> #include <thread>
#include "../src/perf_counters.h" #include "../src/perf_counters.h"
#include "gtest/gtest.h" #include "gtest/gtest.h"
#ifndef GTEST_SKIP #ifndef GTEST_SKIP
struct MsgHandler { struct MsgHandler {
void operator=(std::ostream&){} void operator=(std::ostream&) {}
}; };
#define GTEST_SKIP() return MsgHandler() = std::cout #define GTEST_SKIP() return MsgHandler() = std::cout
#endif #endif
using benchmark::internal::PerfCounters; using benchmark::internal::PerfCounters;
using benchmark::internal::PerfCounterValues; using benchmark::internal::PerfCounterValues;
namespace { namespace {
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines
size_t do_work() { size_t do_work() {
size_t res = 0; size_t res = 0;
for (size_t i = 0; i < 100000000; ++i) res += i * i; for (size_t i = 0; i < 100000000; ++i) res += i * i;
return res; return res;
} }
void measure(size_t threadcount, PerfCounterValues* values1, void measure(size_t threadcount, PerfCounterValues* values1,
PerfCounterValues* values2) { PerfCounterValues* values2) {
CHECK_NE(values1, nullptr); BM_CHECK_NE(values1, nullptr);
CHECK_NE(values2, nullptr); BM_CHECK_NE(values2, nullptr);
std::vector<std::thread> threads(threadcount); std::vector<std::thread> threads(threadcount);
auto work = [&]() { CHECK(do_work() > 1000); }; auto work = [&]() { BM_CHECK(do_work() > 1000); };
// We need to first set up the counters, then start the threads, so the // We need to first set up the counters, then start the threads, so the
// threads would inherit the counters. But later, we need to first destroy the // threads would inherit the counters. But later, we need to first destroy the
// thread pool (so all the work finishes), then measure the counters. So the // thread pool (so all the work finishes), then measure the counters. So the
// scopes overlap, and we need to explicitly control the scope of the // scopes overlap, and we need to explicitly control the scope of the
// threadpool. // threadpool.
auto counters = auto counters =
PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent3}); PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent3});
Show All 28 Lines

third-party/benchmark/test/perf_counters_test.cc

#undef NDEBUG #undef NDEBUG
#include "../src/perf_counters.h" #include "../src/perf_counters.h"
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "output_test.h" #include "output_test.h"
void BM_Simple(benchmark::State& state) { static void BM_Simple(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
benchmark::DoNotOptimize(state.iterations()); benchmark::DoNotOptimize(state.iterations());
} }
} }
BENCHMARK(BM_Simple); BENCHMARK(BM_Simple);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Simple\",$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Simple\",$"}});
void CheckSimple(Results const& e) { static void CheckSimple(Results const& e) {
CHECK_COUNTER_VALUE(e, double, "CYCLES", GT, 0); CHECK_COUNTER_VALUE(e, double, "CYCLES", GT, 0);
CHECK_COUNTER_VALUE(e, double, "BRANCHES", GT, 0.0); CHECK_COUNTER_VALUE(e, double, "BRANCHES", GT, 0.0);
} }
CHECK_BENCHMARK_RESULTS("BM_Simple", &CheckSimple); CHECK_BENCHMARK_RESULTS("BM_Simple", &CheckSimple);
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
if (!benchmark::internal::PerfCounters::kSupported) { if (!benchmark::internal::PerfCounters::kSupported) {
return 0; return 0;
} }
RunOutputTests(argc, argv); RunOutputTests(argc, argv);
} }

third-party/benchmark/test/register_benchmark_test.cc

Show All 24 Linesstruct TestCase {
TestCase(const char* xname) : TestCase(xname, nullptr) {} TestCase(const char* xname) : TestCase(xname, nullptr) {}
TestCase(const char* xname, const char* xlabel) TestCase(const char* xname, const char* xlabel)
: name(xname), label(xlabel) {} : name(xname), label(xlabel) {}
typedef benchmark::BenchmarkReporter::Run Run; typedef benchmark::BenchmarkReporter::Run Run;
void CheckRun(Run const& run) const { void CheckRun(Run const& run) const {
// clang-format off // clang-format off
CHECK(name == run.benchmark_name()) << "expected " << name << " got " BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got "
<< run.benchmark_name(); << run.benchmark_name();
if (label) { if (label) {
CHECK(run.report_label == label) << "expected " << label << " got " BM_CHECK(run.report_label == label) << "expected " << label << " got "
<< run.report_label; << run.report_label;
} else { } else {
CHECK(run.report_label == ""); BM_CHECK(run.report_label.empty());
} }
// clang-format on // clang-format on
} }
}; };
std::vector<TestCase> ExpectedResults; std::vector<TestCase> ExpectedResults;
int AddCases(std::initializer_list<TestCase> const& v) { int AddCases(std::initializer_list<TestCase> const& v) {
for (auto N : v) { for (const auto& N : v) {
ExpectedResults.push_back(N); ExpectedResults.push_back(N);
} }
return 0; return 0;
} }
#define CONCAT(x, y) CONCAT2(x, y) #define CONCAT(x, y) CONCAT2(x, y)
#define CONCAT2(x, y) x##y #define CONCAT2(x, y) x##y
#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__}) #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__})
▲ Show 20 Lines • Show All 128 Lines • Show Last 20 Lines

third-party/benchmark/test/repetitions_test.cc

#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "output_test.h" #include "output_test.h"
// ========================================================================= // // ========================================================================= //
// ------------------------ Testing Basic Output --------------------------- // // ------------------------ Testing Basic Output --------------------------- //
// ========================================================================= // // ========================================================================= //
void BM_ExplicitRepetitions(benchmark::State& state) { static void BM_ExplicitRepetitions(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
} }
} }
BENCHMARK(BM_ExplicitRepetitions)->Repetitions(2); BENCHMARK(BM_ExplicitRepetitions)->Repetitions(2);
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
{{"^BM_ExplicitRepetitions/repeats:2 %console_report$"}}); {{"^BM_ExplicitRepetitions/repeats:2 %console_report$"}});
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
Show All 36 LinesADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_ExplicitRepetitions/repeats:2_mean\",$"}, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_mean\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_ExplicitRepetitions/repeats:2_median\",$"}, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_median\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_ExplicitRepetitions/repeats:2_stddev\",$"}, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_stddev\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ExplicitRepetitions/repeats:2\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ExplicitRepetitions/repeats:2\",%csv_report$"}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ExplicitRepetitions/repeats:2\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ExplicitRepetitions/repeats:2\",%csv_report$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_ExplicitRepetitions/repeats:2_mean\",%csv_report$"}}); {{"^\"BM_ExplicitRepetitions/repeats:2_mean\",%csv_report$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_ExplicitRepetitions/repeats:2_median\",%csv_report$"}}); {{"^\"BM_ExplicitRepetitions/repeats:2_median\",%csv_report$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_ExplicitRepetitions/repeats:2_stddev\",%csv_report$"}}); {{"^\"BM_ExplicitRepetitions/repeats:2_stddev\",%csv_report$"}});
// ========================================================================= // // ========================================================================= //
// ------------------------ Testing Basic Output --------------------------- // // ------------------------ Testing Basic Output --------------------------- //
// ========================================================================= // // ========================================================================= //
void BM_ImplicitRepetitions(benchmark::State& state) { static void BM_ImplicitRepetitions(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
} }
} }
BENCHMARK(BM_ImplicitRepetitions); BENCHMARK(BM_ImplicitRepetitions);
ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}});
ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}});
ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}});
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"},
{"\"family_index\": 1,$", MR_Next}, {"\"family_index\": 1,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"},
{"\"family_index\": 1,$", MR_Next}, {"\"family_index\": 1,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_stddev\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_stddev\",$"},
{"\"family_index\": 1,$", MR_Next}, {"\"family_index\": 1,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions\",%csv_report$"}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions\",%csv_report$"}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_mean\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_mean\",%csv_report$"}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_median\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_median\",%csv_report$"}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_stddev\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_stddev\",%csv_report$"}});
// ========================================================================= // // ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ // // --------------------------- TEST CASES END ------------------------------ //
// ========================================================================= // // ========================================================================= //
int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }

third-party/benchmark/test/report_aggregates_only_test.cc

Show All 13 Linesvoid BM_SummaryRepeat(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
} }
} }
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
const std::string output = GetFileReporterOutput(argc, argv); const std::string output = GetFileReporterOutput(argc, argv);
if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 3 || if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 4 ||
SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 || SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 ||
SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") != SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") !=
1 || 1 ||
SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") != SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") !=
1) { 1 ||
std::cout << "Precondition mismatch. Expected to only find three " SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_cv\"") != 1) {
std::cout << "Precondition mismatch. Expected to only find four "
"occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n" "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n"
"\"name\": \"BM_SummaryRepeat/repeats:3_mean\", " "\"name\": \"BM_SummaryRepeat/repeats:3_mean\", "
"\"name\": \"BM_SummaryRepeat/repeats:3_median\", " "\"name\": \"BM_SummaryRepeat/repeats:3_median\", "
"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire " "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\", "
"\"name\": \"BM_SummaryRepeat/repeats:3_cv\"\nThe entire "
"output:\n"; "output:\n";
std::cout << output; std::cout << output;
return 1; return 1;
} }
return 0; return 0;
} }

third-party/benchmark/test/reporter_output_test.cc

#undef NDEBUG #undef NDEBUG
#include <numeric>
#include <utility> #include <utility>
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "output_test.h" #include "output_test.h"
// ========================================================================= // // ========================================================================= //
// ---------------------- Testing Prologue Output -------------------------- // // ---------------------- Testing Prologue Output -------------------------- //
// ========================================================================= // // ========================================================================= //
▲ Show 20 Lines • Show All 438 Lines • ▼ Show 20 LinesADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
{"\"family_index\": 15,$", MR_Next}, {"\"family_index\": 15,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_median\",$"}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
{"\"family_index\": 15,$", MR_Next}, {"\"family_index\": 15,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
{"\"family_index\": 15,$", MR_Next}, {"\"family_index\": 15,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}}); {"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
{"^\"BM_Repeat/repeats:2_median\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
{"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}}); {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
// but for two repetitions, mean and median is the same, so let's repeat.. // but for two repetitions, mean and median is the same, so let's repeat..
BENCHMARK(BM_Repeat)->Repetitions(3); BENCHMARK(BM_Repeat)->Repetitions(3);
Show All 31 LinesADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
{"\"family_index\": 16,$", MR_Next}, {"\"family_index\": 16,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_median\",$"}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
{"\"family_index\": 16,$", MR_Next}, {"\"family_index\": 16,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
{"\"family_index\": 16,$", MR_Next}, {"\"family_index\": 16,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}}); {"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_mean\",%csv_report$"}, {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_median\",%csv_report$"}, {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
// median differs between even/odd number of repetitions, so just to be sure // median differs between even/odd number of repetitions, so just to be sure
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 LinesADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
{"\"family_index\": 17,$", MR_Next}, {"\"family_index\": 17,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 4,$", MR_Next}, {"\"repetitions\": 4,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next}, {"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_median\",$"}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
{"\"family_index\": 17,$", MR_Next}, {"\"family_index\": 17,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 4,$", MR_Next}, {"\"repetitions\": 4,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next}, {"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
{"\"family_index\": 17,$", MR_Next}, {"\"family_index\": 17,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 4,$", MR_Next}, {"\"repetitions\": 4,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next}}); {"\"iterations\": 4,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4_mean\",%csv_report$"}, {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
{"^\"BM_Repeat/repeats:4_median\",%csv_report$"}, {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
{"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}}); {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
Show All 33 LinesADD_CASES(TC_JSONOut,
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
{"\"family_index\": 19,$", MR_Next}, {"\"family_index\": 19,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
{"\"family_index\": 19,$", MR_Next}, {"\"family_index\": 19,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
{"\"family_index\": 19,$", MR_Next}, {"\"family_index\": 19,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}}); {"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
{"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
{"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
// Test that non-aggregate data is not displayed. // Test that non-aggregate data is not displayed.
// NOTE: this test is kinda bad. we are only testing the display output. // NOTE: this test is kinda bad. we are only testing the display output.
Show All 14 LinesADD_CASES(TC_JSONOut,
{"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
{"\"family_index\": 20,$", MR_Next}, {"\"family_index\": 20,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
{"\"family_index\": 20,$", MR_Next}, {"\"family_index\": 20,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
{"\"family_index\": 20,$", MR_Next}, {"\"family_index\": 20,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}}); {"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
{"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"}, {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
{"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"}, {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
{"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}}); {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
// Test repeats with custom time unit. // Test repeats with custom time unit.
Show All 18 LinesADD_CASES(TC_JSONOut,
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
{"\"family_index\": 21,$", MR_Next}, {"\"family_index\": 21,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"}, {"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
{"\"family_index\": 21,$", MR_Next}, {"\"family_index\": 21,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"}, {"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
{"\"family_index\": 21,$", MR_Next}, {"\"family_index\": 21,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"}}); {"\"time_unit\": \"us\",?$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
{"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
{"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"}, {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
{"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}}); {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 LinesADD_CASES(
{"\"family_index\": 22,$", MR_Next}, {"\"family_index\": 22,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
{"\"family_index\": 22,$", MR_Next}, {"\"family_index\": 22,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
{"\"family_index\": 22,$", MR_Next}, {"\"family_index\": 22,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
{"\"family_index\": 22,$", MR_Next}, {"\"family_index\": 22,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next}, {"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"\",$", MR_Next}, {"\"aggregate_name\": \"\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}, {"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}}); {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
ADD_CASES( ADD_CASES(
TC_CSVOut, TC_CSVOut,
{{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"}, {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/" {"^\"BM_UserStats/iterations:5/repeats:3/"
"manual_time_median\",%csv_report$"}, "manual_time_median\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/" {"^\"BM_UserStats/iterations:5/repeats:3/"
"manual_time_stddev\",%csv_report$"}, "manual_time_stddev\",%csv_report$"},
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}}); {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
// ========================================================================= // // ========================================================================= //
// ------------- Testing relative standard deviation statistics ------------ //
// ========================================================================= //
const auto UserPercentStatistics = [](const std::vector<double>&) {
return 1. / 100.;
};
void BM_UserPercentStats(benchmark::State& state) {
for (auto _ : state) {
state.SetIterationTime(150 / 10e8);
}
}
// clang-format off
BENCHMARK(BM_UserPercentStats)
->Repetitions(3)
->Iterations(5)
->UseManualTime()
->Unit(benchmark::TimeUnit::kNanosecond)
->ComputeStatistics("", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);
// clang-format on
// check that UserPercent-provided stats is calculated, and is after the
// default-ones empty string as name is intentional, it would sort before
// anything else
ADD_CASES(TC_ConsoleOut,
{{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
"]* 150 ns %time [ ]*5$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
"]* 150 ns %time [ ]*5$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
"]* 150 ns %time [ ]*5$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_mean [ ]* 150 ns %time [ ]*3$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_median [ ]* 150 ns %time [ ]*3$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "
"[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});
ADD_CASES(
TC_JSONOut,
{{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"repetition_index\": 0,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"repetition_index\": 1,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"repetition_index\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},
{"\"family_index\": 23,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": "
"\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 3,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"\",$", MR_Next},
{"\"aggregate_unit\": \"percentage\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_mean\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_median\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_stddev\",%csv_report$"},
{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
"manual_time_\",%csv_report$"}});
// ========================================================================= //
// ------------------------- Testing StrEscape JSON ------------------------ // // ------------------------- Testing StrEscape JSON ------------------------ //
// ========================================================================= // // ========================================================================= //
#if 0 // enable when csv testing code correctly handles multi-line fields #if 0 // enable when csv testing code correctly handles multi-line fields
void BM_JSON_Format(benchmark::State& state) { void BM_JSON_Format(benchmark::State& state) {
state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes"); state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
for (auto _ : state) { for (auto _ : state) {
} }
} }
Show All 29 Lines

third-party/benchmark/test/skip_with_error_test.cc

Show All 27 Lines
struct TestCase { struct TestCase {
std::string name; std::string name;
bool error_occurred; bool error_occurred;
std::string error_message; std::string error_message;
typedef benchmark::BenchmarkReporter::Run Run; typedef benchmark::BenchmarkReporter::Run Run;
void CheckRun(Run const& run) const { void CheckRun(Run const& run) const {
CHECK(name == run.benchmark_name()) BM_CHECK(name == run.benchmark_name())
<< "expected " << name << " got " << run.benchmark_name(); << "expected " << name << " got " << run.benchmark_name();
CHECK(error_occurred == run.error_occurred); BM_CHECK(error_occurred == run.error_occurred);
CHECK(error_message == run.error_message); BM_CHECK(error_message == run.error_message);
if (error_occurred) { if (error_occurred) {
// CHECK(run.iterations == 0); // BM_CHECK(run.iterations == 0);
} else { } else {
CHECK(run.iterations != 0); BM_CHECK(run.iterations != 0);
} }
} }
}; };
std::vector<TestCase> ExpectedResults; std::vector<TestCase> ExpectedResults;
int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) { int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
for (auto TC : v) { for (auto TC : v) {
Show All 40 Linesvoid BM_error_before_running_range_for(benchmark::State& state) {
} }
} }
BENCHMARK(BM_error_before_running_range_for); BENCHMARK(BM_error_before_running_range_for);
ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}}); ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}});
void BM_error_during_running(benchmark::State& state) { void BM_error_during_running(benchmark::State& state) {
int first_iter = true; int first_iter = true;
while (state.KeepRunning()) { while (state.KeepRunning()) {
if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) { if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {
assert(first_iter); assert(first_iter);
first_iter = false; first_iter = false;
state.SkipWithError("error message"); state.SkipWithError("error message");
} else { } else {
state.PauseTiming(); state.PauseTiming();
state.ResumeTiming(); state.ResumeTiming();
} }
} }
} }
BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8); BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"}, ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
{"/1/threads:2", true, "error message"}, {"/1/threads:2", true, "error message"},
{"/1/threads:4", true, "error message"}, {"/1/threads:4", true, "error message"},
{"/1/threads:8", true, "error message"}, {"/1/threads:8", true, "error message"},
{"/2/threads:1", false, ""}, {"/2/threads:1", false, ""},
{"/2/threads:2", false, ""}, {"/2/threads:2", false, ""},
{"/2/threads:4", false, ""}, {"/2/threads:4", false, ""},
{"/2/threads:8", false, ""}}); {"/2/threads:8", false, ""}});
void BM_error_during_running_ranged_for(benchmark::State& state) { void BM_error_during_running_ranged_for(benchmark::State& state) {
assert(state.max_iterations > 3 && "test requires at least a few iterations"); assert(state.max_iterations > 3 && "test requires at least a few iterations");
int first_iter = true; bool first_iter = true;
// NOTE: Users should not write the for loop explicitly. // NOTE: Users should not write the for loop explicitly.
for (auto It = state.begin(), End = state.end(); It != End; ++It) { for (auto It = state.begin(), End = state.end(); It != End; ++It) {
if (state.range(0) == 1) { if (state.range(0) == 1) {
assert(first_iter); assert(first_iter);
first_iter = false; first_iter = false;
(void)first_iter;
state.SkipWithError("error message"); state.SkipWithError("error message");
// Test the unfortunate but documented behavior that the ranged-for loop // Test the unfortunate but documented behavior that the ranged-for loop
// doesn't automatically terminate when SkipWithError is set. // doesn't automatically terminate when SkipWithError is set.
assert(++It != End); assert(++It != End);
break; // Required behavior break; // Required behavior
} }
} }
} }
BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5); BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5);
ADD_CASES("BM_error_during_running_ranged_for", ADD_CASES("BM_error_during_running_ranged_for",
{{"/1/iterations:5", true, "error message"}, {{"/1/iterations:5", true, "error message"},
{"/2/iterations:5", false, ""}}); {"/2/iterations:5", false, ""}});
void BM_error_after_running(benchmark::State& state) { void BM_error_after_running(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
benchmark::DoNotOptimize(state.iterations()); benchmark::DoNotOptimize(state.iterations());
} }
if (state.thread_index <= (state.threads / 2)) if (state.thread_index() <= (state.threads() / 2))
state.SkipWithError("error message"); state.SkipWithError("error message");
} }
BENCHMARK(BM_error_after_running)->ThreadRange(1, 8); BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"}, ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
{"/threads:2", true, "error message"}, {"/threads:2", true, "error message"},
{"/threads:4", true, "error message"}, {"/threads:4", true, "error message"},
{"/threads:8", true, "error message"}}); {"/threads:8", true, "error message"}});
void BM_error_while_paused(benchmark::State& state) { void BM_error_while_paused(benchmark::State& state) {
bool first_iter = true; bool first_iter = true;
while (state.KeepRunning()) { while (state.KeepRunning()) {
if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) { if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) {
assert(first_iter); assert(first_iter);
first_iter = false; first_iter = false;
state.PauseTiming(); state.PauseTiming();
state.SkipWithError("error message"); state.SkipWithError("error message");
} else { } else {
state.PauseTiming(); state.PauseTiming();
state.ResumeTiming(); state.ResumeTiming();
} }
Show All 30 Lines

third-party/benchmark/test/spec_arg_test.cc

  • This file was added.
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <limits>
#include <string>
#include <vector>
#include "benchmark/benchmark.h"
// Tests that we can override benchmark-spec value from FLAGS_benchmark_filter
// with argument to RunSpecifiedBenchmarks(...).
namespace {
class TestReporter : public benchmark::ConsoleReporter {
public:
virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE {
return ConsoleReporter::ReportContext(context);
};
virtual void ReportRuns(const std::vector<Run>& report) BENCHMARK_OVERRIDE {
assert(report.size() == 1);
matched_functions.push_back(report[0].run_name.function_name);
ConsoleReporter::ReportRuns(report);
};
TestReporter() {}
virtual ~TestReporter() {}
const std::vector<std::string>& GetMatchedFunctions() const {
return matched_functions;
}
private:
std::vector<std::string> matched_functions;
};
} // end namespace
static void BM_NotChosen(benchmark::State& state) {
assert(false && "SHOULD NOT BE CALLED");
for (auto _ : state) {
}
}
BENCHMARK(BM_NotChosen);
static void BM_Chosen(benchmark::State& state) {
for (auto _ : state) {
}
}
BENCHMARK(BM_Chosen);
int main(int argc, char** argv) {
const std::string flag = "BM_NotChosen";
// Verify that argv specify --benchmark_filter=BM_NotChosen.
bool found = false;
for (int i = 0; i < argc; ++i) {
if (strcmp("--benchmark_filter=BM_NotChosen", argv[i]) == 0) {
found = true;
break;
}
}
assert(found);
benchmark::Initialize(&argc, argv);
// Check that the current flag value is reported accurately via the
// GetBenchmarkFilter() function.
if (flag != benchmark::GetBenchmarkFilter()) {
std::cerr
<< "Seeing different value for flags. GetBenchmarkFilter() returns ["
<< benchmark::GetBenchmarkFilter() << "] expected flag=[" << flag
<< "]\n";
return 1;
}
TestReporter test_reporter;
const char* const spec = "BM_Chosen";
const size_t returned_count =
benchmark::RunSpecifiedBenchmarks(&test_reporter, spec);
assert(returned_count == 1);
const std::vector<std::string> matched_functions =
test_reporter.GetMatchedFunctions();
assert(matched_functions.size() == 1);
if (strcmp(spec, matched_functions.front().c_str()) != 0) {
std::cerr << "Expected benchmark [" << spec << "] to run, but got ["
<< matched_functions.front() << "]\n";
return 2;
}
return 0;
}

third-party/benchmark/test/statistics_gtest.cc

Show All 19 Lines
TEST(StatisticsTest, StdDev) { TEST(StatisticsTest, StdDev) {
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({101, 101, 101, 101}), 0.0); EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({101, 101, 101, 101}), 0.0);
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1, 2, 3}), 1.0); EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1, 2, 3}), 1.0);
EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({2.5, 2.4, 3.3, 4.2, 5.1}), EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({2.5, 2.4, 3.3, 4.2, 5.1}),
1.151086443322134); 1.151086443322134);
} }
TEST(StatisticsTest, CV) {
EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({101, 101, 101, 101}), 0.0);
EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({1, 2, 3}), 1. / 2.);
EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}),
0.32888184094918121);
}
} // end namespace } // end namespace

third-party/benchmark/test/string_util_gtest.cc

//===---------------------------------------------------------------------===// //===---------------------------------------------------------------------===//
// statistics_test - Unit tests for src/statistics.cc // statistics_test - Unit tests for src/statistics.cc
//===---------------------------------------------------------------------===// //===---------------------------------------------------------------------===//
#include "../src/string_util.h"
#include "../src/internal_macros.h" #include "../src/internal_macros.h"
#include "../src/string_util.h"
#include "gtest/gtest.h" #include "gtest/gtest.h"
namespace { namespace {
TEST(StringUtilTest, stoul) { TEST(StringUtilTest, stoul) {
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(0ul, benchmark::stoul("0", &pos)); EXPECT_EQ(0ul, benchmark::stoul("0", &pos));
EXPECT_EQ(1ul, pos); EXPECT_EQ(1ul, pos);
Show All 12 Lines#if ULONG_MAX == 0xFFFFFFFFul
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(0xFFFFFFFFul, benchmark::stoul("4294967295", &pos)); EXPECT_EQ(0xFFFFFFFFul, benchmark::stoul("4294967295", &pos));
EXPECT_EQ(10ul, pos); EXPECT_EQ(10ul, pos);
} }
#elif ULONG_MAX == 0xFFFFFFFFFFFFFFFFul #elif ULONG_MAX == 0xFFFFFFFFFFFFFFFFul
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(0xFFFFFFFFFFFFFFFFul, benchmark::stoul("18446744073709551615", &pos)); EXPECT_EQ(0xFFFFFFFFFFFFFFFFul,
benchmark::stoul("18446744073709551615", &pos));
EXPECT_EQ(20ul, pos); EXPECT_EQ(20ul, pos);
} }
#endif #endif
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(10ul, benchmark::stoul("1010", &pos, 2)); EXPECT_EQ(10ul, benchmark::stoul("1010", &pos, 2));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
Show All 13 Lines#endif
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16)); EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
#ifndef BENCHMARK_HAS_NO_EXCEPTIONS #ifndef BENCHMARK_HAS_NO_EXCEPTIONS
{ { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); }
ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument);
}
#endif #endif
} }
TEST(StringUtilTest, stoi) { TEST(StringUtilTest, stoi){{size_t pos = 0;
{
size_t pos = 0;
EXPECT_EQ(0, benchmark::stoi("0", &pos)); EXPECT_EQ(0, benchmark::stoi("0", &pos));
EXPECT_EQ(1ul, pos); EXPECT_EQ(1ul, pos);
} } // namespace
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(-17, benchmark::stoi("-17", &pos)); EXPECT_EQ(-17, benchmark::stoi("-17", &pos));
EXPECT_EQ(3ul, pos); EXPECT_EQ(3ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(1357, benchmark::stoi("1357", &pos)); EXPECT_EQ(1357, benchmark::stoi("1357", &pos));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(10, benchmark::stoi("1010", &pos, 2)); EXPECT_EQ(10, benchmark::stoi("1010", &pos, 2));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(520, benchmark::stoi("1010", &pos, 8)); EXPECT_EQ(520, benchmark::stoi("1010", &pos, 8));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(1010, benchmark::stoi("1010", &pos, 10)); EXPECT_EQ(1010, benchmark::stoi("1010", &pos, 10));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(4112, benchmark::stoi("1010", &pos, 16)); EXPECT_EQ(4112, benchmark::stoi("1010", &pos, 16));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16)); EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
#ifndef BENCHMARK_HAS_NO_EXCEPTIONS #ifndef BENCHMARK_HAS_NO_EXCEPTIONS
{ { ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); }
ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument);
}
#endif #endif
} }
TEST(StringUtilTest, stod) { TEST(StringUtilTest, stod){{size_t pos = 0;
{
size_t pos = 0;
EXPECT_EQ(0.0, benchmark::stod("0", &pos)); EXPECT_EQ(0.0, benchmark::stod("0", &pos));
EXPECT_EQ(1ul, pos); EXPECT_EQ(1ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(-84.0, benchmark::stod("-84", &pos)); EXPECT_EQ(-84.0, benchmark::stod("-84", &pos));
EXPECT_EQ(3ul, pos); EXPECT_EQ(3ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(1234.0, benchmark::stod("1234", &pos)); EXPECT_EQ(1234.0, benchmark::stod("1234", &pos));
EXPECT_EQ(4ul, pos); EXPECT_EQ(4ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
EXPECT_EQ(1.5, benchmark::stod("1.5", &pos)); EXPECT_EQ(1.5, benchmark::stod("1.5", &pos));
EXPECT_EQ(3ul, pos); EXPECT_EQ(3ul, pos);
} }
{ {
size_t pos = 0; size_t pos = 0;
/* Note: exactly representable as double */ /* Note: exactly representable as double */
EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos)); EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos));
EXPECT_EQ(8ul, pos); EXPECT_EQ(8ul, pos);
} }
#ifndef BENCHMARK_HAS_NO_EXCEPTIONS #ifndef BENCHMARK_HAS_NO_EXCEPTIONS
{ { ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); }
ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument);
}
#endif #endif
} }
TEST(StringUtilTest, StrSplit) { TEST(StringUtilTest, StrSplit) {
EXPECT_EQ(benchmark::StrSplit("", ','), std::vector<std::string>{}); EXPECT_EQ(benchmark::StrSplit("", ','), std::vector<std::string>{});
EXPECT_EQ(benchmark::StrSplit("hello", ','), EXPECT_EQ(benchmark::StrSplit("hello", ','),
std::vector<std::string>({"hello"})); std::vector<std::string>({"hello"}));
EXPECT_EQ(benchmark::StrSplit("hello,there,is,more", ','), EXPECT_EQ(benchmark::StrSplit("hello,there,is,more", ','),
std::vector<std::string>({"hello", "there", "is", "more"})); std::vector<std::string>({"hello", "there", "is", "more"}));
} }
} // end namespace } // end namespace

third-party/benchmark/test/templated_fixture_test.cc

#include "benchmark/benchmark.h"
#include <cassert> #include <cassert>
#include <memory> #include <memory>
#include "benchmark/benchmark.h"
template <typename T> template <typename T>
class MyFixture : public ::benchmark::Fixture { class MyFixture : public ::benchmark::Fixture {
public: public:
MyFixture() : data(0) {} MyFixture() : data(0) {}
T data; T data;
}; };
Show All 14 Lines

third-party/benchmark/test/user_counters_tabular_test.cc

Show All 12 LinesADD_CASES(TC_ConsoleOut,
// clang-format off // clang-format off
{"^[-]+$", MR_Next}, {"^[-]+$", MR_Next},
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next}, {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
{"^[-]+$", MR_Next}, {"^[-]+$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:1_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
{"^BM_Counters_Tabular/repeats:2/threads:2_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
{"^[-]+$", MR_Next}, {"^[-]+$", MR_Next},
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next}, {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
{"^[-]+$", MR_Next}, {"^[-]+$", MR_Next},
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 LinesADD_CASES(TC_JSONOut,
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$", {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bar\": %float,$", MR_Next}, {"\"Bar\": %float,$", MR_Next},
{"\"Bat\": %float,$", MR_Next}, {"\"Bat\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next}, {"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float,$", MR_Next}, {"\"Foo\": %float,$", MR_Next},
{"\"Frob\": %float,$", MR_Next}, {"\"Frob\": %float,$", MR_Next},
{"\"Lob\": %float$", MR_Next}, {"\"Lob\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$"}, {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$", {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bar\": %float,$", MR_Next}, {"\"Bar\": %float,$", MR_Next},
{"\"Bat\": %float,$", MR_Next}, {"\"Bat\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next}, {"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float,$", MR_Next}, {"\"Foo\": %float,$", MR_Next},
{"\"Frob\": %float,$", MR_Next}, {"\"Frob\": %float,$", MR_Next},
{"\"Lob\": %float$", MR_Next}, {"\"Lob\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$"}, {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$", {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bar\": %float,$", MR_Next},
{"\"Bat\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float,$", MR_Next},
{"\"Frob\": %float,$", MR_Next},
{"\"Lob\": %float$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_cv\",$"},
{"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"cv\",$", MR_Next},
{"\"aggregate_unit\": \"percentage\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bar\": %float,$", MR_Next}, {"\"Bar\": %float,$", MR_Next},
{"\"Bat\": %float,$", MR_Next}, {"\"Bat\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next}, {"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float,$", MR_Next}, {"\"Foo\": %float,$", MR_Next},
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 LinesADD_CASES(TC_JSONOut,
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 1,$", MR_Next}, {"\"per_family_instance_index\": 1,$", MR_Next},
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$", {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 2,$", MR_Next}, {"\"threads\": 2,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bar\": %float,$", MR_Next}, {"\"Bar\": %float,$", MR_Next},
{"\"Bat\": %float,$", MR_Next}, {"\"Bat\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next}, {"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float,$", MR_Next}, {"\"Foo\": %float,$", MR_Next},
{"\"Frob\": %float,$", MR_Next}, {"\"Frob\": %float,$", MR_Next},
{"\"Lob\": %float$", MR_Next}, {"\"Lob\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$"}, {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 1,$", MR_Next}, {"\"per_family_instance_index\": 1,$", MR_Next},
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$", {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
MR_Next}, MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 2,$", MR_Next}, {"\"threads\": 2,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bar\": %float,$", MR_Next},
{"\"Bat\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float,$", MR_Next},
{"\"Frob\": %float,$", MR_Next},
{"\"Lob\": %float$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_cv\",$"},
{"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 1,$", MR_Next},
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 2,$", MR_Next},
{"\"aggregate_name\": \"cv\",$", MR_Next},
{"\"aggregate_unit\": \"percentage\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
{"\"Bar\": %float,$", MR_Next}, {"\"Bar\": %float,$", MR_Next},
{"\"Bat\": %float,$", MR_Next}, {"\"Bat\": %float,$", MR_Next},
{"\"Baz\": %float,$", MR_Next}, {"\"Baz\": %float,$", MR_Next},
{"\"Foo\": %float,$", MR_Next}, {"\"Foo\": %float,$", MR_Next},
Show All 11 LinesADD_CASES(TC_CSVOut,
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report," {{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report," {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report," {{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report," {{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report," {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}}); "%float,%float,%float,%float,%float,%float$"}});
ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument // VS2013 does not allow this function to be passed as a lambda argument
// to CHECK_BENCHMARK_RESULTS() // to CHECK_BENCHMARK_RESULTS()
void CheckTabular(Results const& e) { void CheckTabular(Results const& e) {
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1); CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2); CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4); CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8); CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16); CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
▲ Show 20 Lines • Show All 197 Lines • Show Last 20 Lines

third-party/benchmark/test/user_counters_test.cc

Show All 20 Lines
// ========================================================================= // // ========================================================================= //
// ------------------------- Simple Counters Output ------------------------ // // ------------------------- Simple Counters Output ------------------------ //
// ========================================================================= // // ========================================================================= //
void BM_Counters_Simple(benchmark::State& state) { void BM_Counters_Simple(benchmark::State& state) {
for (auto _ : state) { for (auto _ : state) {
} }
state.counters["foo"] = 1; state.counters["foo"] = 1;
state.counters["bar"] = 2 * (double)state.iterations(); state.counters["bar"] = 2 * static_cast<double>(state.iterations());
} }
BENCHMARK(BM_Counters_Simple); BENCHMARK(BM_Counters_Simple);
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
{{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next},
▲ Show 20 Lines • Show All 518 Lines • Show Last 20 Lines

third-party/benchmark/test/user_counters_thousands_test.cc

Show First 20 Lines • Show All 90 Lines • ▼ Show 20 LinesADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
{"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
{"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
{"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
{"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
{"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"},
{"\"family_index\": 0,$", MR_Next}, {"\"family_index\": 0,$", MR_Next},
{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"per_family_instance_index\": 0,$", MR_Next},
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"repetitions\": 2,$", MR_Next}, {"\"repetitions\": 2,$", MR_Next},
{"\"threads\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}, {"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next},
{"\"t0_1000000DefaultBase\": 0\\.(0)*e\\+(0)*,$", MR_Next}, {"\"t0_1000000DefaultBase\": 0\\.(0)*e\\+(0)*,$", MR_Next},
{"\"t1_1000000Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next}, {"\"t1_1000000Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next},
{"\"t2_1000000Base1024\": 0\\.(0)*e\\+(0)*,$", MR_Next}, {"\"t2_1000000Base1024\": 0\\.(0)*e\\+(0)*,$", MR_Next},
{"\"t3_1048576Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next}, {"\"t3_1048576Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next},
Show All 39 Lines

third-party/benchmark/tools/gbench/Inputs/test4_run0.json

  • This file was added.
{
"context": {
"date": "2016-08-02 17:44:46",
"num_cpus": 4,
"mhz_per_cpu": 4228,
"cpu_scaling_enabled": false,
"library_build_type": "release"
},
"benchmarks": [
{
"name": "whocares",
"run_type": "aggregate",
"aggregate_name": "zz",
"aggregate_unit": "percentage",
"iterations": 1000,
"real_time": 0.01,
"cpu_time": 0.10,
"time_unit": "ns"
}
]
}

third-party/benchmark/tools/gbench/Inputs/test4_run1.json

  • This file was added.
{
"context": {
"date": "2016-08-02 17:44:46",
"num_cpus": 4,
"mhz_per_cpu": 4228,
"cpu_scaling_enabled": false,
"library_build_type": "release"
},
"benchmarks": [
{
"name": "whocares",
"run_type": "aggregate",
"aggregate_name": "zz",
"aggregate_unit": "percentage",
"iterations": 1000,
"real_time": 0.005,
"cpu_time": 0.15,
"time_unit": "ns"
}
]
}

third-party/benchmark/tools/gbench/report.py

"""report.py - Utilities for reporting statistics about benchmark results """report.py - Utilities for reporting statistics about benchmark results
""" """
import unittest import unittest
import os import os
import re import re
import copy import copy
import random import random
from scipy.stats import mannwhitneyu from scipy.stats import mannwhitneyu, gmean
from numpy import array
from pandas import Timedelta
class BenchmarkColor(object): class BenchmarkColor(object):
def __init__(self, name, code): def __init__(self, name, code):
self.name = name self.name = name
self.code = code self.code = code
def __repr__(self): def __repr__(self):
▲ Show 20 Lines • Show All 126 Lines • ▼ Show 20 Lines for name in names:
lhs = [x for x in json1['benchmarks'] if x['name'] == name and lhs = [x for x in json1['benchmarks'] if x['name'] == name and
x['time_unit'] == time_unit] x['time_unit'] == time_unit]
rhs = [x for x in json2['benchmarks'] if x['name'] == name and rhs = [x for x in json2['benchmarks'] if x['name'] == name and
x['time_unit'] == time_unit] x['time_unit'] == time_unit]
partitions.append([lhs, rhs]) partitions.append([lhs, rhs])
return partitions return partitions
def get_timedelta_field_as_seconds(benchmark, field_name):
"""
Get value of field_name field of benchmark, which is time with time unit
time_unit, as time in seconds.
"""
time_unit = benchmark['time_unit'] if 'time_unit' in benchmark else 's'
dt = Timedelta(benchmark[field_name], time_unit)
return dt / Timedelta(1, 's')
def calculate_geomean(json):
"""
Extract all real/cpu times from all the benchmarks as seconds,
and calculate their geomean.
"""
times = []
for benchmark in json['benchmarks']:
if 'run_type' in benchmark and benchmark['run_type'] == 'aggregate':
continue
times.append([get_timedelta_field_as_seconds(benchmark, 'real_time'),
get_timedelta_field_as_seconds(benchmark, 'cpu_time')])
return gmean(times) if times else array([])
def extract_field(partition, field_name): def extract_field(partition, field_name):
# The count of elements may be different. We want *all* of them. # The count of elements may be different. We want *all* of them.
lhs = [x[field_name] for x in partition[0]] lhs = [x[field_name] for x in partition[0]]
rhs = [x[field_name] for x in partition[1]] rhs = [x[field_name] for x in partition[1]]
return [lhs, rhs] return [lhs, rhs]
def calc_utest(timings_cpu, timings_time): def calc_utest(timings_cpu, timings_time):
min_rep_cnt = min(len(timings_time[0]), min_rep_cnt = min(len(timings_time[0]),
len(timings_time[1]), len(timings_time[1]),
len(timings_cpu[0]), len(timings_cpu[0]),
len(timings_cpu[1])) len(timings_cpu[1]))
# Does *everything* has at least UTEST_MIN_REPETITIONS repetitions? # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions?
if min_rep_cnt < UTEST_MIN_REPETITIONS: if min_rep_cnt < UTEST_MIN_REPETITIONS:
return False, None, None return False, None, None
time_pvalue = mannwhitneyu( time_pvalue = mannwhitneyu(
timings_time[0], timings_time[1], alternative='two-sided').pvalue timings_time[0], timings_time[1], alternative='two-sided').pvalue
cpu_pvalue = mannwhitneyu( cpu_pvalue = mannwhitneyu(
timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue
return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue
def print_utest(bc_name, utest, utest_alpha, first_col_width, use_color=True): def print_utest(bc_name, utest, utest_alpha, first_col_width, use_color=True):
def get_utest_color(pval): def get_utest_color(pval):
return BC_FAIL if pval >= utest_alpha else BC_OKGREEN return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
# Check if we failed miserably with minimum required repetitions for utest # Check if we failed miserably with minimum required repetitions for utest
if not utest['have_optimal_repetitions'] and utest['cpu_pvalue'] is None and utest['time_pvalue'] is None: if not utest['have_optimal_repetitions'] and utest['cpu_pvalue'] is None and utest['time_pvalue'] is None:
return [] return []
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines for partition in partitions:
'time': calculate_change(bn['real_time'], other_bench['real_time']), 'time': calculate_change(bn['real_time'], other_bench['real_time']),
'cpu': calculate_change(bn['cpu_time'], other_bench['cpu_time']) 'cpu': calculate_change(bn['cpu_time'], other_bench['cpu_time'])
}) })
# After processing the whole partition, if requested, do the U test. # After processing the whole partition, if requested, do the U test.
if utest: if utest:
timings_cpu = extract_field(partition, 'cpu_time') timings_cpu = extract_field(partition, 'cpu_time')
timings_time = extract_field(partition, 'real_time') timings_time = extract_field(partition, 'real_time')
have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(timings_cpu, timings_time) have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(
timings_cpu, timings_time)
if cpu_pvalue and time_pvalue: if cpu_pvalue and time_pvalue:
utest_results = { utest_results = {
'have_optimal_repetitions': have_optimal_repetitions, 'have_optimal_repetitions': have_optimal_repetitions,
'cpu_pvalue': cpu_pvalue, 'cpu_pvalue': cpu_pvalue,
'time_pvalue': time_pvalue, 'time_pvalue': time_pvalue,
'nr_of_repetitions': len(timings_cpu[0]), 'nr_of_repetitions': len(timings_cpu[0]),
'nr_of_repetitions_other': len(timings_cpu[1]) 'nr_of_repetitions_other': len(timings_cpu[1])
} }
Show All 9 Lines for partition in partitions:
'name': benchmark_name, 'name': benchmark_name,
'measurements': measurements, 'measurements': measurements,
'time_unit': time_unit, 'time_unit': time_unit,
'run_type': run_type, 'run_type': run_type,
'aggregate_name': aggregate_name, 'aggregate_name': aggregate_name,
'utest': utest_results 'utest': utest_results
}) })
lhs_gmean = calculate_geomean(json1)
rhs_gmean = calculate_geomean(json2)
if lhs_gmean.any() and rhs_gmean.any():
diff_report.append({
'name': 'OVERALL_GEOMEAN',
'measurements': [{
'real_time': lhs_gmean[0],
'cpu_time': lhs_gmean[1],
'real_time_other': rhs_gmean[0],
'cpu_time_other': rhs_gmean[1],
'time': calculate_change(lhs_gmean[0], rhs_gmean[0]),
'cpu': calculate_change(lhs_gmean[1], rhs_gmean[1])
}],
'time_unit': 's',
'run_type': 'aggregate',
'aggregate_name': 'geomean',
'utest': {}
})
return diff_report return diff_report
def print_difference_report( def print_difference_report(
json_diff_report, json_diff_report,
include_aggregates_only=False, include_aggregates_only=False,
utest=False, utest=False,
utest_alpha=0.05, utest_alpha=0.05,
Show All 23 Linesdef print_difference_report(
fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}"
for benchmark in json_diff_report: for benchmark in json_diff_report:
# *If* we were asked to only include aggregates, # *If* we were asked to only include aggregates,
# and if it is non-aggregate, then don't print it. # and if it is non-aggregate, then don't print it.
if not include_aggregates_only or not 'run_type' in benchmark or benchmark['run_type'] == 'aggregate': if not include_aggregates_only or not 'run_type' in benchmark or benchmark['run_type'] == 'aggregate':
for measurement in benchmark['measurements']: for measurement in benchmark['measurements']:
output_strs += [color_format(use_color, output_strs += [color_format(use_color,
fmt_str, fmt_str,
BC_HEADER, BC_HEADER,
benchmark['name'], benchmark['name'],
first_col_width, first_col_width,
get_color(measurement['time']), get_color(measurement['time']),
measurement['time'], measurement['time'],
get_color(measurement['cpu']), get_color(measurement['cpu']),
measurement['cpu'], measurement['cpu'],
measurement['real_time'], measurement['real_time'],
measurement['real_time_other'], measurement['real_time_other'],
measurement['cpu_time'], measurement['cpu_time'],
measurement['cpu_time_other'], measurement['cpu_time_other'],
endc=BC_ENDC)] endc=BC_ENDC)]
# After processing the measurements, if requested and # After processing the measurements, if requested and
# if applicable (e.g. u-test exists for given benchmark), # if applicable (e.g. u-test exists for given benchmark),
# print the U test. # print the U test.
if utest and benchmark['utest']: if utest and benchmark['utest']:
output_strs += print_utest(benchmark['name'], output_strs += print_utest(benchmark['name'],
benchmark['utest'], benchmark['utest'],
utest_alpha=utest_alpha, utest_alpha=utest_alpha,
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines def test_json_diff_report_pretty_printing(self):
['BM_100xSlower', '+99.0000', '+99.0000', ['BM_100xSlower', '+99.0000', '+99.0000',
'100', '10000', '100', '10000'], '100', '10000', '100', '10000'],
['BM_100xFaster', '-0.9900', '-0.9900', ['BM_100xFaster', '-0.9900', '-0.9900',
'10000', '100', '10000', '100'], '10000', '100', '10000', '100'],
['BM_10PercentCPUToTime', '+0.1000', ['BM_10PercentCPUToTime', '+0.1000',
'-0.1000', '100', '110', '100', '90'], '-0.1000', '100', '110', '100', '90'],
['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'], ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'],
['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], ['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'],
['OVERALL_GEOMEAN', '-0.8344', '-0.8026', '0', '0', '0', '0']
] ]
output_lines_with_header = print_difference_report( output_lines_with_header = print_difference_report(
self.json_diff_report, use_color=False) self.json_diff_report, use_color=False)
output_lines = output_lines_with_header[2:] output_lines = output_lines_with_header[2:]
print("\n") print("\n")
print("\n".join(output_lines_with_header)) print("\n".join(output_lines_with_header))
self.assertEqual(len(output_lines), len(expect_lines)) self.assertEqual(len(output_lines), len(expect_lines))
for i in range(0, len(output_lines)): for i in range(0, len(output_lines)):
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines def test_json_diff_report_output(self):
'utest': {} 'utest': {}
}, },
{ {
'name': 'BM_NotBadTimeUnit', 'name': 'BM_NotBadTimeUnit',
'measurements': [{'time': -0.9000, 'cpu': 0.2000, 'real_time': 0.4, 'real_time_other': 0.04, 'cpu_time': 0.5, 'cpu_time_other': 0.6}], 'measurements': [{'time': -0.9000, 'cpu': 0.2000, 'real_time': 0.4, 'real_time_other': 0.04, 'cpu_time': 0.5, 'cpu_time_other': 0.6}],
'time_unit': 's', 'time_unit': 's',
'utest': {} 'utest': {}
}, },
{
'name': 'OVERALL_GEOMEAN',
'measurements': [{'real_time': 1.193776641714438e-06, 'cpu_time': 1.2144445585302297e-06,
'real_time_other': 1.9768988699420897e-07, 'cpu_time_other': 2.397447755209533e-07,
'time': -0.834399601997324, 'cpu': -0.8025889499549471}],
'time_unit': 's',
'run_type': 'aggregate',
'aggregate_name': 'geomean', 'utest': {}
},
] ]
self.assertEqual(len(self.json_diff_report), len(expected_output)) self.assertEqual(len(self.json_diff_report), len(expected_output))
for out, expected in zip( for out, expected in zip(
self.json_diff_report, expected_output): self.json_diff_report, expected_output):
self.assertEqual(out['name'], expected['name']) self.assertEqual(out['name'], expected['name'])
self.assertEqual(out['time_unit'], expected['time_unit']) self.assertEqual(out['time_unit'], expected['time_unit'])
assert_utest(self, out, expected) assert_utest(self, out, expected)
assert_measurements(self, out, expected) assert_measurements(self, out, expected)
Show All 19 Lines def setUpClass(cls):
cls.json_diff_report = get_difference_report(json1, json2) cls.json_diff_report = get_difference_report(json1, json2)
def test_json_diff_report_pretty_printing(self): def test_json_diff_report_pretty_printing(self):
expect_lines = [ expect_lines = [
['.', '-0.5000', '-0.5000', '10', '5', '10', '5'], ['.', '-0.5000', '-0.5000', '10', '5', '10', '5'],
['./4', '-0.5000', '-0.5000', '40', '20', '40', '20'], ['./4', '-0.5000', '-0.5000', '40', '20', '40', '20'],
['Prefix/.', '-0.5000', '-0.5000', '20', '10', '20', '10'], ['Prefix/.', '-0.5000', '-0.5000', '20', '10', '20', '10'],
['Prefix/./3', '-0.5000', '-0.5000', '30', '15', '30', '15'], ['Prefix/./3', '-0.5000', '-0.5000', '30', '15', '30', '15'],
['OVERALL_GEOMEAN', '-0.5000', '-0.5000', '0', '0', '0', '0']
] ]
output_lines_with_header = print_difference_report( output_lines_with_header = print_difference_report(
self.json_diff_report, use_color=False) self.json_diff_report, use_color=False)
output_lines = output_lines_with_header[2:] output_lines = output_lines_with_header[2:]
print("\n") print("\n")
print("\n".join(output_lines_with_header)) print("\n".join(output_lines_with_header))
self.assertEqual(len(output_lines), len(expect_lines)) self.assertEqual(len(output_lines), len(expect_lines))
for i in range(0, len(output_lines)): for i in range(0, len(output_lines)):
Show All 21 Lines def test_json_diff_report(self):
'time_unit': 'ns', 'time_unit': 'ns',
'utest': {} 'utest': {}
}, },
{ {
'name': u'Prefix/./3', 'name': u'Prefix/./3',
'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 30, 'real_time_other': 15, 'cpu_time': 30, 'cpu_time_other': 15}], 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 30, 'real_time_other': 15, 'cpu_time': 30, 'cpu_time_other': 15}],
'time_unit': 'ns', 'time_unit': 'ns',
'utest': {} 'utest': {}
},
{
'name': 'OVERALL_GEOMEAN',
'measurements': [{'real_time': 2.213363839400641e-08, 'cpu_time': 2.213363839400641e-08,
'real_time_other': 1.1066819197003185e-08, 'cpu_time_other': 1.1066819197003185e-08,
'time': -0.5000000000000009, 'cpu': -0.5000000000000009}],
'time_unit': 's',
'run_type': 'aggregate',
'aggregate_name': 'geomean',
'utest': {}
} }
] ]
self.assertEqual(len(self.json_diff_report), len(expected_output)) self.assertEqual(len(self.json_diff_report), len(expected_output))
for out, expected in zip( for out, expected in zip(
self.json_diff_report, expected_output): self.json_diff_report, expected_output):
self.assertEqual(out['name'], expected['name']) self.assertEqual(out['name'], expected['name'])
self.assertEqual(out['time_unit'], expected['time_unit']) self.assertEqual(out['time_unit'], expected['time_unit'])
assert_utest(self, out, expected) assert_utest(self, out, expected)
Show All 22 Lines def setUpClass(cls):
json1, json2, utest=True) json1, json2, utest=True)
def test_json_diff_report_pretty_printing(self): def test_json_diff_report_pretty_printing(self):
expect_lines = [ expect_lines = [
['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'],
['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'], ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'],
['BM_Two_pvalue', ['BM_Two_pvalue',
'0.6985', '1.0000',
'0.6985', '0.6667',
'U', 'U',
'Test,', 'Test,',
'Repetitions:', 'Repetitions:',
'2', '2',
'vs', 'vs',
'2.', '2.',
'WARNING:', 'WARNING:',
'Results', 'Results',
'unreliable!', 'unreliable!',
'9+', '9+',
'repetitions', 'repetitions',
'recommended.'], 'recommended.'],
['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
['short_pvalue', ['short_pvalue',
'0.7671', '0.7671',
'0.1489', '0.2000',
'U', 'U',
'Test,', 'Test,',
'Repetitions:', 'Repetitions:',
'2', '2',
'vs', 'vs',
'3.', '3.',
'WARNING:', 'WARNING:',
'Results', 'Results',
'unreliable!', 'unreliable!',
'9+', '9+',
'repetitions', 'repetitions',
'recommended.'], 'recommended.'],
['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'], ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'],
['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0']
] ]
output_lines_with_header = print_difference_report( output_lines_with_header = print_difference_report(
self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False) self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False)
output_lines = output_lines_with_header[2:] output_lines = output_lines_with_header[2:]
print("\n") print("\n")
print("\n".join(output_lines_with_header)) print("\n".join(output_lines_with_header))
self.assertEqual(len(output_lines), len(expect_lines)) self.assertEqual(len(output_lines), len(expect_lines))
for i in range(0, len(output_lines)): for i in range(0, len(output_lines)):
parts = [x for x in output_lines[i].split(' ') if x] parts = [x for x in output_lines[i].split(' ') if x]
self.assertEqual(expect_lines[i], parts) self.assertEqual(expect_lines[i], parts)
def test_json_diff_report_pretty_printing_aggregates_only(self): def test_json_diff_report_pretty_printing_aggregates_only(self):
expect_lines = [ expect_lines = [
['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
['BM_Two_pvalue', ['BM_Two_pvalue',
'0.6985', '1.0000',
'0.6985', '0.6667',
'U', 'U',
'Test,', 'Test,',
'Repetitions:', 'Repetitions:',
'2', '2',
'vs', 'vs',
'2.', '2.',
'WARNING:', 'WARNING:',
'Results', 'Results',
'unreliable!', 'unreliable!',
'9+', '9+',
'repetitions', 'repetitions',
'recommended.'], 'recommended.'],
['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
['short_pvalue', ['short_pvalue',
'0.7671', '0.7671',
'0.1489', '0.2000',
'U', 'U',
'Test,', 'Test,',
'Repetitions:', 'Repetitions:',
'2', '2',
'vs', 'vs',
'3.', '3.',
'WARNING:', 'WARNING:',
'Results', 'Results',
'unreliable!', 'unreliable!',
'9+', '9+',
'repetitions', 'repetitions',
'recommended.'], 'recommended.'],
['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0']
] ]
output_lines_with_header = print_difference_report( output_lines_with_header = print_difference_report(
self.json_diff_report, include_aggregates_only=True, utest=True, utest_alpha=0.05, use_color=False) self.json_diff_report, include_aggregates_only=True, utest=True, utest_alpha=0.05, use_color=False)
output_lines = output_lines_with_header[2:] output_lines = output_lines_with_header[2:]
print("\n") print("\n")
print("\n".join(output_lines_with_header)) print("\n".join(output_lines_with_header))
self.assertEqual(len(output_lines), len(expect_lines)) self.assertEqual(len(output_lines), len(expect_lines))
for i in range(0, len(output_lines)): for i in range(0, len(output_lines)):
Show All 24 Lines def test_json_diff_report(self):
'real_time_other': 10, 'real_time_other': 10,
'cpu_time': 90, 'cpu_time': 90,
'cpu_time_other': 89}, 'cpu_time_other': 89},
{'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8, {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8,
'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72} 'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72}
], ],
'time_unit': 'ns', 'time_unit': 'ns',
'utest': { 'utest': {
'have_optimal_repetitions': False, 'cpu_pvalue': 0.6985353583033387, 'time_pvalue': 0.6985353583033387 'have_optimal_repetitions': False, 'cpu_pvalue': 0.6666666666666666, 'time_pvalue': 1.0
} }
}, },
{ {
'name': u'short', 'name': u'short',
'measurements': [ 'measurements': [
{'time': -0.125, {'time': -0.125,
'cpu': -0.0625, 'cpu': -0.0625,
'real_time': 8, 'real_time': 8,
'real_time_other': 7, 'real_time_other': 7,
'cpu_time': 80, 'cpu_time': 80,
'cpu_time_other': 75}, 'cpu_time_other': 75},
{'time': -0.4325, {'time': -0.4325,
'cpu': -0.13506493506493514, 'cpu': -0.13506493506493514,
'real_time': 8, 'real_time': 8,
'real_time_other': 4.54, 'real_time_other': 4.54,
'cpu_time': 77, 'cpu_time': 77,
'cpu_time_other': 66.6} 'cpu_time_other': 66.6}
], ],
'time_unit': 'ns', 'time_unit': 'ns',
'utest': { 'utest': {
'have_optimal_repetitions': False, 'cpu_pvalue': 0.14891467317876572, 'time_pvalue': 0.7670968684102772 'have_optimal_repetitions': False, 'cpu_pvalue': 0.2, 'time_pvalue': 0.7670968684102772
} }
}, },
{ {
'name': u'medium', 'name': u'medium',
'measurements': [ 'measurements': [
{'time': -0.375, {'time': -0.375,
'cpu': -0.3375, 'cpu': -0.3375,
'real_time': 8, 'real_time': 8,
'real_time_other': 5, 'real_time_other': 5,
'cpu_time': 80, 'cpu_time': 80,
'cpu_time_other': 53} 'cpu_time_other': 53}
], ],
'time_unit': 'ns', 'time_unit': 'ns',
'utest': {} 'utest': {}
},
{
'name': 'OVERALL_GEOMEAN',
'measurements': [{'real_time': 8.48528137423858e-09, 'cpu_time': 8.441336246629233e-08,
'real_time_other': 2.2405267593145244e-08, 'cpu_time_other': 2.5453661413660466e-08,
'time': 1.6404861082353634, 'cpu': -0.6984640740519662}],
'time_unit': 's',
'run_type': 'aggregate',
'aggregate_name': 'geomean',
'utest': {}
} }
] ]
self.assertEqual(len(self.json_diff_report), len(expected_output)) self.assertEqual(len(self.json_diff_report), len(expected_output))
for out, expected in zip( for out, expected in zip(
self.json_diff_report, expected_output): self.json_diff_report, expected_output):
self.assertEqual(out['name'], expected['name']) self.assertEqual(out['name'], expected['name'])
self.assertEqual(out['time_unit'], expected['time_unit']) self.assertEqual(out['time_unit'], expected['time_unit'])
assert_utest(self, out, expected) assert_utest(self, out, expected)
Show All 23 Lines def setUpClass(cls):
json1, json2, utest=True) json1, json2, utest=True)
def test_json_diff_report_pretty_printing(self): def test_json_diff_report_pretty_printing(self):
expect_lines = [ expect_lines = [
['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'],
['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'], ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'],
['BM_Two_pvalue', ['BM_Two_pvalue',
'0.6985', '1.0000',
'0.6985', '0.6667',
'U', 'U',
'Test,', 'Test,',
'Repetitions:', 'Repetitions:',
'2', '2',
'vs', 'vs',
'2.', '2.',
'WARNING:', 'WARNING:',
'Results', 'Results',
'unreliable!', 'unreliable!',
'9+', '9+',
'repetitions', 'repetitions',
'recommended.'], 'recommended.'],
['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
['short_pvalue', ['short_pvalue',
'0.7671', '0.7671',
'0.1489', '0.2000',
'U', 'U',
'Test,', 'Test,',
'Repetitions:', 'Repetitions:',
'2', '2',
'vs', 'vs',
'3.', '3.',
'WARNING:', 'WARNING:',
'Results', 'Results',
'unreliable!', 'unreliable!',
'9+', '9+',
'repetitions', 'repetitions',
'recommended.'], 'recommended.'],
['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'] ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'],
['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0']
] ]
output_lines_with_header = print_difference_report( output_lines_with_header = print_difference_report(
self.json_diff_report, self.json_diff_report,
utest=True, utest_alpha=0.05, use_color=False) utest=True, utest_alpha=0.05, use_color=False)
output_lines = output_lines_with_header[2:] output_lines = output_lines_with_header[2:]
print("\n") print("\n")
print("\n".join(output_lines_with_header)) print("\n".join(output_lines_with_header))
self.assertEqual(len(output_lines), len(expect_lines)) self.assertEqual(len(output_lines), len(expect_lines))
Show All 25 Lines def test_json_diff_report(self):
'real_time_other': 10, 'real_time_other': 10,
'cpu_time': 90, 'cpu_time': 90,
'cpu_time_other': 89}, 'cpu_time_other': 89},
{'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8, {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8,
'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72} 'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72}
], ],
'time_unit': 'ns', 'time_unit': 'ns',
'utest': { 'utest': {
'have_optimal_repetitions': False, 'cpu_pvalue': 0.6985353583033387, 'time_pvalue': 0.6985353583033387 'have_optimal_repetitions': False, 'cpu_pvalue': 0.6666666666666666, 'time_pvalue': 1.0
} }
}, },
{ {
'name': u'short', 'name': u'short',
'measurements': [ 'measurements': [
{'time': -0.125, {'time': -0.125,
'cpu': -0.0625, 'cpu': -0.0625,
'real_time': 8, 'real_time': 8,
'real_time_other': 7, 'real_time_other': 7,
'cpu_time': 80, 'cpu_time': 80,
'cpu_time_other': 75}, 'cpu_time_other': 75},
{'time': -0.4325, {'time': -0.4325,
'cpu': -0.13506493506493514, 'cpu': -0.13506493506493514,
'real_time': 8, 'real_time': 8,
'real_time_other': 4.54, 'real_time_other': 4.54,
'cpu_time': 77, 'cpu_time': 77,
'cpu_time_other': 66.6} 'cpu_time_other': 66.6}
], ],
'time_unit': 'ns', 'time_unit': 'ns',
'utest': { 'utest': {
'have_optimal_repetitions': False, 'cpu_pvalue': 0.14891467317876572, 'time_pvalue': 0.7670968684102772 'have_optimal_repetitions': False, 'cpu_pvalue': 0.2, 'time_pvalue': 0.7670968684102772
} }
}, },
{ {
'name': u'medium', 'name': u'medium',
'measurements': [ 'measurements': [
{'real_time_other': 5, {'real_time_other': 5,
'cpu_time': 80, 'cpu_time': 80,
'time': -0.375, 'time': -0.375,
'real_time': 8, 'real_time': 8,
'cpu_time_other': 53, 'cpu_time_other': 53,
'cpu': -0.3375 'cpu': -0.3375
} }
], ],
'utest': {}, 'utest': {},
'time_unit': u'ns', 'time_unit': u'ns',
'aggregate_name': '' 'aggregate_name': ''
},
{
'name': 'OVERALL_GEOMEAN',
'measurements': [{'real_time': 8.48528137423858e-09, 'cpu_time': 8.441336246629233e-08,
'real_time_other': 2.2405267593145244e-08, 'cpu_time_other': 2.5453661413660466e-08,
'time': 1.6404861082353634, 'cpu': -0.6984640740519662}],
'time_unit': 's',
'run_type': 'aggregate',
'aggregate_name': 'geomean',
'utest': {}
}
]
self.assertEqual(len(self.json_diff_report), len(expected_output))
for out, expected in zip(
self.json_diff_report, expected_output):
self.assertEqual(out['name'], expected['name'])
self.assertEqual(out['time_unit'], expected['time_unit'])
assert_utest(self, out, expected)
assert_measurements(self, out, expected)
class TestReportDifferenceForPercentageAggregates(
unittest.TestCase):
@classmethod
def setUpClass(cls):
def load_results():
import json
testInputs = os.path.join(
os.path.dirname(
os.path.realpath(__file__)),
'Inputs')
testOutput1 = os.path.join(testInputs, 'test4_run0.json')
testOutput2 = os.path.join(testInputs, 'test4_run1.json')
with open(testOutput1, 'r') as f:
json1 = json.load(f)
with open(testOutput2, 'r') as f:
json2 = json.load(f)
return json1, json2
json1, json2 = load_results()
cls.json_diff_report = get_difference_report(
json1, json2, utest=True)
def test_json_diff_report_pretty_printing(self):
expect_lines = [
['whocares', '-0.5000', '+0.5000', '0', '0', '0', '0']
]
output_lines_with_header = print_difference_report(
self.json_diff_report,
utest=True, utest_alpha=0.05, use_color=False)
output_lines = output_lines_with_header[2:]
print("\n")
print("\n".join(output_lines_with_header))
self.assertEqual(len(output_lines), len(expect_lines))
for i in range(0, len(output_lines)):
parts = [x for x in output_lines[i].split(' ') if x]
self.assertEqual(expect_lines[i], parts)
def test_json_diff_report(self):
expected_output = [
{
'name': u'whocares',
'measurements': [
{'time': -0.5,
'cpu': 0.5,
'real_time': 0.01,
'real_time_other': 0.005,
'cpu_time': 0.10,
'cpu_time_other': 0.15}
],
'time_unit': 'ns',
'utest': {}
} }
] ]
self.assertEqual(len(self.json_diff_report), len(expected_output)) self.assertEqual(len(self.json_diff_report), len(expected_output))
for out, expected in zip( for out, expected in zip(
self.json_diff_report, expected_output): self.json_diff_report, expected_output):
self.assertEqual(out['name'], expected['name']) self.assertEqual(out['name'], expected['name'])
self.assertEqual(out['time_unit'], expected['time_unit']) self.assertEqual(out['time_unit'], expected['time_unit'])
assert_utest(self, out, expected) assert_utest(self, out, expected)
▲ Show 20 Lines • Show All 78 Lines • Show Last 20 Lines

third-party/update_benchmark.sh

  • This file was added.
PropertyOld ValueNew Value
File Modenull100755
#!/bin/bash
echo "This script deletes `benchmark`, clones it from github, together"
echo "with its dependencies. It then removes .git* files and dirs."
echo "NOTE!!!"
echo "Please double-check the benchmark github wiki for any changes"
echo "to dependencies. Currently, these are limited to googletest."
echo
read -p "Press a key to continue, or Ctrl+C to cancel"
rm -rf benchmark
git clone https://github.com/google/benchmark.git
rm -rf benchmark/.git*