diff --git a/libc/benchmarks/LibcMemoryBenchmark.h b/libc/benchmarks/LibcMemoryBenchmark.h --- a/libc/benchmarks/LibcMemoryBenchmark.h +++ b/libc/benchmarks/LibcMemoryBenchmark.h @@ -185,19 +185,43 @@ std::vector Parameters; }; +/// Memory function prototype and configuration. +using MemcpyFunction = void *(*)(void *__restrict, const void *__restrict, + size_t); +struct MemcpyConfiguration { + MemcpyFunction Function; + llvm::StringRef Name; +}; + +using MemsetFunction = void *(*)(void *, int, size_t); +struct MemsetConfiguration { + MemsetFunction Function; + llvm::StringRef Name; +}; + +using BzeroFunction = void (*)(void *, size_t); +struct BzeroConfiguration { + BzeroFunction Function; + llvm::StringRef Name; +}; + +using MemcmpFunction = int (*)(const void *, const void *, size_t); +struct MemcmpConfiguration { + MemcmpFunction Function; + llvm::StringRef Name; +}; + /// Provides source and destination buffers for the Copy operation as well as /// the associated size distributions. -struct CopyHarness : public ParameterBatch { - CopyHarness(); +struct CopySetup : public ParameterBatch { + CopySetup(); inline static const ArrayRef getDistributions() { return getMemcpySizeDistributions(); } - inline void *Call(ParameterType Parameter, - void *(*memcpy)(void *__restrict, const void *__restrict, - size_t)) { - return memcpy(DstBuffer + Parameter.OffsetBytes, + inline void *Call(ParameterType Parameter, MemcpyFunction Memcpy) { + return Memcpy(DstBuffer + Parameter.OffsetBytes, SrcBuffer + Parameter.OffsetBytes, Parameter.SizeBytes); } @@ -208,21 +232,20 @@ /// Provides destination buffer for the Set operation as well as the associated /// size distributions. -struct SetHarness : public ParameterBatch { - SetHarness(); +struct SetSetup : public ParameterBatch { + SetSetup(); inline static const ArrayRef getDistributions() { return getMemsetSizeDistributions(); } - inline void *Call(ParameterType Parameter, - void *(*memset)(void *, int, size_t)) { - return memset(DstBuffer + Parameter.OffsetBytes, + inline void *Call(ParameterType Parameter, MemsetFunction Memset) { + return Memset(DstBuffer + Parameter.OffsetBytes, Parameter.OffsetBytes % 0xFF, Parameter.SizeBytes); } - inline void *Call(ParameterType Parameter, void (*bzero)(void *, size_t)) { - bzero(DstBuffer + Parameter.OffsetBytes, Parameter.SizeBytes); + inline void *Call(ParameterType Parameter, BzeroFunction Bzero) { + Bzero(DstBuffer + Parameter.OffsetBytes, Parameter.SizeBytes); return DstBuffer.begin(); } @@ -232,16 +255,15 @@ /// Provides left and right buffers for the Comparison operation as well as the /// associated size distributions. -struct ComparisonHarness : public ParameterBatch { - ComparisonHarness(); +struct ComparisonSetup : public ParameterBatch { + ComparisonSetup(); inline static const ArrayRef getDistributions() { return getMemcmpSizeDistributions(); } - inline int Call(ParameterType Parameter, - int (*memcmp)(const void *, const void *, size_t)) { - return memcmp(LhsBuffer + Parameter.OffsetBytes, + inline int Call(ParameterType Parameter, MemcmpFunction Memcmp) { + return Memcmp(LhsBuffer + Parameter.OffsetBytes, RhsBuffer + Parameter.OffsetBytes, Parameter.SizeBytes); } diff --git a/libc/benchmarks/LibcMemoryBenchmark.cpp b/libc/benchmarks/LibcMemoryBenchmark.cpp --- a/libc/benchmarks/LibcMemoryBenchmark.cpp +++ b/libc/benchmarks/LibcMemoryBenchmark.cpp @@ -104,11 +104,11 @@ .concat(llvm::Twine(BufferSize))); } -CopyHarness::CopyHarness() +CopySetup::CopySetup() : ParameterBatch(2), SrcBuffer(ParameterBatch::BufferSize), DstBuffer(ParameterBatch::BufferSize) {} -ComparisonHarness::ComparisonHarness() +ComparisonSetup::ComparisonSetup() : ParameterBatch(2), LhsBuffer(ParameterBatch::BufferSize), RhsBuffer(ParameterBatch::BufferSize) { // The memcmp buffers always compare equal. @@ -116,7 +116,7 @@ memset(RhsBuffer.begin(), 0xF, BufferSize); } -SetHarness::SetHarness() +SetSetup::SetSetup() : ParameterBatch(1), DstBuffer(ParameterBatch::BufferSize) {} } // namespace libc_benchmarks diff --git a/libc/benchmarks/LibcMemoryGoogleBenchmarkMain.cpp b/libc/benchmarks/LibcMemoryGoogleBenchmarkMain.cpp --- a/libc/benchmarks/LibcMemoryGoogleBenchmarkMain.cpp +++ b/libc/benchmarks/LibcMemoryGoogleBenchmarkMain.cpp @@ -2,10 +2,26 @@ #include "LibcMemoryBenchmark.h" #include "MemorySizeDistributions.h" #include "benchmark/benchmark.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Twine.h" +#include #include #include #include +using llvm::Align; +using llvm::ArrayRef; +using llvm::Twine; +using llvm::libc_benchmarks::BzeroConfiguration; +using llvm::libc_benchmarks::ComparisonSetup; +using llvm::libc_benchmarks::CopySetup; +using llvm::libc_benchmarks::MemcmpConfiguration; +using llvm::libc_benchmarks::MemcpyConfiguration; +using llvm::libc_benchmarks::MemorySizeDistribution; +using llvm::libc_benchmarks::MemsetConfiguration; +using llvm::libc_benchmarks::OffsetDistribution; +using llvm::libc_benchmarks::SetSetup; + namespace __llvm_libc { extern void *memcpy(void *__restrict, const void *__restrict, size_t); @@ -15,84 +31,92 @@ } // namespace __llvm_libc -using llvm::Align; -using llvm::ArrayRef; -using llvm::libc_benchmarks::ComparisonHarness; -using llvm::libc_benchmarks::CopyHarness; -using llvm::libc_benchmarks::MemorySizeDistribution; -using llvm::libc_benchmarks::OffsetDistribution; -using llvm::libc_benchmarks::SetHarness; +// List of implementations to test. +static constexpr MemcpyConfiguration kMemcpyConfigurations[] = { + {__llvm_libc::memcpy, "__llvm_libc::memcpy"}}; + +static constexpr MemcmpConfiguration kMemcmpConfigurations[] = { + {__llvm_libc::memcmp, "__llvm_libc::memcmp"}}; + +static constexpr MemsetConfiguration kMemsetConfigurations[] = { + {__llvm_libc::memset, "__llvm_libc::memset"}}; + +static constexpr BzeroConfiguration kBzeroConfigurations[] = { + {__llvm_libc::bzero, "__llvm_libc::bzero"}}; +// Alignment to use for when accessing the buffers. static constexpr Align kBenchmarkAlignment = Align::Constant<1>(); -template struct Randomized : public Harness { - Randomized(benchmark::State &State) - : State(State), Distribution(Harness::getDistributions()[State.range(0)]), +static std::mt19937_64 &getGenerator() { + static std::mt19937_64 Generator( + std::chrono::system_clock::now().time_since_epoch().count()); + return Generator; +} + +template struct Runner { + Runner(benchmark::State &S, llvm::ArrayRef Configurations) + : State(S), Distribution(SetupType::getDistributions()[State.range(0)]), Probabilities(Distribution.Probabilities), SizeSampler(Probabilities.begin(), Probabilities.end()), - OffsetSampler(Harness::BufferSize, Probabilities.size() - 1, - kBenchmarkAlignment) { - for (auto &P : Harness::Parameters) { - P.OffsetBytes = OffsetSampler(Gen); - P.SizeBytes = SizeSampler(Gen); - Harness::checkValid(P); + OffsetSampler(Setup.BufferSize, Probabilities.size() - 1, + kBenchmarkAlignment), + Configuration(Configurations[State.range(1)]) { + for (auto &P : Setup.Parameters) { + P.OffsetBytes = OffsetSampler(getGenerator()); + P.SizeBytes = SizeSampler(getGenerator()); + Setup.checkValid(P); } } - ~Randomized() { - const size_t AvgBytesPerIteration = - Harness::getBatchBytes() / Harness::BatchSize; + ~Runner() { + const size_t AvgBytesPerIteration = Setup.getBatchBytes() / Setup.BatchSize; const size_t TotalBytes = State.iterations() * AvgBytesPerIteration; State.SetBytesProcessed(TotalBytes); - State.SetLabel(Distribution.Name.str()); + State.SetItemsProcessed(State.iterations()); + State.SetLabel((Twine(Configuration.Name) + "," + Distribution.Name).str()); State.counters["bytes_per_cycle"] = benchmark::Counter( TotalBytes / benchmark::CPUInfo::Get().cycles_per_second, benchmark::Counter::kIsRate); } - template inline void runBatch(Function foo) { - for (const auto &P : Harness::Parameters) - benchmark::DoNotOptimize(Harness::Call(P, foo)); + inline void runBatch() { + for (const auto &P : Setup.Parameters) + benchmark::DoNotOptimize(Setup.Call(P, Configuration.Function)); } + size_t getBatchSize() const { return Setup.BatchSize; } + private: + SetupType Setup; benchmark::State &State; - Harness UP; MemorySizeDistribution Distribution; ArrayRef Probabilities; std::discrete_distribution SizeSampler; OffsetDistribution OffsetSampler; - std::mt19937_64 Gen; + ConfigurationType Configuration; }; -template static int64_t getMaxIndex() { - return Harness::getDistributions().size() - 1; -} - -void BM_Memcpy(benchmark::State &State) { - Randomized Harness(State); - while (State.KeepRunningBatch(Harness.BatchSize)) - Harness.runBatch(__llvm_libc::memcpy); -} -BENCHMARK(BM_Memcpy)->DenseRange(0, getMaxIndex()); +#define BENCHMARK_MEMORY_FUNCTION(BM_NAME, SETUP, CONFIGURATION_TYPE, \ + CONFIGURATION_ARRAY_REF) \ + void BM_NAME(benchmark::State &State) { \ + Runner Setup(State, CONFIGURATION_ARRAY_REF); \ + const size_t BatchSize = Setup.getBatchSize(); \ + while (State.KeepRunningBatch(BatchSize)) \ + Setup.runBatch(); \ + } \ + BENCHMARK(BM_NAME)->Apply([](benchmark::internal::Benchmark *benchmark) { \ + const int64_t DistributionSize = SETUP::getDistributions().size(); \ + const int64_t ConfigurationSize = CONFIGURATION_ARRAY_REF.size(); \ + for (int64_t DistIndex = 0; DistIndex < DistributionSize; ++DistIndex) \ + for (int64_t ConfIndex = 0; ConfIndex < ConfigurationSize; ++ConfIndex) \ + benchmark->Args({DistIndex, ConfIndex}); \ + }) -void BM_Memcmp(benchmark::State &State) { - Randomized Harness(State); - while (State.KeepRunningBatch(Harness.BatchSize)) - Harness.runBatch(__llvm_libc::memcmp); -} -BENCHMARK(BM_Memcmp)->DenseRange(0, getMaxIndex()); - -void BM_Memset(benchmark::State &State) { - Randomized Harness(State); - while (State.KeepRunningBatch(Harness.BatchSize)) - Harness.runBatch(__llvm_libc::memset); -} -BENCHMARK(BM_Memset)->DenseRange(0, getMaxIndex()); - -void BM_Bzero(benchmark::State &State) { - Randomized Harness(State); - while (State.KeepRunningBatch(Harness.BatchSize)) - Harness.runBatch(__llvm_libc::bzero); -} -BENCHMARK(BM_Bzero)->DenseRange(0, getMaxIndex()); +BENCHMARK_MEMORY_FUNCTION(BM_Memcpy, CopySetup, MemcpyConfiguration, + llvm::makeArrayRef(kMemcpyConfigurations)); +BENCHMARK_MEMORY_FUNCTION(BM_Memcmp, ComparisonSetup, MemcmpConfiguration, + llvm::makeArrayRef(kMemcmpConfigurations)); +BENCHMARK_MEMORY_FUNCTION(BM_Memset, SetSetup, MemsetConfiguration, + llvm::makeArrayRef(kMemsetConfigurations)); +BENCHMARK_MEMORY_FUNCTION(BM_Bzero, SetSetup, BzeroConfiguration, + llvm::makeArrayRef(kBzeroConfigurations));