[libc] Add a Google Benchmark target to support continuous monitoring of memory operation performance

[gchatelet]

Summary

The next step is to be able to benchmark several implementations at once and compare which one performs best on a particular machine.

[gchatelet]

• rebase and add missing new line at EOF

libc/benchmarks/CMakeLists.txt

foreach(fq_config_name IN LISTS fq_implementations)
     endif()
   endforeach()
 endfunction()
 
 add_libc_multi_impl_benchmark(memcpy)
 add_libc_multi_impl_benchmark(memset)
 add_libc_multi_impl_benchmark(bzero)
 add_libc_multi_impl_benchmark(memcmp)
+
+#==============================================================================
+# Google Benchmarking tool
+#==============================================================================
+
+# This target uses the Google Benchmark facility to report throughput for llvm
+# libc memory functions compiled for the host machine. This is useful to
+# continuously monitor the performance of the memory functions.
+add_executable(libc.benchmarks.memory_functions.opt_host
+  EXCLUDE_FROM_ALL
+  LibcMemoryGoogleBenchmarkMain.cpp
+)
+
+target_link_libraries(libc.benchmarks.memory_functions.opt_host
+  PRIVATE
+  libc-memory-benchmark
+  libc.src.string.memcmp_opt_host
+  libc.src.string.memcpy_opt_host
+  libc.src.string.memset_opt_host
+  libc.src.string.bzero_opt_host
+  benchmark_main
+)

libc/benchmarks/LibcMemoryBenchmark.h

struct ParameterBatch {
   std::vector<ParameterType> Parameters;
 };
 
 /// Provides source and destination buffers for the Copy operation as well as
 /// the associated size distributions.
 struct CopyHarness : public ParameterBatch {
   CopyHarness();
 
-  static const ArrayRef<MemorySizeDistribution> Distributions;
+  inline static const ArrayRef<MemorySizeDistribution> getDistributions() {
+    return getMemcpySizeDistributions();
+  }
 
   inline void *Call(ParameterType Parameter,
                     void *(*memcpy)(void *__restrict, const void *__restrict,
                                     size_t)) {
     return memcpy(DstBuffer + Parameter.OffsetBytes,
                   SrcBuffer + Parameter.OffsetBytes, Parameter.SizeBytes);
   }
 
 private:
   AlignedBuffer SrcBuffer;
   AlignedBuffer DstBuffer;
 };
 
 /// Provides destination buffer for the Set operation as well as the associated
 /// size distributions.
 struct SetHarness : public ParameterBatch {
   SetHarness();
 
-  static const ArrayRef<MemorySizeDistribution> Distributions;
+  inline static const ArrayRef<MemorySizeDistribution> getDistributions() {
+    return getMemsetSizeDistributions();
+  }
 
   inline void *Call(ParameterType Parameter,
                     void *(*memset)(void *, int, size_t)) {
     return memset(DstBuffer + Parameter.OffsetBytes,
                   Parameter.OffsetBytes % 0xFF, Parameter.SizeBytes);
   }
 
   inline void *Call(ParameterType Parameter, void (*bzero)(void *, size_t)) {
     bzero(DstBuffer + Parameter.OffsetBytes, Parameter.SizeBytes);
     return DstBuffer.begin();
   }
 
 private:
   AlignedBuffer DstBuffer;
 };
 
 /// Provides left and right buffers for the Comparison operation as well as the
 /// associated size distributions.
 struct ComparisonHarness : public ParameterBatch {
   ComparisonHarness();
 
-  static const ArrayRef<MemorySizeDistribution> Distributions;
+  inline static const ArrayRef<MemorySizeDistribution> getDistributions() {
+    return getMemcmpSizeDistributions();
+  }
 
   inline int Call(ParameterType Parameter,
                   int (*memcmp)(const void *, const void *, size_t)) {
     return memcmp(LhsBuffer + Parameter.OffsetBytes,
                   RhsBuffer + Parameter.OffsetBytes, Parameter.SizeBytes);
   }
 
 private:
   AlignedBuffer LhsBuffer;
   AlignedBuffer RhsBuffer;
 };
 
 } // namespace libc_benchmarks
 } // namespace llvm
 
 #endif // LLVM_LIBC_UTILS_BENCHMARK_MEMORY_BENCHMARK_H

libc/benchmarks/LibcMemoryBenchmark.cpp

report_fatal_error(
         llvm::Twine("Call would result in buffer overflow: Offset=")
             .concat(llvm::Twine(P.OffsetBytes))
             .concat(", Size=")
             .concat(llvm::Twine(P.SizeBytes))
             .concat(", BufferSize=")
             .concat(llvm::Twine(BufferSize)));
 }
 
-const ArrayRef<MemorySizeDistribution> CopyHarness::Distributions =
-    getMemcpySizeDistributions();
-const ArrayRef<MemorySizeDistribution> ComparisonHarness::Distributions =
-    getMemcmpSizeDistributions();
-const ArrayRef<MemorySizeDistribution> SetHarness::Distributions =
-    getMemsetSizeDistributions();
-
 CopyHarness::CopyHarness()
     : ParameterBatch(2), SrcBuffer(ParameterBatch::BufferSize),
       DstBuffer(ParameterBatch::BufferSize) {}
 
 ComparisonHarness::ComparisonHarness()
     : ParameterBatch(2), LhsBuffer(ParameterBatch::BufferSize),
       RhsBuffer(ParameterBatch::BufferSize) {
   // The memcmp buffers always compare equal.

libc/benchmarks/LibcMemoryGoogleBenchmarkMain.cpp

+#include "LibcBenchmark.h"
+#include "LibcMemoryBenchmark.h"
+#include "MemorySizeDistributions.h"
+#include "benchmark/benchmark.h"
+#include <cstdint>
+#include <random>
+#include <vector>
+
+namespace __llvm_libc {
+
+extern void *memcpy(void *__restrict, const void *__restrict, size_t);
+extern void *memset(void *, int, size_t);
+extern void bzero(void *, size_t);
+extern int memcmp(const void *, const void *, size_t);
+
+} // namespace __llvm_libc
+
+using llvm::Align;
+using llvm::ArrayRef;
+using llvm::libc_benchmarks::ComparisonHarness;
+using llvm::libc_benchmarks::CopyHarness;
+using llvm::libc_benchmarks::MemorySizeDistribution;
+using llvm::libc_benchmarks::OffsetDistribution;
+using llvm::libc_benchmarks::SetHarness;
+
+static constexpr Align kBenchmarkAlignment = Align::Constant<1>();
+
+template <typename Harness> struct Randomized : public Harness {
+  Randomized(benchmark::State &State)
+      : State(State), Distribution(Harness::getDistributions()[State.range(0)]),
+        Probabilities(Distribution.Probabilities),
+        SizeSampler(Probabilities.begin(), Probabilities.end()),
+        OffsetSampler(Harness::BufferSize, Probabilities.size() - 1,
+                      kBenchmarkAlignment) {
+    for (auto &P : Harness::Parameters) {
+      P.OffsetBytes = OffsetSampler(Gen);
+      P.SizeBytes = SizeSampler(Gen);
+      Harness::checkValid(P);
+    }
+  }
+
+  ~Randomized() {
+    const size_t AvgBytesPerIteration =
+        Harness::getBatchBytes() / Harness::BatchSize;
+    const size_t TotalBytes = State.iterations() * AvgBytesPerIteration;
+    State.SetBytesProcessed(TotalBytes);
+    State.SetLabel(Distribution.Name.str());
+    State.counters["bytes_per_cycle"] = benchmark::Counter(
+        TotalBytes / benchmark::CPUInfo::Get().cycles_per_second,
+        benchmark::Counter::kIsRate);
+  }
+
+  template <typename Function> inline void runBatch(Function foo) {
+    for (const auto &P : Harness::Parameters)
+      benchmark::DoNotOptimize(Harness::Call(P, foo));
+  }
+
+private:
+  benchmark::State &State;
+  Harness UP;
+  MemorySizeDistribution Distribution;
+  ArrayRef<double> Probabilities;
+  std::discrete_distribution<unsigned> SizeSampler;
+  OffsetDistribution OffsetSampler;
+  std::mt19937_64 Gen;
+};
+
+template <typename Harness> static int64_t getMaxIndex() {
+  return Harness::getDistributions().size() - 1;
+}
+
+void BM_Memcpy(benchmark::State &State) {
+  Randomized<CopyHarness> Harness(State);
+  while (State.KeepRunningBatch(Harness.BatchSize))
+    Harness.runBatch(__llvm_libc::memcpy);
+}
+BENCHMARK(BM_Memcpy)->DenseRange(0, getMaxIndex<CopyHarness>());
+
+void BM_Memcmp(benchmark::State &State) {
+  Randomized<ComparisonHarness> Harness(State);
+  while (State.KeepRunningBatch(Harness.BatchSize))
+    Harness.runBatch(__llvm_libc::memcmp);
+}
+BENCHMARK(BM_Memcmp)->DenseRange(0, getMaxIndex<ComparisonHarness>());
+
+void BM_Memset(benchmark::State &State) {
+  Randomized<SetHarness> Harness(State);
+  while (State.KeepRunningBatch(Harness.BatchSize))
+    Harness.runBatch(__llvm_libc::memset);
+}
+BENCHMARK(BM_Memset)->DenseRange(0, getMaxIndex<SetHarness>());
+
+void BM_Bzero(benchmark::State &State) {
+  Randomized<SetHarness> Harness(State);
+  while (State.KeepRunningBatch(Harness.BatchSize))
+    Harness.runBatch(__llvm_libc::bzero);
+}
+BENCHMARK(BM_Bzero)->DenseRange(0, getMaxIndex<SetHarness>());