[libc][automemcpy] Discard aggrated samples from JSON

[gchatelet]

Summary

The benchmark framework synthesizes fake "aggregate" Samples representing mean, median and cv.
We're only interested in "iteration" samples.

[gchatelet]

• Reverting print function, this will go in a different patch.

[courbet]

libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h

82

This needs documentation.

[gchatelet]

• Add documentation

[gchatelet]

rebase and squash

libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h

 // Identifies a Sample by its distribution and function. Used as a key in a map.
 struct SampleId {
   FunctionId Function;
   DistributionId Distribution;
   COMPARABLE_AND_HASHABLE(SampleId, Function.Type, Function.Name,
                           Distribution.Name)
 };
 
+// The type of Samples as reported by the Google Benchmark's JSON result file.
+// We are only interested in the "iteration" samples, the "aggregate" ones
+// represent derived metrics such as 'mean' or 'median'.
+enum class SampleType { UNKNOWN, ITERATION, AGGREGATE };
+
 // A SampleId with an associated measured throughput.
 struct Sample {
   SampleId Id;
+  SampleType Type = SampleType::UNKNOWN;
   double BytesPerSecond = 0;
 };
 
 // This function collects Samples that belong to the same distribution and
 // function and retains the median one. It then stores each of them into a
 // 'FunctionData' and returns them as a vector.
 std::vector<FunctionData> getThroughputs(ArrayRef<Sample> Samples);
 

libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp

static void processPerDistributionData(PerDistributionData &Data) {
   const size_t HalfSize = Samples.size() / 2;
   std::nth_element(Samples.begin(), Samples.begin() + HalfSize, Samples.end());
   Data.BytesPerSecondMedian = Samples[HalfSize];
 }
 
 std::vector<FunctionData> getThroughputs(ArrayRef<Sample> Samples) {
   std::unordered_map<FunctionId, FunctionData, FunctionId::Hasher> Functions;
   for (const auto &S : Samples) {
+    if (S.Type != SampleType::ITERATION)
+      break;
     auto &Function = Functions[S.Id.Function];
     auto &Data = Function.PerDistributionData[S.Id.Distribution.Name];
     Data.BytesPerSecondSamples.push_back(S.BytesPerSecond);
   }
 
   std::vector<FunctionData> Output;
   for (auto &[FunctionId, Function] : Functions) {
     Function.Id = FunctionId;

libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp

 static StringRef getInternalizedString(StringRef VolatileStr) {
   static llvm::StringSet StringCache;
   return StringCache.insert(VolatileStr).first->getKey();
 }
 
 // Helper function for the LLVM JSON API.
 bool fromJSON(const json::Value &V, Sample &Out, json::Path P) {
   std::string Label;
+  std::string RunType;
   json::ObjectMapper O(V, P);
   if (O && O.map("bytes_per_second", Out.BytesPerSecond) &&
-      O.map("label", Label)) {
+      O.map("run_type", RunType) && O.map("label", Label)) {
     const auto LabelPair = StringRef(Label).split(',');
     Out.Id.Function.Name = getInternalizedString(LabelPair.first);
     Out.Id.Function.Type = getFunctionDescriptor(LabelPair.first).Type;
     Out.Id.Distribution.Name = getInternalizedString(LabelPair.second);
+    Out.Type = StringSwitch<SampleType>(RunType)
+                   .Case("aggregate", SampleType::AGGREGATE)
+                   .Case("iteration", SampleType::ITERATION);
     return true;
   }
   return false;
 }
 
 // An object to represent the content of the JSON file.
 // This is easier to parse/serialize JSON when the structures of the json file
 // maps the structure of the object.

libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp

 namespace automemcpy {
 namespace {
 
 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsOneSample) {
   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
   static constexpr DistributionId DistA = {{"A"}};
   static constexpr SampleId Id = {Foo1, DistA};
   static constexpr Sample kSamples[] = {
-      Sample{Id, 4},
+      Sample{Id, SampleType::ITERATION, 4},
+      Sample{Id, SampleType::AGGREGATE, -1}, // Aggegates gets discarded
   };
 
   const std::vector<FunctionData> Data = getThroughputs(kSamples);
   EXPECT_THAT(Data, SizeIs(1));
   EXPECT_THAT(Data[0].Id, Foo1);
   EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1));
   // A single value is provided.
   const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name);
   EXPECT_THAT(DistributionData.BytesPerSecondMedian, 4);
   EXPECT_THAT(DistributionData.BytesPerSecondMean, 4);
   EXPECT_THAT(DistributionData.BytesPerSecondVariance, 0);
 }
 
 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsManySamplesSameBucket) {
   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
   static constexpr DistributionId DistA = {{"A"}};
   static constexpr SampleId Id = {Foo1, DistA};
-  static constexpr Sample kSamples[] = {Sample{Id, 4}, Sample{Id, 5},
-                                        Sample{Id, 5}};
+  static constexpr Sample kSamples[] = {Sample{Id, SampleType::ITERATION, 4},
+                                        Sample{Id, SampleType::ITERATION, 5},
+                                        Sample{Id, SampleType::ITERATION, 5}};
 
   const std::vector<FunctionData> Data = getThroughputs(kSamples);
   EXPECT_THAT(Data, SizeIs(1));
   EXPECT_THAT(Data[0].Id, Foo1);
   EXPECT_THAT(Data[0].PerDistributionData, SizeIs(1));
   // When multiple values are provided we pick the median one (here median of 4,
   // 5, 5).
   const auto &DistributionData = Data[0].PerDistributionData.lookup(DistA.Name);
   EXPECT_THAT(DistributionData.BytesPerSecondMedian, 5);
   EXPECT_THAT(DistributionData.BytesPerSecondMean, DoubleNear(4.6, 0.1));
   EXPECT_THAT(DistributionData.BytesPerSecondVariance, DoubleNear(0.33, 0.01));
 }
 
 TEST(AutomemcpyJsonResultsAnalyzer, getThroughputsServeralFunctionAndDist) {
   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
   static constexpr DistributionId DistA = {{"A"}};
   static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
   static constexpr DistributionId DistB = {{"B"}};
   static constexpr Sample kSamples[] = {
-      Sample{{Foo1, DistA}, 1}, Sample{{Foo1, DistB}, 2},
-      Sample{{Foo2, DistA}, 3}, Sample{{Foo2, DistB}, 4}};
+      Sample{{Foo1, DistA}, SampleType::ITERATION, 1},
+      Sample{{Foo1, DistB}, SampleType::ITERATION, 2},
+      Sample{{Foo2, DistA}, SampleType::ITERATION, 3},
+      Sample{{Foo2, DistB}, SampleType::ITERATION, 4}};
   // Data is aggregated per function.
   const std::vector<FunctionData> Data = getThroughputs(kSamples);
   EXPECT_THAT(Data, SizeIs(2)); // 2 functions Foo1 and Foo2.
   // Each function has data for both distributions DistA and DistB.
   EXPECT_THAT(Data[0].PerDistributionData, SizeIs(2));
   EXPECT_THAT(Data[1].PerDistributionData, SizeIs(2));
 }
 
 TEST(AutomemcpyJsonResultsAnalyzer, getScore) {
   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
   static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
   static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY};
   static constexpr DistributionId Dist = {{"A"}};
-  static constexpr Sample kSamples[] = {Sample{{Foo1, Dist}, 1},
-                                        Sample{{Foo2, Dist}, 2},
-                                        Sample{{Foo3, Dist}, 3}};
+  static constexpr Sample kSamples[] = {
+      Sample{{Foo1, Dist}, SampleType::ITERATION, 1},
+      Sample{{Foo2, Dist}, SampleType::ITERATION, 2},
+      Sample{{Foo3, Dist}, SampleType::ITERATION, 3}};
 
   // Data is aggregated per function.
   std::vector<FunctionData> Data = getThroughputs(kSamples);
 
   // Sort Data by function name so we can test them.
   std::sort(
       Data.begin(), Data.end(),
       [](const FunctionData &A, const FunctionData &B) { return A.Id < B.Id; });
TEST(AutomemcpyJsonResultsAnalyzer, castVotes) {
   static constexpr double kAbsErr = 0.01;
 
   static constexpr FunctionId Foo1 = {"memcpy1", FunctionType::MEMCPY};
   static constexpr FunctionId Foo2 = {"memcpy2", FunctionType::MEMCPY};
   static constexpr FunctionId Foo3 = {"memcpy3", FunctionType::MEMCPY};
   static constexpr DistributionId DistA = {{"A"}};
   static constexpr DistributionId DistB = {{"B"}};
   static constexpr Sample kSamples[] = {
-      Sample{{Foo1, DistA}, 0}, Sample{{Foo1, DistB}, 30},
-      Sample{{Foo2, DistA}, 1}, Sample{{Foo2, DistB}, 100},
-      Sample{{Foo3, DistA}, 7}, Sample{{Foo3, DistB}, 100},
+      Sample{{Foo1, DistA}, SampleType::ITERATION, 0},
+      Sample{{Foo1, DistB}, SampleType::ITERATION, 30},
+      Sample{{Foo2, DistA}, SampleType::ITERATION, 1},
+      Sample{{Foo2, DistB}, SampleType::ITERATION, 100},
+      Sample{{Foo3, DistA}, SampleType::ITERATION, 7},
+      Sample{{Foo3, DistB}, SampleType::ITERATION, 100},
   };
 
   // DistA Thoughput ranges from 0 to 7.
   // DistB Thoughput ranges from 30 to 100.
 
   // Data is aggregated per function.
   std::vector<FunctionData> Data = getThroughputs(kSamples);