[BOLT] Add support for the latest perf tool

[maksfb]

Summary

The latest perf tool can return non-empty buffer when executing
buildid-list command, even when perf.data was recorded with -B flag.
Some binaries will be listed without the ID, while others may have a
recorded ID. Allow invalid entires on the input, while checking the
valid ones for the match.

[maksfb]

Disable perf tool lookup in the unit test.

bolt/include/bolt/Profile/DataAggregator.h

private:
 
   /// Register a trace between two LBR entries supplied in execution order.
   bool doTrace(const LBREntry &First, const LBREntry &Second,
                uint64_t Count = 1);
 
   /// Parser helpers
   /// Return false if we exhausted our parser buffer and finished parsing
   /// everything
-  bool hasData();
+  bool hasData() const { return !ParsingBuf.empty(); }
 
   /// Print heat map based on LBR samples.
   std::error_code printLBRHeatMap();
 
   /// Parse a single perf sample containing a PID associated with a sequence of
   /// LBR entries. If the PID does not correspond to the binary we are looking
   /// for, return std::errc::no_such_process. If other parsing errors occur,
   /// return the error. Otherwise, return the parsed sample.
private:
 
   /// Parse output of `perf script --show-task-events`, and forked processes
   /// to the set of tracked PIDs.
   std::error_code parseTaskEvents();
 
   /// Parse a single pair of binary full path and associated build-id
   Optional<std::pair<StringRef, StringRef>> parseNameBuildIDPair();
 
-  /// Parse the output generated by "perf buildid-list" to extract build-ids
-  /// and return a file name matching a given \p FileBuildID.
-  Optional<StringRef> getFileNameForBuildID(StringRef FileBuildID);
-
   /// Coordinate reading and parsing of pre-aggregated file
   ///
   /// The regular perf2bolt aggregation job is to read perf output directly.
   /// However, if the data is coming from a database instead of perf, one could
   /// write a query to produce a pre-aggregated file. This function deals with
   /// this case.
   ///
   /// The pre-aggregated file contains aggregated LBR data, but without binary
private:
   /// If \p FileBuildID has no match, then issue an error and exit.
   void processFileBuildID(StringRef FileBuildID);
 
   /// Debugging dump methods
   void dump() const;
   void dump(const LBREntry &LBR) const;
   void dump(const PerfBranchSample &Sample) const;
   void dump(const PerfMemSample &Sample) const;
+
+public:
+  /// If perf.data was collected without build ids, the buildid-list may contain
+  /// incomplete entries. Return true if the buffer containing
+  /// "perf buildid-list" output has only valid entries and is non- empty.
+  /// Return false otherwise.
+  bool hasAllBuildIDs();
+
+  /// Parse the output generated by "perf buildid-list" to extract build-ids
+  /// and return a file name matching a given \p FileBuildID.
+  Optional<StringRef> getFileNameForBuildID(StringRef FileBuildID);
 };
 } // namespace bolt
 } // namespace llvm
 
 #endif

bolt/include/bolt/Profile/DataReader.h

protected:
   bool NoLBRMode{false};
   bool BATMode{false};
   StringSet<> EventNames;
   static const char FieldSeparator = ' ';
 
   /// Maps of common LTO names to possible matching profiles.
   StringMap<std::vector<FuncBranchData *>> LTOCommonNameMap;
   StringMap<std::vector<FuncMemData *>> LTOCommonNameMemMap;
+
+public:
+  void setParsingBuffer(StringRef Buffer) { ParsingBuf = Buffer; }
 };
 
 } // namespace bolt
 
 /// DenseMapInfo allows us to use the DenseMap LLVM data structure to store
 /// Locations
 template <> struct DenseMapInfo<bolt::Location> {
   static inline bolt::Location getEmptyKey() {

bolt/lib/Profile/DataAggregator.cpp

 static cl::opt<unsigned long long>
 MaxSamples("max-samples",
   cl::init(-1ULL),
   cl::desc("maximum number of samples to read from LBR profile"),
   cl::Optional,
   cl::Hidden,
   cl::cat(AggregatorCategory));
 
-static cl::opt<bool> ReadPreAggregated(
+cl::opt<bool> ReadPreAggregated(
     "pa", cl::desc("skip perf and read data from a pre-aggregated file format"),
     cl::cat(AggregatorCategory));
 
 static cl::opt<bool>
 TimeAggregator("time-aggr",
   cl::desc("time BOLT aggregator"),
   cl::init(false),
   cl::ZeroOrMore,
void DataAggregator::processFileBuildID(StringRef FileBuildID) {
   if (std::error_code EC = MB.getError()) {
     errs() << "Cannot open " << BuildIDProcessInfo.StdoutPath.data() << ": "
            << EC.message() << "\n";
     return;
   }
 
   FileBuf = std::move(*MB);
   ParsingBuf = FileBuf->getBuffer();
-  if (ParsingBuf.empty()) {
-    errs() << "PERF2BOLT-WARNING: build-id will not be checked because perf "
-              "data was recorded without it\n";
-    return;
-  }
 
-  Col = 0;
-  Line = 1;
   Optional<StringRef> FileName = getFileNameForBuildID(FileBuildID);
   if (!FileName) {
+    if (hasAllBuildIDs()) {
       errs() << "PERF2BOLT-ERROR: failed to match build-id from perf output. "
                 "This indicates the input binary supplied for data aggregation "
                 "is not the same recorded by perf when collecting profiling "
                 "data, or there were no samples recorded for the binary. "
                 "Use -ignore-build-id option to override.\n";
       if (!opts::IgnoreBuildID)
         abort();
+    } else {
+      errs() << "PERF2BOLT-WARNING: build-id will not be checked because perf "
+                "data was recorded without it\n";
+      return;
+    }
   } else if (*FileName != llvm::sys::path::filename(BC->getFilename())) {
     errs() << "PERF2BOLT-WARNING: build-id matched a different file name\n";
     BuildIDBinaryName = std::string(*FileName);
   } else {
     outs() << "PERF2BOLT: matched build-id and file name\n";
   }
 
   return;
if (!checkAndConsumeNewLine()) {
     return make_error_code(llvm::errc::io_error);
   }
 
   return AggregatedLBREntry{From.get(), To.get(),
                             static_cast<uint64_t>(Frequency.get()), Mispreds,
                             Type};
 }
 
-bool DataAggregator::hasData() {
-  if (ParsingBuf.size() == 0)
-    return false;
-
-  return true;
-}
-
 bool DataAggregator::ignoreKernelInterrupt(LBREntry &LBR) const {
   return opts::IgnoreInterruptLBR &&
          (LBR.From >= KernelBaseAddr || LBR.To >= KernelBaseAddr);
 }
 
 std::error_code DataAggregator::printLBRHeatMap() {
   outs() << "PERF2BOLT: parse branch events...\n";
   NamedRegionTimer T("parseBranch", "Parsing branch events", TimerGroupName,

 DataAggregator::parseNameBuildIDPair() {
   while (checkAndConsumeFS()) {
   }
 
   ErrorOr<StringRef> BuildIDStr = parseString(FieldSeparator, true);
   if (std::error_code EC = BuildIDStr.getError())
     return NoneType();
 
+  // If one of the strings is missing, don't issue a parsing error, but still
+  // do not return a value.
+  if (ParsingBuf[0] == '\n')
+    return NoneType();
+
   ErrorOr<StringRef> NameStr = parseString(FieldSeparator, true);
   if (std::error_code EC = NameStr.getError())
     return NoneType();
 
   consumeRestOfLine();
   return std::make_pair(NameStr.get(), BuildIDStr.get());
 }
 
+bool DataAggregator::hasAllBuildIDs() {
+  const StringRef SavedParsingBuf = ParsingBuf;
+
+  if (!hasData())
+    return false;
+
+  bool HasInvalidEntries = false;
+  while (hasData()) {
+    if (!parseNameBuildIDPair()) {
+      HasInvalidEntries = true;
+      break;
+    }
+  }
+
+  ParsingBuf = SavedParsingBuf;
+
+  return !HasInvalidEntries;
+}
+
 Optional<StringRef>
 DataAggregator::getFileNameForBuildID(StringRef FileBuildID) {
+  const StringRef SavedParsingBuf = ParsingBuf;
+
+  StringRef FileName;
   while (hasData()) {
     Optional<std::pair<StringRef, StringRef>> IDPair = parseNameBuildIDPair();
-    if (!IDPair)
-      return NoneType();
+    if (!IDPair) {
+      consumeRestOfLine();
+      continue;
+    }
 
-    if (IDPair->second.startswith(FileBuildID))
-      return sys::path::filename(IDPair->first);
+    if (IDPair->second.startswith(FileBuildID)) {
+      FileName = sys::path::filename(IDPair->first);
+      break;
     }
+  }
+
+  ParsingBuf = SavedParsingBuf;
+
+  if (!FileName.empty())
+    return FileName;
+
   return NoneType();
 }
 
 std::error_code
 DataAggregator::writeAggregatedFile(StringRef OutputFilename) const {
   std::error_code EC;
   raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::OpenFlags::OF_None);
   if (EC)

bolt/unittests/CMakeLists.txt

 add_custom_target(BoltUnitTests)
 set_target_properties(BoltUnitTests PROPERTIES FOLDER "BOLT tests")
 
 function(add_bolt_unittest test_dirname)
   add_unittest(BoltUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
 add_subdirectory(Core)
+add_subdirectory(Profile)

bolt/unittests/Profile/CMakeLists.txt

+add_bolt_unittest(ProfileTests
+  DataAggregator.cpp
+  )
+
+target_link_libraries(ProfileTests
+  PRIVATE
+  LLVMBOLTProfile
+  )
+

bolt/unittests/Profile/DataAggregator.cpp

+//===- bolt/unittests/Profile/DataAggregator.cpp --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Profile/DataAggregator.h"
+#include "llvm/Support/CommandLine.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::bolt;
+
+namespace opts {
+extern cl::opt<bool> ReadPreAggregated;
+}
+
+TEST(DataAggregatorTest, buildID) {
+  // Avoid looking for perf tool.
+  opts::ReadPreAggregated = true;
+
+  DataAggregator DA("<pseudo input>");
+  Optional<StringRef> FileName;
+
+  DA.setParsingBuffer("");
+  ASSERT_FALSE(DA.hasAllBuildIDs());
+  FileName = DA.getFileNameForBuildID("1234");
+  ASSERT_FALSE(FileName);
+
+  StringRef PartialValidBuildIDs = "     File0\n"
+                                   "1111 File1\n"
+                                   "     File2\n";
+  DA.setParsingBuffer(PartialValidBuildIDs);
+  ASSERT_FALSE(DA.hasAllBuildIDs());
+  FileName = DA.getFileNameForBuildID("0000");
+  ASSERT_FALSE(FileName);
+  FileName = DA.getFileNameForBuildID("1111");
+  ASSERT_EQ(*FileName, "File1");
+
+  StringRef AllValidBuildIDs = "0000 File0\n"
+                               "1111 File1\n"
+                               "2222 File2\n";
+  DA.setParsingBuffer(AllValidBuildIDs);
+  ASSERT_TRUE(DA.hasAllBuildIDs());
+  FileName = DA.getFileNameForBuildID("1234");
+  ASSERT_FALSE(FileName);
+  FileName = DA.getFileNameForBuildID("2222");
+  ASSERT_EQ(*FileName, "File2");
+}