[llvm-profgen] Support to load debug info from a second binary

[wlei]

Summary

For reducing binary size purpose, the binary's debug info and executable segment can be separated(like using objcopy --only-keep-debug). Here add support in llvm-profgen to use two binaries as input. The original one is executable binary and added for debug info only binary. Adding a flag --debug-binary=file-path, with this, the binary will load debug info from debug binary.

[wenlei]

llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test

10–13 ↗

(On Diff #395153)

Let's use a dedicated/targeted test case for this scenario, instead of making existing test more complicated.

[hoy]

llvm/test/tools/llvm-profgen/inline-noprobe2.test

112 ↗

(On Diff #395153)

We should be able to use llvm-objcopy in the test, so the existing test binary can be reused and with an extra test step running llvm-objcopy.

[wlei]

change symbolizer patch to point to debug info path.
addressing reviewers' feedback.

[wlei]

llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test

10–13 ↗

(On Diff #395153)

Separated this part into a new test.

llvm/test/tools/llvm-profgen/inline-noprobe2.test

112 ↗

(On Diff #395153)

Good point! Changed.

[wlei]

Updating D115948: [llvm-profgen] Support to load debug info from a second binary

[hoy]

lgtm, thanks.

[wenlei]

lgtm.

llvm/test/tools/llvm-profgen/separate-debuginfo-binary.test

+; RUN: llvm-objcopy --strip-debug %S/Inputs/inline-noprobe.perfbin %t1
+; RUN: llvm-objcopy --only-keep-debug %S/Inputs/inline-noprobe.perfbin %t2
+; RUN: echo -e "0\n0" > %t
+; RUN: llvm-profgen --format=text --unsymbolized-profile=%t --binary=%t1 --debug-binary=%t2 --output=%t3 --fill-zero-for-all-funcs
+; RUN: FileCheck %s --input-file %t3 --check-prefix=CHECK
+
+; RUN: llvm-objcopy --strip-debug %S/Inputs/inline-cs-pseudoprobe.perfbin %t4
+; RUN: llvm-objcopy --only-keep-debug %S/Inputs/inline-cs-pseudoprobe.perfbin %t5
+; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-pseudoprobe.perfscript --binary=%t4 --debug-binary=%t5 --output=%t6 --profile-summary-hot-count=0 --csspgo-preinliner=0
+; RUN: FileCheck %s --input-file %t6 --check-prefix=CHECK-CS-PROBE
+
+; CHECK: bar:0:0
+; CHECK:  1: 0
+; CHECK:  5: 0
+; CHECK: foo:0:0
+; CHECK:  0: 0
+; CHECK:  2.1: 0
+; CHECK:  3: 0
+; CHECK:  3.2: 0
+; CHECK:  4: 0
+; CHECK:  3.1: bar:0
+; CHECK:   1: 0
+; CHECK:   65533: 0
+; CHECK:  3.2: bar:0
+; CHECK:   1: 0
+; CHECK:   7: 0
+; CHECK: main:0:0
+; CHECK:  0: 0
+; CHECK:  2: 0
+; CHECK:  1: foo:0
+; CHECK:   2.1: 0
+; CHECK:   3: 0
+; CHECK:   3.2: 0
+; CHECK:   4: 0
+; CHECK:   65526: 0
+; CHECK:   3.1: bar:0
+; CHECK:    1: 0
+; CHECK:    65533: 0
+; CHECK:   3.2: bar:0
+; CHECK:    1: 0
+
+
+; CHECK-CS-PROBE: [main:2 @ foo]:74:0
+; CHECK-CS-PROBE:   1: 0
+; CHECK-CS-PROBE:   2: 15
+; CHECK-CS-PROBE:   3: 15
+; CHECK-CS-PROBE:   4: 14
+; CHECK-CS-PROBE:   5: 1
+; CHECK-CS-PROBE:   6: 15
+; CHECK-CS-PROBE:   7: 0
+; CHECK-CS-PROBE:   8: 14 bar:14
+; CHECK-CS-PROBE:   9: 0
+; CHECK-CS-PROBE: !CFGChecksum: 563088904013236
+; CHECK-CS-PROBE: [main:2 @ foo:8 @ bar]:28:14
+; CHECK-CS-PROBE:   1: 14
+; CHECK-CS-PROBE:   4: 14
+; CHECK-CS-PROBE: !CFGChecksum: 72617220756

llvm/tools/llvm-profgen/ProfiledBinary.h

private:
   // exist. It gives us the best possible estimate for function's post-inline,
   // post-optimization byte size.
   ContextTrieNode RootContext;
 };
 
 using OffsetRange = std::pair<uint64_t, uint64_t>;
 
 class ProfiledBinary {
-  // Absolute path of the binary.
+  // Absolute path of the executable binary.
   std::string Path;
+  // Path of the debug info binary.
+  std::string DebugBinaryPath;
+  // Path of symbolizer path which should be pointed to binary with debug info.
+  StringRef SymbolizerPath;
   // The target triple.
   Triple TheTriple;
   // The runtime base address that the first executable segment is loaded at.
   uint64_t BaseAddress = 0;
   // The runtime base address that the first loadabe segment is loaded at.
   uint64_t FirstLoadableAddress = 0;
   // The preferred load address of each executable segment.
   std::vector<uint64_t> PreferredTextSegmentAddresses;
class ProfiledBinary {
   ///   1. Text sections, including the main code section and the PLT
   ///   entries that will be used to handle cross-module call transitions.
   ///   2. The .debug_line section, used by Dwarf-based profile generation.
   ///   3. Pseudo probe related sections, used by probe-based profile
   ///   generation.
   void load();
 
 public:
-  ProfiledBinary(const StringRef Path)
-      : Path(Path), ProEpilogTracker(this),
+  ProfiledBinary(const StringRef ExeBinPath, const StringRef DebugBinPath)
+      : Path(ExeBinPath), DebugBinaryPath(DebugBinPath), ProEpilogTracker(this),
         TrackFuncContextSize(EnableCSPreInliner &&
                              UseContextCostForPreInliner) {
+    // Point to executable binary if debug info binary is not specified.
+    SymbolizerPath = DebugBinPath.empty() ? ExeBinPath : DebugBinPath;
     setupSymbolizer();
     load();
   }
   uint64_t virtualAddrToOffset(uint64_t VirtualAddress) const {
     return VirtualAddress - BaseAddress;
   }
   uint64_t offsetToVirtualAddr(uint64_t Offset) const {
     return Offset + BaseAddress;

llvm/tools/llvm-profgen/ProfiledBinary.cpp

void ProfiledBinary::warnNoFuncEntry() {
   emitWarningSummary(NoFuncEntryNum, BinaryFunctions.size(),
                      "of functions failed to determine function entry due to "
                      "inconsistent name from symbol table and dwarf info.");
 }
 
 void ProfiledBinary::load() {
   // Attempt to open the binary.
   OwningBinary<Binary> OBinary = unwrapOrError(createBinary(Path), Path);
-  Binary &Binary = *OBinary.getBinary();
+  Binary &ExeBinary = *OBinary.getBinary();
 
-  auto *Obj = dyn_cast<ELFObjectFileBase>(&Binary);
+  auto *Obj = dyn_cast<ELFObjectFileBase>(&ExeBinary);
   if (!Obj)
     exitWithError("not a valid Elf image", Path);
 
   TheTriple = Obj->makeTriple();
   // Current only support X86
   if (!TheTriple.isX86())
     exitWithError("unsupported target", TheTriple.getTriple());
   LLVM_DEBUG(dbgs() << "Loading " << Path << "\n");
 
   // Find the preferred load address for text sections.
   setPreferredTextSegmentAddresses(Obj);
 
   // Decode pseudo probe related section
   decodePseudoProbe(Obj);
 
   // Load debug info of subprograms from DWARF section.
-  loadSymbolsFromDWARF(*cast<ObjectFile>(&Binary));
+  // If path of debug info binary is specified, use the debug info from it,
+  // otherwise use the debug info from the executable binary.
+  if (!DebugBinaryPath.empty()) {
+    OwningBinary<Binary> DebugPath =
+        unwrapOrError(createBinary(DebugBinaryPath), DebugBinaryPath);
+    loadSymbolsFromDWARF(*dyn_cast<ObjectFile>(DebugPath.getBinary()));
+  } else {
+    loadSymbolsFromDWARF(*dyn_cast<ObjectFile>(&ExeBinary));
+  }
 
   // Disassemble the text sections.
   disassemble(Obj);
 
   // Track size for optimized inlinees when probe is available
   if (UsePseudoProbes && TrackFuncContextSize)
     FuncSizeTracker.trackInlineesOptimizedAway(ProbeDecoder);
 

 
 SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
                                                    bool UseCanonicalFnName,
                                                    bool UseProbeDiscriminator) {
   assert(this == IP.Binary &&
          "Binary should only symbolize its own instruction");
   auto Addr = object::SectionedAddress{IP.Offset + getPreferredBaseAddress(),
                                        object::SectionedAddress::UndefSection};
-  DIInliningInfo InlineStack =
-      unwrapOrError(Symbolizer->symbolizeInlinedCode(Path, Addr), getName());
+  DIInliningInfo InlineStack = unwrapOrError(
+      Symbolizer->symbolizeInlinedCode(SymbolizerPath.str(), Addr),
+      SymbolizerPath);
 
   SampleContextFrameVector CallStack;
   for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {
     const auto &CallerFrame = InlineStack.getFrame(I);
     if (CallerFrame.FunctionName == "<invalid>")
       break;
 
     StringRef FunctionName(CallerFrame.FunctionName);

llvm/tools/llvm-profgen/llvm-profgen.cpp

static cl::opt<std::string> UnsymbolizedProfFilename(
     "unsymbolized-profile", cl::value_desc("unsymbolized profile"),
     cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated,
     cl::desc("Path of the unsymbolized profile created by "
              "`llvm-profgen` with `--skip-symbolization`"),
     cl::cat(ProfGenCategory));
 static cl::alias UPA("up", cl::desc("Alias for --unsymbolized-profile"),
                      cl::aliasopt(UnsymbolizedProfFilename));
 
-static cl::opt<std::string> BinaryPath(
-    "binary", cl::value_desc("binary"), cl::Required,
-    cl::desc("Path of profiled binary, only one binary is supported."),
+static cl::opt<std::string>
+    BinaryPath("binary", cl::value_desc("binary"), cl::Required,
+               cl::desc("Path of profiled executable binary."),
+               cl::cat(ProfGenCategory));
+
+static cl::opt<std::string> DebugBinPath(
+    "debug-binary", cl::value_desc("debug-binary"), cl::ZeroOrMore,
+    cl::desc("Path of debug info binary, llvm-profgen will load the DWARF info "
+             "from it instead of the executable binary."),
     cl::cat(ProfGenCategory));
 
 extern cl::opt<bool> ShowDisassemblyOnly;
 extern cl::opt<bool> ShowSourceLocations;
 extern cl::opt<bool> SkipSymbolization;
 
 using namespace llvm;
 using namespace sampleprof;
int main(int argc, const char *argv[]) {
   InitializeAllDisassemblers();
 
   cl::HideUnrelatedOptions({&ProfGenCategory, &getColorCategory()});
   cl::ParseCommandLineOptions(argc, argv, "llvm SPGO profile generator\n");
   validateCommandLine();
 
   // Load symbols and disassemble the code of a given binary.
   std::unique_ptr<ProfiledBinary> Binary =
-      std::make_unique<ProfiledBinary>(BinaryPath);
+      std::make_unique<ProfiledBinary>(BinaryPath, DebugBinPath);
   if (ShowDisassemblyOnly)
     return EXIT_SUCCESS;
 
   PerfInputFile PerfFile = getPerfInputFile();
   std::unique_ptr<PerfReaderBase> Reader =
       PerfReaderBase::create(Binary.get(), PerfFile);
   // Parse perf events and samples
   Reader->parsePerfTraces();