[CSSPGO][llvm-profgen] Instruction symbolization

[wlei]

Summary

This stack of changes introduces llvm-profgen utility which generates a profile data file from given perf script data files for sample-based PGO. It’s part of(not only) the CSSPGO work. Specifically to support context-sensitive with/without pseudo probe profile, it implements a series of functionalities including perf trace parsing, instruction symbolization, LBR stack/call frame stack unwinding, pseudo probe decoding, etc. Also high throughput is achieved by multiple levels of sample aggregation and compatible format with one stop is generated at the end. Please refer to: https://groups.google.com/g/llvm-dev/c/1p1rdYbL93s for the CSSPGO RFC.

This change adds the support of instruction symbolization. Given the RVA on an instruction pointer, a full calling context can be printed side-by-side with the disassembly code.
E.g.

Disassembly of section .text [0x0, 0x4a]:

<funcA>:
    0:	mov	eax, edi                           funcA:0
    2:	mov	ecx, dword ptr [rip]               funcLeaf:2 @ funcA:1
    8:	lea	edx, [rcx + 3]                     fib:2 @ funcLeaf:2 @ funcA:1
    b:	cmp	ecx, 3                             fib:2 @ funcLeaf:2 @ funcA:1
    e:	cmovl	edx, ecx                           fib:2 @ funcLeaf:2 @ funcA:1
   11:	sub	eax, edx                           funcLeaf:2 @ funcA:1
   13:	ret                                        funcA:2
   14:	nop	word ptr cs:[rax + rax]
   1e:	nop

<funcLeaf>:
   20:	mov	eax, edi                           funcLeaf:1
   22:	mov	ecx, dword ptr [rip]               funcLeaf:2
   28:	lea	edx, [rcx + 3]                     fib:2 @ funcLeaf:2
   2b:	cmp	ecx, 3                             fib:2 @ funcLeaf:2
   2e:	cmovl	edx, ecx                           fib:2 @ funcLeaf:2
   31:	sub	eax, edx                           funcLeaf:2
   33:	ret                                        funcLeaf:3
   34:	nop	word ptr cs:[rax + rax]
   3e:	nop

<fib>:
   40:	lea	eax, [rdi + 3]                     fib:2
   43:	cmp	edi, 3                             fib:2
   46:	cmovl	eax, edi                           fib:2
   49:	ret                                        fib:8

Test Plan:
ninja check-llvm

[wlei]

rebase

[wenlei]

llvm/tools/llvm-profgen/CallContext.h

22

Name it FrameLocation? SourceLocation sounds too general and not different from LineLocation.

25

Name it FrameLocationStack? Also would be good to update the comment to be accurate - this is not a caller, callee pair.

41

std::ostringstream is probably better since there can be quite a few appendings. Same for getReversedLocWithContext.

[wlei]

address reviewers' comments and rebase

[wlei]

fix lint

[wlei]

rebase

[wlei]

rebase and fix clang-tidy warning

[wlei]

miss one clang-tidy warning, fix it

[wlei]

Fix clang-tidy and the wrong type which caused an overflow

[wmi]

llvm/tools/llvm-profgen/ProfiledBinary.cpp

328

Add some message for the assert(this == IP.Binary && "...");

[wlei]

[NFC]Add description to an assertion

llvm/test/tools/llvm-profgen/symbolize.ll

+; REQUIRES: x86-registered-target
+; RUN: llc -filetype=obj %s -o %t
+; RUN: llvm-profgen --binary=%t --perfscript=%s --output=%t1 -show-disassembly -x86-asm-syntax=intel --show-source-locations | FileCheck %s --match-full-lines 
+
+; CHECK: Disassembly of section .text [0x0, 0x4a]:
+; CHECK: <funcA>:
+; CHECK:        0:  mov eax, edi                         funcA:0
+; CHECK:        2:  mov ecx, dword ptr [rip]             funcLeaf:2 @ funcA:1
+; CHECK:        8:  lea edx, [rcx + 3]                   fib:2 @ funcLeaf:2 @ funcA:1
+; CHECK:        b:  cmp ecx, 3                           fib:2 @ funcLeaf:2 @ funcA:1
+; CHECK:        e:  cmovl edx, ecx                       fib:2 @ funcLeaf:2 @ funcA:1
+; CHECK:       11:  sub eax, edx                         funcLeaf:2 @ funcA:1
+; CHECK:       13:  ret                                  funcA:2
+; CHECK:       14:  nop word ptr cs:[rax + rax]
+; CHECK:       1e:  nop
+; CHECK: <funcLeaf>:
+; CHECK:      20: mov eax, edi                           funcLeaf:1
+; CHECK:      22: mov ecx, dword ptr [rip]               funcLeaf:2
+; CHECK:      28: lea edx, [rcx + 3]                     fib:2 @ funcLeaf:2
+; CHECK:      2b: cmp ecx, 3                             fib:2 @ funcLeaf:2
+; CHECK:      2e: cmovl edx, ecx                         fib:2 @ funcLeaf:2
+; CHECK:      31: sub eax, edx                           funcLeaf:2
+; CHECK:      33: ret                                    funcLeaf:3
+; CHECK:      34: nop word ptr cs:[rax + rax]
+; CHECK:      3e: nop
+; CHECK: <fib>:
+; CHECK:      40: lea eax, [rdi + 3]                     fib:2
+; CHECK:      43: cmp edi, 3                             fib:2
+; CHECK:      46: cmovl eax, edi                         fib:2
+; CHECK:      49: ret                                    fib:8
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@factor = dso_local global i32 3
+
+define dso_local i32 @funcA(i32 %x) !dbg !12 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %x, metadata !16, metadata !DIExpression()), !dbg !18
+  call void @llvm.dbg.value(metadata i32 %x, metadata !19, metadata !DIExpression()), !dbg !22
+  %0 = load volatile i32, i32* @factor, align 4, !dbg !24, !tbaa !25
+  call void @llvm.dbg.value(metadata i32 %0, metadata !29, metadata !DIExpression()), !dbg !32
+  %cmp.i.i = icmp slt i32 %0, 3, !dbg !34
+  %add.i.i = add nsw i32 %0, 3, !dbg !36
+  %retval.0.i.i = select i1 %cmp.i.i, i32 %0, i32 %add.i.i, !dbg !36
+  %sub.i = sub nsw i32 %x, %retval.0.i.i, !dbg !37
+  call void @llvm.dbg.value(metadata i32 %sub.i, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i32 %sub.i, metadata !17, metadata !DIExpression()), !dbg !18
+  ret i32 %sub.i, !dbg !38
+}
+
+define dso_local i32 @funcLeaf(i32 %x) !dbg !20 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %x, metadata !19, metadata !DIExpression()), !dbg !39
+  %0 = load volatile i32, i32* @factor, align 4, !dbg !40, !tbaa !25
+  call void @llvm.dbg.value(metadata i32 %0, metadata !29, metadata !DIExpression()), !dbg !41
+  %cmp.i = icmp slt i32 %0, 3, !dbg !43
+  %add.i = add nsw i32 %0, 3, !dbg !44
+  %retval.0.i = select i1 %cmp.i, i32 %0, i32 %add.i, !dbg !44
+  %sub = sub nsw i32 %x, %retval.0.i, !dbg !45
+  call void @llvm.dbg.value(metadata i32 %sub, metadata !19, metadata !DIExpression()), !dbg !39
+  ret i32 %sub, !dbg !46
+}
+
+define dso_local i32 @fib(i32 %x) !dbg !30 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %x, metadata !29, metadata !DIExpression()), !dbg !47
+  %cmp = icmp slt i32 %x, 3, !dbg !48
+  %add = add nsw i32 %x, 3, !dbg !49
+  %retval.0 = select i1 %cmp, i32 %x, i32 %add, !dbg !49
+  ret i32 %retval.0, !dbg !50
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!8, !9, !10}
+
+!1 = distinct !DIGlobalVariable(name: "factor", scope: !2, file: !3, line: 3, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, emissionKind: FullDebug)
+!3 = !DIFile(filename: "test.c", directory: "test")
+!4 = !{}
+!6 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{i32 7, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!12 = distinct !DISubprogram(name: "funcA", scope: !3, file: !3, line: 6, type: !13, scopeLine: 6, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !15)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!7, !7}
+!15 = !{!16, !17}
+!16 = !DILocalVariable(name: "x", arg: 1, scope: !12, file: !3, line: 6, type: !7)
+!17 = !DILocalVariable(name: "r", scope: !12, file: !3, line: 7, type: !7)
+!18 = !DILocation(line: 0, scope: !12)
+!19 = !DILocalVariable(name: "x", arg: 1, scope: !20, file: !3, line: 22, type: !7)
+!20 = distinct !DISubprogram(name: "funcLeaf", scope: !3, file: !3, line: 22, type: !13, scopeLine: 23, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !21)
+!21 = !{!19}
+!22 = !DILocation(line: 0, scope: !20, inlinedAt: !23)
+!23 = distinct !DILocation(line: 7, column: 11, scope: !12)
+!24 = !DILocation(line: 24, column: 12, scope: !20, inlinedAt: !23)
+!25 = !{!26, !26, i64 0}
+!26 = !{!"int", !27, i64 0}
+!27 = !{!"omnipotent char", !28, i64 0}
+!28 = !{!"Simple C/C++ TBAA"}
+!29 = !DILocalVariable(name: "x", arg: 1, scope: !30, file: !3, line: 11, type: !7)
+!30 = distinct !DISubprogram(name: "fib", scope: !3, file: !3, line: 11, type: !13, scopeLine: 12, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !31)
+!31 = !{!29}
+!32 = !DILocation(line: 0, scope: !30, inlinedAt: !33)
+!33 = distinct !DILocation(line: 24, column: 8, scope: !20, inlinedAt: !23)
+!34 = !DILocation(line: 13, column: 9, scope: !35, inlinedAt: !33)
+!35 = distinct !DILexicalBlock(scope: !30, file: !3, line: 13, column: 7)
+!36 = !DILocation(line: 13, column: 7, scope: !30, inlinedAt: !33)
+!37 = !DILocation(line: 24, column: 5, scope: !20, inlinedAt: !23)
+!38 = !DILocation(line: 8, column: 3, scope: !12)
+!39 = !DILocation(line: 0, scope: !20)
+!40 = !DILocation(line: 24, column: 12, scope: !20)
+!41 = !DILocation(line: 0, scope: !30, inlinedAt: !42)
+!42 = distinct !DILocation(line: 24, column: 8, scope: !20)
+!43 = !DILocation(line: 13, column: 9, scope: !35, inlinedAt: !42)
+!44 = !DILocation(line: 13, column: 7, scope: !30, inlinedAt: !42)
+!45 = !DILocation(line: 24, column: 5, scope: !20)
+!46 = !DILocation(line: 25, column: 3, scope: !20)
+!47 = !DILocation(line: 0, scope: !30)
+!48 = !DILocation(line: 13, column: 9, scope: !35)
+!49 = !DILocation(line: 13, column: 7, scope: !30)
+!50 = !DILocation(line: 19, column: 1, scope: !30)

llvm/tools/llvm-profgen/CMakeLists.txt

 include_directories(
   ${LLVM_MAIN_SRC_DIR}/lib/Target/X86
   ${LLVM_BINARY_DIR}/lib/Target/X86
   )
 set(LLVM_LINK_COMPONENTS
   AllTargetsDescs
   AllTargetsDisassemblers
   Core
   MC
   MCDisassembler
   Object
   Support
+  Symbolize
   )
 
 add_llvm_tool(llvm-profgen
   llvm-profgen.cpp
   ProfiledBinary.cpp
   )

llvm/tools/llvm-profgen/CallContext.h

+//===-- CallContext.h - Call Context Handler -----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_PROGEN_CALLCONTEXT_H
+#define LLVM_TOOLS_LLVM_PROGEN_CALLCONTEXT_H
+
+#include "llvm/ProfileData/SampleProf.h"
+#include <string>
+#include <vector>
+
+namespace llvm {
+namespace sampleprof {
+
+// Function name, LineLocation
+typedef std::pair<std::string, LineLocation> SourceLocation;
+
+// Caller, callee
+typedef SmallVector<SourceLocation, 4> SourceLocationVec;
+
+inline std::string getCallSite(const SourceLocation &Callsite) {
+  std::string CallsiteStr = Callsite.first;
+  CallsiteStr += ":";
+  CallsiteStr += Twine(Callsite.second.LineOffset).str();
+  if (Callsite.second.Discriminator > 0) {
+    CallsiteStr += ".";
+    CallsiteStr += Twine(Callsite.second.Discriminator).str();
+  }
+  return CallsiteStr;
+}
+
+// TODO: This operation is expansive. If it ever gets called multiple times we
+// may think of making a class wrapper with internal states for it.
+inline std::string getLocWithContext(const SourceLocationVec &Context) {
+  std::string ContextStr;
+  for (const auto &Callsite : Context) {
+    if (!ContextStr.empty())
+      ContextStr += " @ ";
+    ContextStr += getCallSite(Callsite);
+  }
+  return ContextStr;
+}
+
+// Reverse call context, i.e., in the order of callee frames to caller frames,
+// is useful during instruction printing or pseudo probe printing.
+inline std::string getReversedLocWithContext(const SourceLocationVec &Context) {
+  std::string ContextStr;
+  for (const auto &Callsite : reverse(Context)) {
+    if (!ContextStr.empty())
+      ContextStr += " @ ";
+    ContextStr += getCallSite(Callsite);
+  }
+  return ContextStr;
+}
+} // namespace sampleprof
+} // end namespace llvm
+
+#endif

llvm/tools/llvm-profgen/ProfiledBinary.h

 //===-- ProfiledBinary.h - Binary decoder -----------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TOOLS_LLVM_PROGEN_PROFILEDBINARY_H
 #define LLVM_TOOLS_LLVM_PROGEN_PROFILEDBINARY_H
 
+#include "CallContext.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/DebugInfo/Symbolize/Symbolize.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Support/Path.h"
 #include <set>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
 using namespace llvm::object;
 
 namespace llvm {
 namespace sampleprof {
 
+class ProfiledBinary;
+
+struct InstructionPointer {
+  ProfiledBinary *Binary;
+  // Offset to the base address of the executable segment of the binary.
+  uint64_t Offset;
+  // Index to the sorted code address array of the binary.
+  uint64_t Index;
+
+  InstructionPointer(ProfiledBinary *Binary, uint64_t Offset)
+      : Binary(Binary), Offset(Offset) {
+    Index = 0;
+  }
+};
+
 class ProfiledBinary {
   // Absolute path of the binary.
   std::string Path;
   // The target triple.
   Triple TheTriple;
   // The runtime base address that the executable sections are loaded at.
   uint64_t BaseAddress;
   // The preferred base address that the executable sections are loaded at.
   uint64_t PreferredBaseAddress;
-  // A list of text sections sorted by start RVA and size. Used to check
-  // if a given RVA is a valid code address.
+  // A list of text sections sorted by start offset and size. Used to check
+  // if a given offset is a valid code address.
   std::set<std::pair<uint64_t, uint64_t>> TextSections;
-  // Function RVA to name mapping.
+  // Function offset to name mapping.
   std::unordered_map<uint64_t, std::string> FuncStartAddrMap;
-  // An array of RVAs of all instructions sorted in increasing order. The
+  // An array of offsets of all instructions sorted in increasing order. The
   // sorting is needed to fast advance to the next forward/backward instruction.
   std::vector<uint64_t> CodeAddrs;
-  // A set of call instruction RVAs. Used by virtual unwinding.
+  // A set of call instruction offsets. Used by virtual unwinding.
   std::unordered_set<uint64_t> CallAddrs;
-  // A set of return instruction RVAs. Used by virtual unwinding.
+  // A set of return instruction offsets. Used by virtual unwinding.
   std::unordered_set<uint64_t> RetAddrs;
 
+  // The symbolizer used to get inline context for an instruction.
+  std::unique_ptr<symbolize::LLVMSymbolizer> Symbolizer;
+
   bool IsLoaded;
 
   void setPreferredBaseAddress(const ELFObjectFileBase *O);
 
+  void setupSymbolizer();
+
   /// Dissassemble the text section and build various address maps.
   void disassemble(const ELFObjectFileBase *O);
 
+  /// Symbolize a given instruction pointer and return a full call context.
+  SourceLocationVec symbolize(const InstructionPointer &I);
+
 public:
   ProfiledBinary() {}
-  ProfiledBinary(StringRef Path)
-      : Path(Path), BaseAddress(0), IsLoaded(false) {}
+  ProfiledBinary(StringRef Path) : Path(Path), BaseAddress(0), IsLoaded(false) {
+    setupSymbolizer();
+  }
 
   const StringRef getPath() const { return Path; }
   const StringRef getName() const { return llvm::sys::path::filename(Path); }
   uint64_t getBaseAddress() const { return BaseAddress; }
   void setBaseAddress(uint64_t Address) { BaseAddress = Address; }
 
   /// Decode the interesting parts of the binary and build internal data
   /// structures. On high level, the parts of interest are:

llvm/tools/llvm-profgen/ProfiledBinary.cpp

 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCInstrAnalysis.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/X86TargetParser.h"
 
 #define DEBUG_TYPE "load-binary"
 
 using namespace llvm;
 
-static cl::opt<bool> ShowDisassembly("show-disassembly", cl::Hidden,
+static cl::opt<bool> ShowDisassembly("show-disassembly", cl::ReallyHidden,
                                      cl::init(false), cl::ZeroOrMore,
                                      cl::desc("Print disassembled code."));
 
+static cl::opt<bool> ShowSourceLocations("show-source-locations",
+                                         cl::ReallyHidden, cl::init(false),
+                                         cl::ZeroOrMore,
+                                         cl::desc("Print source locations."));
+
 namespace llvm {
 namespace sampleprof {
 
 static const Target *getTarget(const ObjectFile *Obj) {
   Triple TheTriple = Obj->makeTriple();
   std::string Error;
   std::string ArchName;
   const Target *TheTarget =
void ProfiledBinary::disassemble(const ELFObjectFileBase *Obj) {
   // Dissassemble a text section.
   for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
        SI != SE; ++SI) {
     const SectionRef &Section = *SI;
     if (!Section.isText())
       continue;
 
     uint64_t ImageLoadAddr = PreferredBaseAddress;
-    uint64_t SectionRVA = Section.getAddress() - ImageLoadAddr;
+    uint64_t SectionOffset = Section.getAddress() - ImageLoadAddr;
     uint64_t SectSize = Section.getSize();
     if (!SectSize)
       continue;
 
     // Register the text section.
-    TextSections.insert({SectionRVA, SectSize});
+    TextSections.insert({SectionOffset, SectSize});
 
     if (ShowDisassembly) {
       StringRef SectionName = unwrapOrError(Section.getName(), FileName);
       outs() << "\nDisassembly of section " << SectionName;
-      outs() << " [" << format("0x%" PRIx64, SectionRVA) << ", "
-             << format("0x%" PRIx64, SectionRVA + SectSize) << "]:\n\n";
+      outs() << " [" << format("0x%" PRIx64, SectionOffset) << ", "
+             << format("0x%" PRIx64, SectionOffset + SectSize) << "]:\n\n";
     }
 
     // Get the section data.
     ArrayRef<uint8_t> Bytes =
         arrayRefFromStringRef(unwrapOrError(Section.getContents(), FileName));
 
     // Get the list of all the symbols in this section.
     SectionSymbolsTy &Symbols = AllSymbols[Section];
 
     // Disassemble symbol by symbol.
     for (unsigned SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
-      uint64_t StartRVA = Symbols[SI].Addr - ImageLoadAddr;
-      uint64_t EndRVA = (SI + 1 < SE) ? Symbols[SI + 1].Addr - ImageLoadAddr
-                                      : SectionRVA + SectSize;
-      if (StartRVA >= EndRVA)
+      uint64_t Start = Symbols[SI].Addr - ImageLoadAddr;
+      uint64_t End = (SI + 1 < SE) ? Symbols[SI + 1].Addr - ImageLoadAddr
+                                   : SectionOffset + SectSize;
+      if (Start >= End)
         continue;
 
       std::string SymbolName = Symbols[SI].Name.str();
       if (ShowDisassembly)
         outs() << '<' << SymbolName << ">:\n";
 
-      uint64_t RVA = StartRVA;
-      while (RVA < EndRVA) {
+      uint64_t Offset = Start;
+      while (Offset < End) {
         MCInst Inst;
         uint64_t Size;
         // Disassemble an instruction.
-        bool Disassembled =
-            DisAsm->getInstruction(Inst, Size, Bytes.slice(RVA - SectionRVA),
-                                   RVA + ImageLoadAddr, nulls());
+        bool Disassembled = DisAsm->getInstruction(
+            Inst, Size, Bytes.slice(Offset - SectionOffset),
+            Offset + ImageLoadAddr, nulls());
         if (Disassembled) {
           if (ShowDisassembly) {
-            outs() << format("%8" PRIx64 ":", RVA);
-            IP->printInst(&Inst, RVA + Size, "", *STI.get(), outs());
+            outs() << format("%8" PRIx64 ":", Offset);
+            size_t Start = outs().tell();
+            IP->printInst(&Inst, Offset + Size, "", *STI.get(), outs());
+            if (ShowSourceLocations) {
+              unsigned Cur = outs().tell() - Start;
+              if (Cur < 40)
+                outs().indent(40 - Cur);
+              InstructionPointer Inst(this, Offset);
+              outs() << getReversedLocWithContext(symbolize(Inst));
+            }
             outs() << "\n";
           }
           // Populate address maps.
-          CodeAddrs.push_back(RVA);
+          CodeAddrs.push_back(Offset);
           if (InstIsCall(Inst, TheTriple))
-            CallAddrs.insert(RVA);
+            CallAddrs.insert(Offset);
           else if (InstIsReturn(Inst, TheTriple))
-            RetAddrs.insert(RVA);
+            RetAddrs.insert(Offset);
         } else {
           exitWithError("disassembling error", FileName);
         }
 
-        RVA += Size;
+        Offset += Size;
       }
 
       if (ShowDisassembly)
         outs() << "\n";
 
-      FuncStartAddrMap[StartRVA] = Symbols[SI].Name.str();
-    }
+      FuncStartAddrMap[Start] = Symbols[SI].Name.str();
+    }
+  }
+}
+
+void ProfiledBinary::setupSymbolizer() {
+  symbolize::LLVMSymbolizer::Options SymbolizerOpts;
+  SymbolizerOpts.PrintFunctions =
+      DILineInfoSpecifier::FunctionNameKind::LinkageName;
+  SymbolizerOpts.Demangle = false;
+  SymbolizerOpts.DefaultArch = TheTriple.getArchName().str();
+  SymbolizerOpts.UseSymbolTable = false;
+  SymbolizerOpts.RelativeAddresses = false;
+  Symbolizer = std::make_unique<symbolize::LLVMSymbolizer>(SymbolizerOpts);
+
+SourceLocationVec ProfiledBinary::symbolize(const InstructionPointer &I) {
+  assert(this == I.Binary);
+  auto Addr = object::SectionedAddress{I.Offset + PreferredBaseAddress,
+                                       object::SectionedAddress::UndefSection};
+  DIInliningInfo InlineStack =
+      unwrapOrError(Symbolizer->symbolizeInlinedCode(Path, Addr), getName());
+
+  SourceLocationVec CallStack;
+
+  for (int i = InlineStack.getNumberOfFrames() - 1; i >= 0; i--) {
+    const auto &CallerFrame = InlineStack.getFrame(i);
+    if (CallerFrame.FunctionName == "<invalid>")
+      break;
+    LineLocation Line(CallerFrame.Line - CallerFrame.StartLine,
+                      CallerFrame.Discriminator);
+    SourceLocation Callsite(CallerFrame.FunctionName, Line);
+    CallStack.push_back(Callsite);
   }
+
+  return CallStack;
 }
 } // namespace sampleprof
 } // end namespace llvm

llvm/tools/llvm-profgen/llvm-profgen.cpp

static cl::list<std::string>
     BinaryFilenames("binary", cl::value_desc("binary"), cl::OneOrMore,
                     llvm::cl::MiscFlags::CommaSeparated,
                     cl::desc("Input profiled binary files"));
 
 static cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
                                            cl::Required,
                                            cl::desc("Output profile file"));
 
-static cl::opt<bool> ShowMmapEvents("show-mmap-events", cl::Hidden,
+static cl::opt<bool> ShowMmapEvents("show-mmap-events", cl::ReallyHidden,
                                     cl::init(false), cl::ZeroOrMore,
                                     cl::desc("Print binary load events."));
 
 namespace llvm {
 namespace sampleprof {
 
 using BinaryMap = StringMap<ProfiledBinary>;
 using BinaryAddressMap = std::map<uint64_t, ProfiledBinary *>;