[InstrProf][llvm-profdata] Dump profile correlation data as YAML

[ellis]

Summary

Change the behavior of the llvm-profdata show --debug-info= command to dump a YAML file when using debug info correlation since it provides more information in a parseable format.

[wenlei]

cc @xur

[phosek]

Could we make this selectable? We already have the --json option in llvm-profdata for sample profile data. Could we replace it with a --output-format=[json|yaml] option and extend it to support other output types, including debug info correlation. You could then use llvm-profdata show --debug-info=%t --output-format=yaml.

[MaskRay]

Adding --output-format=[json|yaml] looks good to me.

[ellis]

I like the idea of --output-format=yaml since it easily allows us to preserve the original behavior. I'll add this in another diff and update this.

[ellis]

Add the "Probes" key to Yaml output so that we can add more keys in the future. Also add D135127 as a parent diff to use the --output-format=yaml option.

[ellis]

Rebase.

[phosek]

LGTM

[ellis]

Rebase

compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c

 // RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %s
-// RUN: llvm-profdata show --debug-info=%t --detailed-summary --show-prof-sym-list | FileCheck %s
+// RUN: llvm-profdata show --debug-info=%t | FileCheck %s --match-full-lines
 
 // RUN: %clang_pgogen -o %t.no.dbg -mllvm --debug-info-correlate -mllvm --disable-vp=true %s
 // RUN: not llvm-profdata show --debug-info=%t.no.dbg 2>&1 | FileCheck %s --check-prefix NO-DBG
 // NO-DBG: unable to correlate profile: could not find any profile metadata in debug info
 
+// CHECK: - Function Name:   a
+// CHECK:   Linkage Name:    a
+// CHECK:   CFG Hash:        0x[[#%.1X,HASH:]]
+// CHECK:   Counter Offset:  0x0
+// CHECK:   Num Counters:    1
+// CHECK:   File:            [[FILE:'.*']]
+// CHECK:   Line:            [[@LINE+1]]
 void a() {}
-void b() {}
-int main() { return 0; }
 
-// CHECK: a
-// CHECK: b
-// CHECK: main
-// CHECK: Counters section size: 0x18 bytes
-// CHECK: Found 3 functions
+// CHECK: - Function Name:   main
+// CHECK:   Linkage Name:    main
+// CHECK:   CFG Hash:        0x[[#%.1X,HASH:]]
+// CHECK:   Counter Offset:  0x8
+// CHECK:   Num Counters:    1
+// CHECK:   File:            [[FILE]]
+// CHECK:   Line:            [[@LINE+1]]
+int main() { return 0; }

llvm/include/llvm/ProfileData/InstrProfCorrelator.h

 
 #ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
 #define LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
 
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/YAMLTraits.h"
 #include <vector>
 
 namespace llvm {
 class DWARFContext;
 class DWARFDie;
 namespace object {
 class ObjectFile;
 }
 
 /// InstrProfCorrelator - A base class used to create raw instrumentation data
 /// to their functions.
 class InstrProfCorrelator {
 public:
   static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
   get(StringRef DebugInfoFilename);
 
   /// Construct a ProfileData vector used to correlate raw instrumentation data
   /// to their functions.
   virtual Error correlateProfileData() = 0;
 
+  /// Process debug info and dump the correlation data.
+  virtual Error dumpYaml(raw_ostream &OS) = 0;
+
   /// Return the number of ProfileData elements.
   llvm::Optional<size_t> getDataSize() const;
 
   /// Return a pointer to the names string that this class constructs.
   const char *getNamesPointer() const { return Names.c_str(); }
 
   /// Return the number of bytes in the names string.
   size_t getNamesSize() const { return Names.size(); }
struct Context {
     get(std::unique_ptr<MemoryBuffer> Buffer, const object::ObjectFile &Obj);
     std::unique_ptr<MemoryBuffer> Buffer;
     /// The address range of the __llvm_prf_cnts section.
     uint64_t CountersSectionStart;
     uint64_t CountersSectionEnd;
     /// True if target and host have different endian orders.
     bool ShouldSwapBytes;
   };
-  const std::unique_ptr<InstrProfCorrelator::Context> Ctx;
+  const std::unique_ptr<Context> Ctx;
 
   InstrProfCorrelator(InstrProfCorrelatorKind K, std::unique_ptr<Context> Ctx)
       : Ctx(std::move(Ctx)), Kind(K) {}
 
   std::string Names;
   std::vector<std::string> NamesVec;
 
+  struct Probe {
+    std::string FunctionName;
+    Optional<std::string> LinkageName;
+    yaml::Hex64 CFGHash;
+    yaml::Hex64 CounterOffset;
+    uint32_t NumCounters;
+    Optional<std::string> FilePath;
+    Optional<int> LineNumber;
+  };
+
+  friend struct yaml::MappingTraits<Probe>;
+  friend struct yaml::SequenceElementTraits<Probe>;
+
 private:
   static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
   get(std::unique_ptr<MemoryBuffer> Buffer);
 
   const InstrProfCorrelatorKind Kind;
 };
 
 /// InstrProfCorrelatorImpl - A child of InstrProfCorrelator with a template
public:
   static llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
   get(std::unique_ptr<InstrProfCorrelator::Context> Ctx,
       const object::ObjectFile &Obj);
 
 protected:
   std::vector<RawInstrProf::ProfileData<IntPtrT>> Data;
 
   Error correlateProfileData() override;
-  virtual void correlateProfileDataImpl() = 0;
+  virtual void correlateProfileDataImpl(
+      std::vector<InstrProfCorrelator::Probe> *Probes = nullptr) = 0;
+
+  Error dumpYaml(raw_ostream &OS) override;
 
   void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset,
                 IntPtrT FunctionPtr, uint32_t NumCounters);
 
 private:
   InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind,
                           std::unique_ptr<InstrProfCorrelator::Context> Ctx)
       : InstrProfCorrelator(Kind, std::move(Ctx)){};
private:
   ///         DW_AT_name  ("CFG Hash")
   ///         DW_AT_const_value (12345678)
   ///       DW_TAG_LLVM_annotation
   ///         DW_AT_name  ("Num Counters")
   ///         DW_AT_const_value (2)
   ///       NULL
   ///     NULL
   /// \endcode
-  void correlateProfileDataImpl() override;
+  void correlateProfileDataImpl(
+      std::vector<InstrProfCorrelator::Probe> *Probes = nullptr) override;
 };
 
 } // end namespace llvm
 
 #endif // LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H

llvm/lib/ProfileData/InstrProfCorrelator.cpp

return make_error<InstrProfError>(
         "could not find any profile metadata in debug info");
   auto Result =
       collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
   CounterOffsets.clear();
   NamesVec.clear();
   return Result;
 }
 
+template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
+  static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
+    io.mapRequired("Function Name", P.FunctionName);
+    io.mapOptional("Linkage Name", P.LinkageName);
+    io.mapRequired("CFG Hash", P.CFGHash);
+    io.mapRequired("Counter Offset", P.CounterOffset);
+    io.mapRequired("Num Counters", P.NumCounters);
+    io.mapOptional("File", P.FilePath);
+    io.mapOptional("Line", P.LineNumber);
+  }
+};
+
+template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
+  static const bool flow = false;
+};
+
+template <class IntPtrT>
+Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(raw_ostream &OS) {
+  std::vector<InstrProfCorrelator::Probe> Probes;
+  correlateProfileDataImpl(&Probes);
+  if (Probes.empty())
+    return make_error<InstrProfError>(
+        instrprof_error::unable_to_correlate_profile,
+        "could not find any profile metadata in debug info");
+  yaml::Output YamlOS(OS);
+  YamlOS << Probes;
+  return Error::success();
+}
+
 template <class IntPtrT>
 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
                                                 uint64_t CFGHash,
                                                 IntPtrT CounterOffset,
                                                 IntPtrT FunctionPtr,
                                                 uint32_t NumCounters) {
   // Check if a probe was already added for this counter offset.
   if (!CounterOffsets.insert(CounterOffset).second)
bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
   if (!Die.hasChildren())
     return false;
   if (const char *Name = Die.getName(DINameKind::ShortName))
     return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
   return false;
 }
 
 template <class IntPtrT>
-void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() {
+void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
+    std::vector<InstrProfCorrelator::Probe> *Probes) {
   auto maybeAddProbe = [&](DWARFDie Die) {
     if (!isDIEOfProbe(Die))
       return;
     Optional<const char *> FunctionName;
     Optional<uint64_t> CFGHash;
     Optional<uint64_t> CounterPtr = getLocation(Die);
-    auto FunctionPtr =
-        dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc));
+    auto FnDie = Die.getParent();
+    auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
     Optional<uint64_t> NumCounters;
     for (const DWARFDie &Child : Die.children()) {
       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
         continue;
       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
       if (!AnnotationFormName || !AnnotationFormValue)
         continue;
if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
       LLVM_DEBUG(Die.dump(dbgs()));
       return;
     }
     if (!FunctionPtr) {
       LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
                         << "\n");
       LLVM_DEBUG(Die.dump(dbgs()));
     }
-    this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart,
+    IntPtrT CounterOffset = *CounterPtr - CountersStart;
+    if (Probes) {
+      InstrProfCorrelator::Probe P;
+      P.FunctionName = *FunctionName;
+      if (auto Name = FnDie.getName(DINameKind::LinkageName))
+        P.LinkageName = Name;
+      P.CFGHash = *CFGHash;
+      P.CounterOffset = CounterOffset;
+      P.NumCounters = *NumCounters;
+      auto FilePath = FnDie.getDeclFile(
+          DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
+      if (!FilePath.empty())
+        P.FilePath = FilePath;
+      if (auto LineNumber = FnDie.getDeclLine())
+        P.LineNumber = LineNumber;
+      Probes->push_back(P);
+    } else {
+      this->addProbe(*FunctionName, *CFGHash, CounterOffset,
                      FunctionPtr.value_or(0), *NumCounters);
+    }
   };
   for (auto &CU : DICtx->normal_units())
     for (const auto &Entry : CU->dies())
       maybeAddProbe(DWARFDie(CU.get(), &Entry));
   for (auto &CU : DICtx->dwo_units())
     for (const auto &Entry : CU->dies())
       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 }

llvm/tools/llvm-profdata/llvm-profdata.cpp

static int showMemProfProfile(const std::string &Filename,
   std::unique_ptr<llvm::memprof::RawMemProfReader> Reader(
       ReaderOr.get().release());
 
   Reader->printYAML(OS);
   return 0;
 }
 
 static int showDebugInfoCorrelation(const std::string &Filename,
-                                    bool ShowDetailedSummary,
-                                    bool ShowProfileSymbolList,
                                     raw_fd_ostream &OS) {
   std::unique_ptr<InstrProfCorrelator> Correlator;
   if (auto Err = InstrProfCorrelator::get(Filename).moveInto(Correlator))
     exitWithError(std::move(Err), Filename);
-  if (auto Err = Correlator->correlateProfileData())
+  if (auto Err = Correlator->dumpYaml(OS))
     exitWithError(std::move(Err), Filename);
-
-  InstrProfSymtab Symtab;
-  if (auto Err = Symtab.create(
-          StringRef(Correlator->getNamesPointer(), Correlator->getNamesSize())))
-    exitWithError(std::move(Err), Filename);
-
-  if (ShowProfileSymbolList)
-    Symtab.dumpNames(OS);
-  // TODO: Read "Profile Data Type" from debug info to compute and show how many
-  // counters the section holds.
-  if (ShowDetailedSummary)
-    OS << "Counters section size: 0x"
-       << Twine::utohexstr(Correlator->getCountersSectionSize()) << " bytes\n";
-  OS << "Found " << Correlator->getDataSize() << " functions\n";
-
   return 0;
 }
 
 static int show_main(int argc, const char *argv[]) {
   cl::opt<std::string> Filename(cl::Positional, cl::desc("<profdata-file>"));
 
   cl::opt<bool> ShowCounts("counts", cl::init(false),
                            cl::desc("Show counter values for shown functions"));
static int show_main(int argc, const char *argv[]) {
   raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
   if (EC)
     exitWithErrorCode(EC, OutputFilename);
 
   if (ShowAllFunctions && !ShowFunction.empty())
     WithColor::warning() << "-function argument ignored: showing all functions\n";
 
   if (!DebugInfoFilename.empty())
-    return showDebugInfoCorrelation(DebugInfoFilename, ShowDetailedSummary,
-                                    ShowProfileSymbolList, OS);
+    return showDebugInfoCorrelation(DebugInfoFilename, OS);
 
   if (ProfileKind == instr)
     return showInstrProfile(
         Filename, ShowCounts, TopNFunctions, ShowIndirectCallTargets,
         ShowMemOPSizes, ShowDetailedSummary, DetailedSummaryCutoffs,
         ShowAllFunctions, ShowCS, ValueCutoff, OnlyListBelow, ShowFunction,
         TextFormat, ShowBinaryIds, ShowCovered, OS);
   if (ProfileKind == sample)