[InstrProf][llvm-profdata] Dump profile correlation data as YAML

[ellis]

Summary

Change the behavior of the llvm-profdata show --debug-info= command to dump a YAML file when using debug info correlation since it provides more information in a parseable format.

[wenlei]

cc @xur

[phosek]

Could we make this selectable? We already have the --json option in llvm-profdata for sample profile data. Could we replace it with a --output-format=[json|yaml] option and extend it to support other output types, including debug info correlation. You could then use llvm-profdata show --debug-info=%t --output-format=yaml.

[MaskRay]

Adding --output-format=[json|yaml] looks good to me.

[ellis]

I like the idea of --output-format=yaml since it easily allows us to preserve the original behavior. I'll add this in another diff and update this.

[ellis]

Add the "Probes" key to Yaml output so that we can add more keys in the future. Also add D135127 as a parent diff to use the --output-format=yaml option.

[ellis]

Rebase.

[phosek]

LGTM

[ellis]

Rebase

compiler-rt/test/profile/Linux/instrprof-show-debug-info-correlation.c

 // RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %s
 // RUN: llvm-profdata show --debug-info=%t --detailed-summary --show-prof-sym-list | FileCheck %s
+// RUN: llvm-profdata show --debug-info=%t --output-format=yaml | FileCheck %s --match-full-lines --check-prefix YAML
 
 // RUN: %clang_pgogen -o %t.no.dbg -mllvm --debug-info-correlate -mllvm --disable-vp=true %s
 // RUN: not llvm-profdata show --debug-info=%t.no.dbg 2>&1 | FileCheck %s --check-prefix NO-DBG
 // NO-DBG: unable to correlate profile: could not find any profile metadata in debug info
 
+// CHECK: a
+// YAML: Probes:
+// YAML:   - Function Name:   a
+// YAML:     Linkage Name:    a
+// YAML:     CFG Hash:        0x[[#%.1X,HASH:]]
+// YAML:     Counter Offset:  0x0
+// YAML:     Num Counters:    1
+// YAML:     File:            [[FILE:'.*']]
+// YAML:     Line:            [[@LINE+1]]
 void a() {}
-void b() {}
-int main() { return 0; }
 
-// CHECK: a
 // CHECK: b
+// YAML:   - Function Name:   b
+// YAML:     Linkage Name:    b
+// YAML:     CFG Hash:        0x[[#%.1X,HASH:]]
+// YAML:     Counter Offset:  0x8
+// YAML:     Num Counters:    1
+// YAML:     File:            [[FILE:'.*']]
+// YAML:     Line:            [[@LINE+1]]
+void b() {}
+
 // CHECK: main
+// YAML:   - Function Name:   main
+// YAML:     Linkage Name:    main
+// YAML:     CFG Hash:        0x[[#%.1X,HASH:]]
+// YAML:     Counter Offset:  0x10
+// YAML:     Num Counters:    1
+// YAML:     File:            [[FILE]]
+// YAML:     Line:            [[@LINE+1]]
+int main() { return 0; }
+
 // CHECK: Counters section size: 0x18 bytes
 // CHECK: Found 3 functions

llvm/include/llvm/ProfileData/InstrProfCorrelator.h

 
 #ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
 #define LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
 
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/YAMLTraits.h"
 #include <vector>
 
 namespace llvm {
 class DWARFContext;
 class DWARFDie;
 namespace object {
 class ObjectFile;
 }
 
 /// InstrProfCorrelator - A base class used to create raw instrumentation data
 /// to their functions.
 class InstrProfCorrelator {
 public:
   static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
   get(StringRef DebugInfoFilename);
 
   /// Construct a ProfileData vector used to correlate raw instrumentation data
   /// to their functions.
   virtual Error correlateProfileData() = 0;
 
+  /// Process debug info and dump the correlation data.
+  virtual Error dumpYaml(raw_ostream &OS) = 0;
+
   /// Return the number of ProfileData elements.
   llvm::Optional<size_t> getDataSize() const;
 
   /// Return a pointer to the names string that this class constructs.
   const char *getNamesPointer() const { return Names.c_str(); }
 
   /// Return the number of bytes in the names string.
   size_t getNamesSize() const { return Names.size(); }
struct Context {
     get(std::unique_ptr<MemoryBuffer> Buffer, const object::ObjectFile &Obj);
     std::unique_ptr<MemoryBuffer> Buffer;
     /// The address range of the __llvm_prf_cnts section.
     uint64_t CountersSectionStart;
     uint64_t CountersSectionEnd;
     /// True if target and host have different endian orders.
     bool ShouldSwapBytes;
   };
-  const std::unique_ptr<InstrProfCorrelator::Context> Ctx;
+  const std::unique_ptr<Context> Ctx;
 
   InstrProfCorrelator(InstrProfCorrelatorKind K, std::unique_ptr<Context> Ctx)
       : Ctx(std::move(Ctx)), Kind(K) {}
 
   std::string Names;
   std::vector<std::string> NamesVec;
 
+  struct Probe {
+    std::string FunctionName;
+    Optional<std::string> LinkageName;
+    yaml::Hex64 CFGHash;
+    yaml::Hex64 CounterOffset;
+    uint32_t NumCounters;
+    Optional<std::string> FilePath;
+    Optional<int> LineNumber;
+  };
+
+  struct CorrelationData {
+    std::vector<Probe> Probes;
+  };
+
+  friend struct yaml::MappingTraits<Probe>;
+  friend struct yaml::SequenceElementTraits<Probe>;
+  friend struct yaml::MappingTraits<CorrelationData>;
+
 private:
   static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
   get(std::unique_ptr<MemoryBuffer> Buffer);
 
   const InstrProfCorrelatorKind Kind;
 };
 
 /// InstrProfCorrelatorImpl - A child of InstrProfCorrelator with a template
public:
   static llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
   get(std::unique_ptr<InstrProfCorrelator::Context> Ctx,
       const object::ObjectFile &Obj);
 
 protected:
   std::vector<RawInstrProf::ProfileData<IntPtrT>> Data;
 
   Error correlateProfileData() override;
-  virtual void correlateProfileDataImpl() = 0;
+  virtual void correlateProfileDataImpl(
+      InstrProfCorrelator::CorrelationData *Data = nullptr) = 0;
+
+  Error dumpYaml(raw_ostream &OS) override;
 
   void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset,
                 IntPtrT FunctionPtr, uint32_t NumCounters);
 
 private:
   InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind,
                           std::unique_ptr<InstrProfCorrelator::Context> Ctx)
       : InstrProfCorrelator(Kind, std::move(Ctx)){};
private:
   ///         DW_AT_name  ("CFG Hash")
   ///         DW_AT_const_value (12345678)
   ///       DW_TAG_LLVM_annotation
   ///         DW_AT_name  ("Num Counters")
   ///         DW_AT_const_value (2)
   ///       NULL
   ///     NULL
   /// \endcode
-  void correlateProfileDataImpl() override;
+  void correlateProfileDataImpl(
+      InstrProfCorrelator::CorrelationData *Data = nullptr) override;
 };
 
 } // end namespace llvm
 
 #endif // LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H

llvm/lib/ProfileData/InstrProfCorrelator.cpp

return make_error<InstrProfError>(
         "could not find any profile metadata in debug info");
   auto Result =
       collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
   CounterOffsets.clear();
   NamesVec.clear();
   return Result;
 }
 
+template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {
+  static void mapping(yaml::IO &io,
+                      InstrProfCorrelator::CorrelationData &Data) {
+    io.mapRequired("Probes", Data.Probes);
+  }
+};
+
+template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
+  static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
+    io.mapRequired("Function Name", P.FunctionName);
+    io.mapOptional("Linkage Name", P.LinkageName);
+    io.mapRequired("CFG Hash", P.CFGHash);
+    io.mapRequired("Counter Offset", P.CounterOffset);
+    io.mapRequired("Num Counters", P.NumCounters);
+    io.mapOptional("File", P.FilePath);
+    io.mapOptional("Line", P.LineNumber);
+  }
+};
+
+template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
+  static const bool flow = false;
+};
+
+template <class IntPtrT>
+Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(raw_ostream &OS) {
+  InstrProfCorrelator::CorrelationData Data;
+  correlateProfileDataImpl(&Data);
+  if (Data.Probes.empty())
+    return make_error<InstrProfError>(
+        instrprof_error::unable_to_correlate_profile,
+        "could not find any profile metadata in debug info");
+  yaml::Output YamlOS(OS);
+  YamlOS << Data;
+  return Error::success();
+}
+
 template <class IntPtrT>
 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
                                                 uint64_t CFGHash,
                                                 IntPtrT CounterOffset,
                                                 IntPtrT FunctionPtr,
                                                 uint32_t NumCounters) {
   // Check if a probe was already added for this counter offset.
   if (!CounterOffsets.insert(CounterOffset).second)
bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
   if (!Die.hasChildren())
     return false;
   if (const char *Name = Die.getName(DINameKind::ShortName))
     return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
   return false;
 }
 
 template <class IntPtrT>
-void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() {
+void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
+    InstrProfCorrelator::CorrelationData *Data) {
   auto maybeAddProbe = [&](DWARFDie Die) {
     if (!isDIEOfProbe(Die))
       return;
     Optional<const char *> FunctionName;
     Optional<uint64_t> CFGHash;
     Optional<uint64_t> CounterPtr = getLocation(Die);
-    auto FunctionPtr =
-        dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc));
+    auto FnDie = Die.getParent();
+    auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
     Optional<uint64_t> NumCounters;
     for (const DWARFDie &Child : Die.children()) {
       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
         continue;
       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
       if (!AnnotationFormName || !AnnotationFormValue)
         continue;
if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
       LLVM_DEBUG(Die.dump(dbgs()));
       return;
     }
     if (!FunctionPtr) {
       LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
                         << "\n");
       LLVM_DEBUG(Die.dump(dbgs()));
     }
-    this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart,
+    IntPtrT CounterOffset = *CounterPtr - CountersStart;
+    if (Data) {
+      InstrProfCorrelator::Probe P;
+      P.FunctionName = *FunctionName;
+      if (auto Name = FnDie.getName(DINameKind::LinkageName))
+        P.LinkageName = Name;
+      P.CFGHash = *CFGHash;
+      P.CounterOffset = CounterOffset;
+      P.NumCounters = *NumCounters;
+      auto FilePath = FnDie.getDeclFile(
+          DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
+      if (!FilePath.empty())
+        P.FilePath = FilePath;
+      if (auto LineNumber = FnDie.getDeclLine())
+        P.LineNumber = LineNumber;
+      Data->Probes.push_back(P);
+    } else {
+      this->addProbe(*FunctionName, *CFGHash, CounterOffset,
                      FunctionPtr.value_or(0), *NumCounters);
+    }
   };
   for (auto &CU : DICtx->normal_units())
     for (const auto &Entry : CU->dies())
       maybeAddProbe(DWARFDie(CU.get(), &Entry));
   for (auto &CU : DICtx->dwo_units())
     for (const auto &Entry : CU->dies())
       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 }

llvm/tools/llvm-profdata/llvm-profdata.cpp

 }
 
 static int showDebugInfoCorrelation(const std::string &Filename,
                                     bool ShowDetailedSummary,
                                     bool ShowProfileSymbolList,
                                     OutputFormat OFormat, raw_fd_ostream &OS) {
   if (OFormat == OutputFormat::Json)
     exitWithError("JSON output is not supported for debug info correlation");
-  if (OFormat == OutputFormat::Yaml)
-    exitWithError("YAML output is not supported for debug info correlation");
   std::unique_ptr<InstrProfCorrelator> Correlator;
   if (auto Err = InstrProfCorrelator::get(Filename).moveInto(Correlator))
     exitWithError(std::move(Err), Filename);
+  if (OFormat == OutputFormat::Yaml) {
+    if (auto Err = Correlator->dumpYaml(OS))
+      exitWithError(std::move(Err), Filename);
+    return 0;
+  }
 
   if (auto Err = Correlator->correlateProfileData())
     exitWithError(std::move(Err), Filename);
 
   InstrProfSymtab Symtab;
   if (auto Err = Symtab.create(
           StringRef(Correlator->getNamesPointer(), Correlator->getNamesSize())))
     exitWithError(std::move(Err), Filename);