[dsymutil] Upstream emitting of papertrail warnings.

[JDevlieghere]

Summary

When running dsymutil as part of your build system, it can be desirable for warnings to be part of the end product, rather than just being emitted to the output stream. This patch upstreams that functionality.

[aprantl]

Might be nice to also have this available as a command line option.

llvm/trunk/docs/CommandGuide/dsymutil.rst

 
  Specifies an alternate ``path`` to place the dSYM bundle. The default dSYM
  bundle path is created by appending ``.dSYM`` to the executable name.
 
 .. option:: --oso-prepend-path=<path>
 
  Specifies a ``path`` to prepend to all debug symbol object file paths.
 
+.. option:: --papertrail
+
+ When running dsymutil as part of your build system, it can be desirable for
+ warnings to be part of the end product, rather than just being emitted to the
+ output stream. When enabled warnings are embedded in the linked DWARF debug
+ information.
+
 .. option:: -s, --symtab
 
  Dumps the symbol table found in *executable* or object file(s) and exits.
 
 .. option:: --toolchain
 
  Embed the toolchain in the dSYM bundle's property list.
 

llvm/trunk/lib/CodeGen/AsmPrinter/DIE.cpp

 ///
 void DIEBlock::EmitValue(const AsmPrinter *Asm, dwarf::Form Form) const {
   switch (Form) {
   default: llvm_unreachable("Improper form for block");
   case dwarf::DW_FORM_block1: Asm->emitInt8(Size);    break;
   case dwarf::DW_FORM_block2: Asm->emitInt16(Size);   break;
   case dwarf::DW_FORM_block4: Asm->emitInt32(Size);   break;
   case dwarf::DW_FORM_block:  Asm->EmitULEB128(Size); break;
+  case dwarf::DW_FORM_string: break;
   case dwarf::DW_FORM_data16: break;
   }
 
   for (const auto &V : values())
     V.EmitValue(Asm);
 }
 
 /// SizeOf - Determine size of block data in bytes.

llvm/trunk/test/tools/dsymutil/X86/papertrail-warnings.test

+RUN: RC_DEBUG_OPTIONS=1 dsymutil -f %p/../Inputs/basic.macho.x86_64 -o - | llvm-dwarfdump -v - | FileCheck %s
+
+CHECK: .debug_info contents:
+CHECK:  Compile Unit:
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer {{.*}}"dsymutil
+CHECK:    DW_AT_name {{.*}}"/Inputs/basic1.macho.x86_64.o"
+CHECK:    DW_TAG_constant [2]
+CHECK:      DW_AT_name {{.*}}"dsymutil_warning"
+CHECK:      DW_AT_artificial [DW_FORM_flag] (0x01)
+CHECK:      DW_AT_const_value {{.*}}"unable to open object file: No such file or directory"
+CHECK:    NULL
+CHECK:  Compile Unit:
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer {{.*}}"dsymutil
+CHECK:    DW_AT_name {{.*}}"/Inputs/basic2.macho.x86_64.o"
+CHECK:    DW_TAG_constant [2]
+CHECK:      DW_AT_name {{.*}}"dsymutil_warning"
+CHECK:      DW_AT_artificial [DW_FORM_flag] (0x01)
+CHECK:      DW_AT_const_value {{.*}}"unable to open object file: No such file or directory"
+CHECK:    NULL
+CHECK:  Compile Unit:
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer {{.*}}"dsymutil
+CHECK:    DW_AT_name {{.*}}"/Inputs/basic3.macho.x86_64.o"
+CHECK:    DW_TAG_constant [2]
+CHECK:      DW_AT_name {{.*}}"dsymutil_warning"
+CHECK:      DW_AT_artificial [DW_FORM_flag] (0x01)
+CHECK:      DW_AT_const_value {{.*}}"unable to open object file: No such file or directory"
+CHECK:    NULL
 No newline at end of file

llvm/trunk/test/tools/dsymutil/cmdline.test

 RUN: dsymutil -help 2>&1 | FileCheck --check-prefix=HELP %s
 HELP: OVERVIEW: manipulate archived DWARF debug symbol files.
 HELP: USAGE: dsymutil{{[^ ]*}} [options] <input files>
 HELP-NOT: -reverse-iterate
 HELP: Specific Options:
 HELP: -arch=<arch>
 HELP: -dump-debug-map
 HELP: -flat
 HELP: -minimize
 HELP: -no-odr
 HELP: -no-output
 HELP: -no-swiftmodule-timestamp
 HELP: -num-threads=<n>
 HELP: -o=<filename>
 HELP: -oso-prepend-path=<path>
+HELP: -papertrail
 HELP: -symtab
 HELP: -toolchain
 HELP: -update
 HELP: -verbose
 HELP: -verify
 HELP: -y
 HELP-NOT: -reverse-iterate
 
 RUN: dsymutil --version 2>&1 | FileCheck --check-prefix=VERSION %s
 VERSION: {{ version }}

llvm/trunk/test/tools/dsymutil/debug-map-parsing.test

 NOT-FOUND: warning: (x86_64) {{.*}}/Inputs/basic1.macho.x86_64.o unable to open object file: {{[Nn]o}} such file
 NOT-FOUND: warning: (x86_64) {{.*}}/Inputs/basic2.macho.x86_64.o unable to open object file: {{[Nn]o}} such file
 NOT-FOUND: warning: (x86_64) {{.*}}/Inputs/basic3.macho.x86_64.o unable to open object file: {{[Nn]o}} such file
 NOT-FOUND: ---
 NOT-FOUND-NEXT: triple: 'x86_64-apple-darwin'
 NOT-FOUND-NEXT: binary-path:{{.*}}/Inputs/basic.macho.x86_64
 NOT-FOUND-NEXT: ...
 
-Check that we correctly error out on invalid executatble.
+Check that we correctly error out on invalid executable.
 
 NO-EXECUTABLE: cannot parse{{.*}}/inexistant': {{[Nn]o}} such file
 NO-EXECUTABLE-NOT: ---

llvm/trunk/tools/dsymutil/DebugMap.h

public:
   }
 
   uint8_t getType() const { return Type; }
 
   iterator_range<StringMap<SymbolMapping>::const_iterator> symbols() const {
     return make_range(Symbols.begin(), Symbols.end());
   }
 
+  bool empty() const { return Symbols.empty(); }
+
+  void addWarning(StringRef Warning) { Warnings.push_back(Warning); }
+  const std::vector<std::string> &getWarnings() const { return Warnings; }
+
   void print(raw_ostream &OS) const;
 #ifndef NDEBUG
   void dump() const;
 #endif
 
 private:
   friend class DebugMap;
 
   /// DebugMapObjects can only be constructed by the owning DebugMap.
   DebugMapObject(StringRef ObjectFilename,
                  sys::TimePoint<std::chrono::seconds> Timestamp, uint8_t Type);
 
   std::string Filename;
   sys::TimePoint<std::chrono::seconds> Timestamp;
   StringMap<SymbolMapping> Symbols;
   DenseMap<uint64_t, DebugMapEntry *> AddressToMapping;
   uint8_t Type;
 
+  std::vector<std::string> Warnings;
+
   /// For YAMLIO support.
   ///@{
   friend yaml::MappingTraits<dsymutil::DebugMapObject>;
   friend yaml::SequenceTraits<std::vector<std::unique_ptr<DebugMapObject>>>;
 
   DebugMapObject() = default;
 
 public:

llvm/trunk/tools/dsymutil/DwarfLinker.cpp

 
 private:
   /// Remembers the newest DWARF version we've seen in a unit.
   void maybeUpdateMaxDwarfVersion(unsigned Version) {
     if (MaxDwarfVersion < Version)
       MaxDwarfVersion = Version;
   }
 
+  /// Emit warnings as Dwarf compile units to leave a trail after linking.
+  bool emitPaperTrailWarnings(const DebugMapObject &DMO, const DebugMap &Map,
+                              OffsetsStringPool &StringPool);
+
   /// Keeps track of relocations.
   class RelocationManager {
     struct ValidReloc {
       uint32_t Offset;
       uint32_t Size;
       uint64_t Addend;
       const DebugMapObject::DebugMapEntry *Mapping;
 
for (auto &CurrentUnit : CompileUnits) {
     CurrentUnit->fixupForwardReferences();
     Linker.Streamer->emitCompileUnitHeader(*CurrentUnit);
     if (!CurrentUnit->getOutputUnitDIE())
       continue;
     Linker.Streamer->emitDIE(*CurrentUnit->getOutputUnitDIE());
   }
 }
 
+bool DwarfLinker::emitPaperTrailWarnings(const DebugMapObject &DMO,
+                                         const DebugMap &Map,
+                                         OffsetsStringPool &StringPool) {
+  if (DMO.getWarnings().empty() || !DMO.empty())
+    return false;
+
+  Streamer->switchToDebugInfoSection(/* Version */ 2);
+  DIE *CUDie = DIE::get(DIEAlloc, dwarf::DW_TAG_compile_unit);
+  CUDie->setOffset(11);
+  StringRef Producer = StringPool.internString("dsymutil");
+  StringRef File = StringPool.internString(DMO.getObjectFilename());
+  CUDie->addValue(DIEAlloc, dwarf::DW_AT_producer, dwarf::DW_FORM_strp,
+                  DIEInteger(StringPool.getStringOffset(Producer)));
+  DIEBlock *String = new (DIEAlloc) DIEBlock();
+  DIEBlocks.push_back(String);
+  for (auto &C : File)
+    String->addValue(DIEAlloc, dwarf::Attribute(0), dwarf::DW_FORM_data1,
+                     DIEInteger(C));
+  String->addValue(DIEAlloc, dwarf::Attribute(0), dwarf::DW_FORM_data1,
+                   DIEInteger(0));
+
+  CUDie->addValue(DIEAlloc, dwarf::DW_AT_name, dwarf::DW_FORM_string, String);
+  for (const auto &Warning : DMO.getWarnings()) {
+    DIE &ConstDie = CUDie->addChild(DIE::get(DIEAlloc, dwarf::DW_TAG_constant));
+    ConstDie.addValue(
+        DIEAlloc, dwarf::DW_AT_name, dwarf::DW_FORM_strp,
+        DIEInteger(StringPool.getStringOffset("dsymutil_warning")));
+    ConstDie.addValue(DIEAlloc, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag,
+                      DIEInteger(1));
+    ConstDie.addValue(DIEAlloc, dwarf::DW_AT_const_value, dwarf::DW_FORM_strp,
+                      DIEInteger(StringPool.getStringOffset(Warning)));
+  }
+  unsigned Size = 4 /* FORM_strp */ + File.size() + 1 +
+                  DMO.getWarnings().size() * (4 + 1 + 4) +
+                  1 /* End of children */;
+  DIEAbbrev Abbrev = CUDie->generateAbbrev();
+  AssignAbbrev(Abbrev);
+  CUDie->setAbbrevNumber(Abbrev.getNumber());
+  Size += getULEB128Size(Abbrev.getNumber());
+  // Abbreviation ordering needed for classic compatibility.
+  for (auto &Child : CUDie->children()) {
+    Abbrev = Child.generateAbbrev();
+    AssignAbbrev(Abbrev);
+    Child.setAbbrevNumber(Abbrev.getNumber());
+    Size += getULEB128Size(Abbrev.getNumber());
+  }
+  CUDie->setSize(Size);
+  auto &Asm = Streamer->getAsmPrinter();
+  Asm.emitInt32(11 + CUDie->getSize() - 4);
+  Asm.emitInt16(2);
+  Asm.emitInt32(0);
+  Asm.emitInt8(Map.getTriple().isArch64Bit() ? 8 : 4);
+  Streamer->emitDIE(*CUDie);
+  OutputDebugInfoSize += 11 /* Header */ + Size;
+
+  return true;
+}
+
 bool DwarfLinker::link(const DebugMap &Map) {
   if (!createStreamer(Map.getTriple(), OutFile))
     return false;
 
   // Size of the DIEs (and headers) generated for the linked output.
   OutputDebugInfoSize = 0;
   // A unique ID that identifies each compile unit.
   unsigned UnitID = 0;
if (LinkContext.DMO.getType() == MachO::N_AST) {
       }
 
       // Copy the module into the .swift_ast section.
       if (!Options.NoOutput)
         Streamer->emitSwiftAST((*ErrorOrMem)->getBuffer());
       continue;
     }
 
-    if (!LinkContext.ObjectFile)
+    if (emitPaperTrailWarnings(LinkContext.DMO, Map, OffsetsStringPool))
+      continue;
 
+    if (!LinkContext.ObjectFile)
       continue;
 
     // Look for relocations that correspond to debug map entries.
 
     if (LLVM_LIKELY(!Options.Update) &&
         !LinkContext.RelocMgr.findValidRelocsInDebugInfo(
             *LinkContext.ObjectFile, LinkContext.DMO)) {
       if (Options.Verbose)
for (const auto &CU : LinkContext.DwarfContext->compile_units()) {
                                    UniquingStringPool, ODRContexts, UnitID)) {
         LinkContext.CompileUnits.push_back(llvm::make_unique<CompileUnit>(
             *CU, UnitID++, !Options.NoODR && !Options.Update, ""));
         maybeUpdateMaxDwarfVersion(CU->getVersion());
       }
     }
   }
 
+  // If we haven't seen any CUs, pick an arbitrary valid Dwarf version anyway,
+  // to be able to emit papertrail warnings.
+  if (MaxDwarfVersion == 0)
+    MaxDwarfVersion = 3;
+
   ThreadPool pool(2);
 
   // These variables manage the list of processed object files.
   // The mutex and condition variable are to ensure that this is thread safe.
   std::mutex ProcessedFilesMutex;
   std::condition_variable ProcessedFilesConditionVariable;
   BitVector ProcessedFiles(NumObjects, false);
 

llvm/trunk/tools/dsymutil/MachODebugMapParser.cpp

 namespace {
 using namespace llvm;
 using namespace llvm::dsymutil;
 using namespace llvm::object;
 
 class MachODebugMapParser {
 public:
   MachODebugMapParser(StringRef BinaryPath, ArrayRef<std::string> Archs,
-                      StringRef PathPrefix = "", bool Verbose = false)
+                      StringRef PathPrefix = "",
+                      bool PaperTrailWarnings = false, bool Verbose = false)
       : BinaryPath(BinaryPath), Archs(Archs.begin(), Archs.end()),
-        PathPrefix(PathPrefix), MainBinaryHolder(Verbose),
-        CurrentObjectHolder(Verbose), CurrentDebugMapObject(nullptr) {}
+        PathPrefix(PathPrefix), PaperTrailWarnings(PaperTrailWarnings),
+        MainBinaryHolder(Verbose), CurrentObjectHolder(Verbose),
+        CurrentDebugMapObject(nullptr) {}
 
   /// Parses and returns the DebugMaps of the input binary. The binary contains
   /// multiple maps in case it is a universal binary.
   /// \returns an error in case the provided BinaryPath doesn't exist
   /// or isn't of a supported type.
   ErrorOr<std::vector<std::unique_ptr<DebugMap>>> parse();
 
   /// Walk the symbol table and dump it.
   bool dumpStab();
 
 private:
   std::string BinaryPath;
   SmallVector<StringRef, 1> Archs;
   std::string PathPrefix;
+  bool PaperTrailWarnings;
 
   /// Owns the MemoryBuffer for the main binary.
   BinaryHolder MainBinaryHolder;
   /// Map of the binary symbol addresses.
   StringMap<uint64_t> MainBinarySymbolAddresses;
   StringRef MainBinaryStrings;
   /// The constructed DebugMap.
   std::unique_ptr<DebugMap> Result;
private:
   }
   void dumpOneBinaryStab(const MachOObjectFile &MainBinary,
                          StringRef BinaryPath);
 
   void Warning(const Twine &Msg, StringRef File = StringRef()) {
     warn_ostream() << "("
                    << MachOUtils::getArchName(Result->getTriple().getArchName())
                    << ") " << File << " " << Msg << "\n";
+
+    if (PaperTrailWarnings) {
+      if (!File.empty())
+        Result->addDebugMapObject(File, sys::TimePoint<std::chrono::seconds>());
+      if (Result->end() != Result->begin())
+        (*--Result->end())->addWarning(Msg.str());
+    }
   }
 };
 
 } // anonymous namespace
 
 /// Reset the parser state corresponding to the current object
 /// file. This is to be called after an object file is finished
 /// processing.
for (const auto &Sym : MainBinary.symbols()) {
     MainBinarySymbolAddresses[Name] = Addr;
   }
 }
 
 namespace llvm {
 namespace dsymutil {
 llvm::ErrorOr<std::vector<std::unique_ptr<DebugMap>>>
 parseDebugMap(StringRef InputFile, ArrayRef<std::string> Archs,
-              StringRef PrependPath, bool Verbose, bool InputIsYAML) {
-  if (!InputIsYAML) {
-    MachODebugMapParser Parser(InputFile, Archs, PrependPath, Verbose);
-    return Parser.parse();
-  } else {
+              StringRef PrependPath, bool PaperTrailWarnings, bool Verbose,
+              bool InputIsYAML) {
+  if (InputIsYAML)
     return DebugMap::parseYAMLDebugMap(InputFile, PrependPath, Verbose);
-  }
+
+  MachODebugMapParser Parser(InputFile, Archs, PrependPath, PaperTrailWarnings,
+                             Verbose);
+  return Parser.parse();
 }
 
 bool dumpStab(StringRef InputFile, ArrayRef<std::string> Archs,
               StringRef PrependPath) {
   MachODebugMapParser Parser(InputFile, Archs, PrependPath, false);
   return Parser.dumpStab();
 }
 } // namespace dsymutil
 } // namespace llvm

llvm/trunk/tools/dsymutil/dsymutil.h

struct LinkOptions {
   LinkOptions() = default;
 };
 
 /// Extract the DebugMaps from the given file.
 /// The file has to be a MachO object file. Multiple debug maps can be
 /// returned when the file is universal (aka fat) binary.
 ErrorOr<std::vector<std::unique_ptr<DebugMap>>>
 parseDebugMap(StringRef InputFile, ArrayRef<std::string> Archs,
-              StringRef PrependPath, bool Verbose, bool InputIsYAML);
+              StringRef PrependPath, bool PaperTrailWarnings, bool Verbose,
+              bool InputIsYAML);
 
 /// Dump the symbol table
 bool dumpStab(StringRef InputFile, ArrayRef<std::string> Archs,
               StringRef PrependPath = "");
 
 /// Link the Dwarf debug info as directed by the passed DebugMap \p DM into a
 /// DwarfFile named \p OutputFilename. \returns false if the link failed.
 bool linkDwarf(raw_fd_ostream &OutFile, const DebugMap &DM,

llvm/trunk/tools/dsymutil/dsymutil.cpp

 
 static opt<bool> Verify("verify", desc("Verify the linked DWARF debug info."),
                         cat(DsymCategory));
 
 static opt<std::string>
     Toolchain("toolchain", desc("Embed toolchain information in dSYM bundle."),
               cat(DsymCategory));
 
+static opt<bool>
+    PaperTrailWarnings("papertrail",
+                       desc("Embed warnings in the linked DWARF debug info."),
+                       cat(DsymCategory));
+
 static bool createPlistFile(llvm::StringRef Bin, llvm::StringRef BundleRoot) {
   if (NoOutput)
     return true;
 
   // Create plist file to write to.
   llvm::SmallString<128> InfoPlist(BundleRoot);
   llvm::sys::path::append(InfoPlist, "Contents/Info.plist");
   std::error_code EC;
if (!FlatOut && OutputFileOpt == "-") {
     return 1;
   }
 
   if (InputsOrErr->size() > 1 && FlatOut && !OutputFileOpt.empty()) {
     error_ostream() << "cannot use -o with multiple inputs in flat mode\n";
     return 1;
   }
 
+  if (getenv("RC_DEBUG_OPTIONS"))
+    PaperTrailWarnings = true;
+
+  if (PaperTrailWarnings && InputIsYAMLDebugMap)
+    warn_ostream() << "Paper trail warnings are not supported for YAML input";
+
   for (const auto &Arch : ArchFlags)
     if (Arch != "*" && Arch != "all" &&
         !llvm::object::MachOObjectFile::isValidArch(Arch)) {
       error_ostream() << "unsupported cpu architecture: '" << Arch << "'\n";
       return 1;
     }
 
   for (auto &InputFile : *InputsOrErr) {
     // Dump the symbol table for each input file and requested arch
     if (DumpStab) {
       if (!dumpStab(InputFile, ArchFlags, OsoPrependPath))
         return 1;
       continue;
     }
 
-    auto DebugMapPtrsOrErr = parseDebugMap(InputFile, ArchFlags, OsoPrependPath,
-                                           Verbose, InputIsYAMLDebugMap);
+    auto DebugMapPtrsOrErr =
+        parseDebugMap(InputFile, ArchFlags, OsoPrependPath, PaperTrailWarnings,
 
     if (auto EC = DebugMapPtrsOrErr.getError()) {
       error_ostream() << "cannot parse the debug map for '" << InputFile
                       << "': " << EC.message() << '\n';
       return 1;
     }
 
     if (OptionsOrErr->Update) {