[dsymutil] Automatically generate a reproducer when dsymutil crashes

[JDevlieghere]

Summary

Automatically generate a reproducer when dsymutil crashes. We already support generating reproducers with the --gen-reproducer flag, but this automates the generation on a crash.

rdar://68357665

[aprantl]

llvm/tools/dsymutil/Reproducer.cpp

54

LLVM has a macro that expands to the preferred bugreport URL. Maybe include that?

llvm/tools/dsymutil/Reproducer.h

69

default initialize?

[JDevlieghere]

llvm/tools/dsymutil/Reproducer.cpp

54

We still print that before as part of the stack trace and I don't want to mess with that.

[JDevlieghere]

• Add the ability to disable the reproducers.
• Add test for --reproducer aliases.
• Update the docs.

llvm/test/tools/dsymutil/X86/reproducer.test

 # UNSUPPORTED: system-windows
 # This test is unsupported on windows because it creates paths
 # longer than MAX_PATH
 
 # Recreate the folder structure in a temp directory we can remove later.
 RUN: rm -rf %t
 RUN: mkdir -p %t/Inputs
 RUN: cp %p/../Inputs/basic.macho.x86_64 %t/Inputs
 RUN: cp %p/../Inputs/basic1.macho.x86_64.o %t/Inputs
 RUN: cp %p/../Inputs/basic2.macho.x86_64.o %t/Inputs
 RUN: cp %p/../Inputs/basic3.macho.x86_64.o %t/Inputs
 
 # Verify all the files are present.
 RUN: dsymutil -f -o - -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | llvm-dwarfdump -a - | FileCheck %s
 
 # Create a reproducer.
-RUN: env DSYMUTIL_REPRODUCER_PATH=%t.repro dsymutil -gen-reproducer -f -o %t.generate -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | FileCheck %s --check-prefixes=REPRODUCER
+RUN: env DSYMUTIL_REPRODUCER_PATH=%t.repro dsymutil -gen-reproducer -f -o %t.generate -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefixes=REPRODUCER
 RUN: llvm-dwarfdump -a %t.generate | FileCheck %s
 
 # Remove the input files and verify that was successful.
 RUN: rm -rf %t
 RUN: not dsymutil -f -o %t.error -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefix=ERROR
 
 # Use the reproducer.
 RUN: dsymutil -use-reproducer %t.repro -f -o - -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | llvm-dwarfdump -a - | FileCheck %s

 CHECK:    DW_TAG_const_type
 CHECK-NEXT:      DW_AT_type (0x00000079
 CHECK:    DW_TAG_base_type
 CHECK-NEXT:      DW_AT_name ("char")
 CHECK-NEXT:      DW_AT_encoding (DW_ATE_signed_char)
 CHECK-NEXT:      DW_AT_byte_size (0x01)
 CHECK:      NULL
 
-REPRODUCER: reproducer written
+REPRODUCER: Reproducer written
 ERROR: error: cannot parse the debug map
 CONFLICT: cannot combine --gen-reproducer and --use-reproducer

llvm/tools/dsymutil/Reproducer.h

 #include "llvm/Support/FileCollector.h"
 #include "llvm/Support/VirtualFileSystem.h"
 
 namespace llvm {
 namespace dsymutil {
 
 /// The reproducer mode.
 enum class ReproducerMode {
-  Generate,
+  GenerateOnExit,
+  GenerateOnCrash,
   Use,
   Off,
 };
 
 /// The reproducer class manages the sate related to reproducers in dsymutil.
 /// Instances should be created with Reproducer::createReproducer. An instance
 /// of this class is returned when reproducers are off. The VFS returned by
 /// this instance is the real file system.
 class Reproducer {
 public:
   Reproducer();
   virtual ~Reproducer();
 
   IntrusiveRefCntPtr<vfs::FileSystem> getVFS() const { return VFS; }
 
+  virtual void generate(){};
+
   /// Create a Reproducer instance based on the given mode.
   static llvm::Expected<std::unique_ptr<Reproducer>>
-  createReproducer(ReproducerMode Mode, StringRef Root);
+  createReproducer(ReproducerMode Mode, StringRef Root, int Argc, char **Argv);
 
 protected:
   IntrusiveRefCntPtr<vfs::FileSystem> VFS;
 };
 
 /// Reproducer instance used to generate a new reproducer. The VFS returned by
 /// this instance is a FileCollectorFileSystem that tracks every file used by
 /// dsymutil.
 class ReproducerGenerate : public Reproducer {
 public:
-  ReproducerGenerate(std::error_code &EC);
+  ReproducerGenerate(std::error_code &EC, int Argc, char **Argv,
+                     bool GenerateOnExit);
   ~ReproducerGenerate() override;
 
+  virtual void generate() override;
+
 private:
   /// The path to the reproducer.
   std::string Root;
 
   /// The FileCollector used by the FileCollectorFileSystem.
   std::shared_ptr<FileCollector> FC;
+
+  /// The input arguments to build the reproducer invocation.
+  llvm::SmallVector<llvm::StringRef, 0> Args;
+
+  /// Whether to generate the reproducer on destruction.
+  bool GenerateOnExit;
+
+  /// Whether we already generated the reproducer.
+  bool Generated;
 };
 
 /// Reproducer instance used to use an existing reproducer. The VFS returned by
 /// this instance is a RedirectingFileSystem that remaps paths to their
 /// counterpart in the reproducer.
 class ReproducerUse : public Reproducer {
 public:
   ReproducerUse(StringRef Root, std::error_code &EC);

llvm/tools/dsymutil/Reproducer.cpp

if (const char *Path = getenv("DSYMUTIL_REPRODUCER_PATH")) {
     EC = sys::fs::createUniqueDirectory("dsymutil", Root);
   }
   return EC ? "" : std::string(Root);
 }
 
 Reproducer::Reproducer() : VFS(vfs::getRealFileSystem()) {}
 Reproducer::~Reproducer() = default;
 
-ReproducerGenerate::ReproducerGenerate(std::error_code &EC)
-    : Root(createReproducerDir(EC)) {
+ReproducerGenerate::ReproducerGenerate(std::error_code &EC, int Argc,
+                                       char **Argv, bool GenerateOnExit)
+    : Root(createReproducerDir(EC)), GenerateOnExit(GenerateOnExit),
+      Generated(false) {
+  for (int I = 0; I < Argc; ++I)
+    Args.push_back(Argv[I]);
   if (!Root.empty())
     FC = std::make_shared<FileCollector>(Root, Root);
   VFS = FileCollector::createCollectorVFS(vfs::getRealFileSystem(), FC);
 }
 
 ReproducerGenerate::~ReproducerGenerate() {
+  if (GenerateOnExit && !Generated)
+    generate();
+}
+
+void ReproducerGenerate::generate() {
   if (!FC)
     return;
+  Generated = true;
   FC->copyFiles(false);
   SmallString<128> Mapping(Root);
   sys::path::append(Mapping, "mapping.yaml");
   FC->writeMapping(Mapping.str());
-  outs() << "reproducer written to " << Root << '\n';
+  errs() << "********************\n";
+  errs() << "Reproducer written to " << Root << '\n';
+  errs() << "Please include the reproducer and the following invocation in "
+            "your bug report:\n";
+  for (llvm::StringRef Arg : Args)
+    errs() << Arg << ' ';
+  errs() << "--use-reproducer " << Root << '\n';
+  errs() << "********************\n";
 }
 
 ReproducerUse::~ReproducerUse() = default;
 
 ReproducerUse::ReproducerUse(StringRef Root, std::error_code &EC) {
   SmallString<128> Mapping(Root);
   sys::path::append(Mapping, "mapping.yaml");
   ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
       vfs::getRealFileSystem()->getBufferForFile(Mapping.str());
 
   if (!Buffer) {
     EC = Buffer.getError();
     return;
   }
 
   VFS = llvm::vfs::getVFSFromYAML(std::move(Buffer.get()), nullptr, Mapping);
 }
 
 llvm::Expected<std::unique_ptr<Reproducer>>
-Reproducer::createReproducer(ReproducerMode Mode, StringRef Root) {
-  switch (Mode) {
-  case ReproducerMode::Generate: {
-    std::error_code EC;
-    std::unique_ptr<Reproducer> Repro =
-        std::make_unique<ReproducerGenerate>(EC);
-    if (EC)
-      return errorCodeToError(EC);
-    return std::move(Repro);
+Reproducer::createReproducer(ReproducerMode Mode, StringRef Root, int Argc,
+                             char **Argv) {
+
+  std::unique_ptr<Reproducer> Repro;
+  switch (Mode) {
+  case ReproducerMode::GenerateOnExit:
+    Repro = std::make_unique<ReproducerGenerate>(EC, Argc, Argv, true);
+    break;
+  case ReproducerMode::GenerateOnCrash:
+    Repro = std::make_unique<ReproducerGenerate>(EC, Argc, Argv, false);
+    break;
+  case ReproducerMode::Use:
+    Repro = std::make_unique<ReproducerUse>(Root, EC);
+    break;
+  case ReproducerMode::Off:
+    Repro = std::make_unique<Reproducer>();
+    break;
   }
-  case ReproducerMode::Use: {
-    std::error_code EC;
-    std::unique_ptr<Reproducer> Repro =
-        std::make_unique<ReproducerUse>(Root, EC);
-    if (EC)
-      return errorCodeToError(EC);
-    return std::move(Repro);
-  }
-  case ReproducerMode::Off:
-    return std::make_unique<Reproducer>();
-  }
-  llvm_unreachable("All cases handled above.");
+  return Repro;
 }

llvm/tools/dsymutil/dsymutil.cpp

 #include "llvm/DebugInfo/DWARF/DWARFVerifier.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/FileCollector.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/WithColor.h"
struct DsymutilOptions {
   std::string SymbolMap;
   std::string OutputFile;
   std::string Toolchain;
   std::string ReproducerPath;
   std::vector<std::string> Archs;
   std::vector<std::string> InputFiles;
   unsigned NumThreads;
   DWARFVerify Verify = DWARFVerify::None;
-  ReproducerMode ReproMode = ReproducerMode::Off;
+  ReproducerMode ReproMode = ReproducerMode::GenerateOnCrash;
   dsymutil::LinkOptions LinkOpts;
 };
 
 /// Return a list of input files. This function has logic for dealing with the
 /// special case where we might have dSYM bundles as input. The function
 /// returns an error when the directory structure doesn't match that of a dSYM
 /// bundle.
 static Expected<std::vector<std::string>> getInputs(opt::InputArgList &Args,
Options.LinkOpts.KeepFunctionForStatic =
       Args.hasArg(OPT_keep_func_for_static);
 
   if (opt::Arg *ReproducerPath = Args.getLastArg(OPT_use_reproducer)) {
     Options.ReproMode = ReproducerMode::Use;
     Options.ReproducerPath = ReproducerPath->getValue();
   }
 
   if (Args.hasArg(OPT_gen_reproducer))
-    Options.ReproMode = ReproducerMode::Generate;
+    Options.ReproMode = ReproducerMode::GenerateOnExit;
 
   if (Expected<DwarfLinkerAccelTableKind> AccelKind = getAccelTableKind(Args)) {
     Options.LinkOpts.TheAccelTableKind = *AccelKind;
   } else {
     return AccelKind.takeError();
   }
 
   if (opt::Arg *SymbolMap = Args.getLastArg(OPT_symbolmap))
int dsymutil_main(int argc, char **argv) {
 
   auto &Options = *OptionsOrErr;
 
   InitializeAllTargetInfos();
   InitializeAllTargetMCs();
   InitializeAllTargets();
   InitializeAllAsmPrinters();
 
-  auto Repro =
-      Reproducer::createReproducer(Options.ReproMode, Options.ReproducerPath);
+  auto Repro = Reproducer::createReproducer(Options.ReproMode,
+                                            Options.ReproducerPath, argc, argv);
   if (!Repro) {
     WithColor::error() << toString(Repro.takeError());
     return EXIT_FAILURE;
   }
 
   Options.LinkOpts.VFS = (*Repro)->getVFS();
 
   for (const auto &Arch : Options.Archs)
const bool NeedsTempFiles =
         (DebugMapPtrsOrErr->size() != 1 || Options.LinkOpts.Update);
     bool VerifyOutput = flagIsSet(Options.Verify, DWARFVerify::Output);
     if (VerifyOutput && Options.LinkOpts.NoOutput) {
       WithColor::warning()
           << "skipping output verification because --no-output was passed\n";
       VerifyOutput = false;
     }
 
-    SmallVector<MachOUtils::ArchAndFile, 4> TempFiles;
+    // Set up a crash recovery context.
+    CrashRecoveryContext::Enable();
+    CrashRecoveryContext CRC;
+    CRC.DumpStackAndCleanupOnFailure = true;
+
     std::atomic_char AllOK(1);
+    SmallVector<MachOUtils::ArchAndFile, 4> TempFiles;
+
+    const bool Crashed = !CRC.RunSafely([&]() {
       for (auto &Map : *DebugMapPtrsOrErr) {
         if (Options.LinkOpts.Verbose || Options.DumpDebugMap)
           Map->print(outs());
 
         if (Options.DumpDebugMap)
           continue;
 
         if (!Options.SymbolMap.empty())
           Options.LinkOpts.Translator = SymMapLoader.Load(InputFile, *Map);
 
         if (Map->begin() == Map->end())
           WithColor::warning()
               << "no debug symbols in executable (-arch "
-            << MachOUtils::getArchName(Map->getTriple().getArchName()) << ")\n";
+              << MachOUtils::getArchName(Map->getTriple().getArchName())
+              << ")\n";
 
         // Using a std::shared_ptr rather than std::unique_ptr because move-only
         // types don't work with std::bind in the ThreadPool implementation.
         std::shared_ptr<raw_fd_ostream> OS;
 
         std::string OutputFile = OutputLocationOrErr->DWARFFile;
         if (NeedsTempFiles) {
           TempFiles.emplace_back(Map->getTriple().getArchName().str());
 
           auto E = TempFiles.back().createTempFile();
           if (E) {
             WithColor::error() << toString(std::move(E));
-          return EXIT_FAILURE;
+            AllOK.fetch_and(false);
+            return;
           }
 
           auto &TempFile = *(TempFiles.back().File);
           OS = std::make_shared<raw_fd_ostream>(TempFile.FD,
                                                 /*shouldClose*/ false);
           OutputFile = TempFile.TmpName;
         } else {
           std::error_code EC;
           OS = std::make_shared<raw_fd_ostream>(
-            Options.LinkOpts.NoOutput ? "-" : OutputFile, EC, sys::fs::OF_None);
+              Options.LinkOpts.NoOutput ? "-" : OutputFile, EC,
+              sys::fs::OF_None);
           if (EC) {
             WithColor::error() << OutputFile << ": " << EC.message();
-          return EXIT_FAILURE;
+            AllOK.fetch_and(false);
+            return;
           }
         }
 
-      auto LinkLambda = [&, OutputFile](std::shared_ptr<raw_fd_ostream> Stream,
-                                        LinkOptions Options) {
+        auto LinkLambda = [&,
+                           OutputFile](std::shared_ptr<raw_fd_ostream> Stream,
           AllOK.fetch_and(
               linkDwarf(*Stream, BinHolder, *Map, std::move(Options)));
           Stream->flush();
           if (VerifyOutput) {
             AllOK.fetch_and(verifyOutput(
                 OutputFile, Map->getTriple().getArchName(), Options.Verbose));
           }
         };
 
         // FIXME: The DwarfLinker can have some very deep recursion that can max
         // out the (significantly smaller) stack when using threads. We don't
         // want this limitation when we only have a single thread.
         if (S.ThreadsRequested == 1)
           LinkLambda(OS, Options.LinkOpts);
         else
           Threads.async(LinkLambda, OS, Options.LinkOpts);
       }
 
       Threads.wait();
+    });
+
+    if (Crashed)
+      (*Repro)->generate();
 
     if (!AllOK)
       return EXIT_FAILURE;
 
     if (NeedsTempFiles) {
       // Universal Mach-O files can't have an archicture slice that starts
       // beyond the 4GB boundary. "lipo" can creeate a 64 bit universal header,
       // but not all tools can parse these files so we want to return an error