Reduce the memory footprint of dsymutil. (NFC)

[aprantl]

Summary

This (partially) fixes a regression introduced by
https://reviews.llvm.org/D43945 / r327399, which parallelized
DwarfLinker. This patch avoids parsing and allocating the memory for
all input DIEs up front and instead only allocates them in the
concurrent loop in the AnalyzeLambda. At the end of the loop the
memory from the LinkContext is cleared again.

This reduces the peak memory needed to link the debug info of a
non-modular build of the Swift compiler by >3GB.

rdar://problem/43444464

[friss]

tools/dsymutil/DwarfLinker.cpp

2468–2472

You're changing the semantics here. registerModuleReference would not have been called in Update mode and now it will.

Also, why not simply if (CUDie)?

2502

Don't you want Quiet = true ?

2505

I would adda comment explaining that registerModuleReference() won't do any work here because the modules have been registered previously. It's just here to filter out module CUs.

[friss]

tools/dsymutil/DwarfLinker.cpp

2471

Here UnitID is always the same in this loop. Pretty sure that's bad.

[aprantl]

Address (most) review feedback. Thanks!

[aprantl]

tools/dsymutil/DwarfLinker.cpp

2468–2472

That doesn't really matter because they would have been resolved in the first pass, but I'm changing it anyway.

2471

UnitDie is an inout parameter, it is being incremented by loadClangModule() if it creates a new module.

[friss]

Thanks, this LGTM

tools/dsymutil/DwarfLinker.cpp

2471

I see, thanks.

tools/dsymutil/DwarfLinker.h

private:
   /// pointing to the module, and a DW_AT_gnu_dwo_id with the module
   /// hash.
   bool registerModuleReference(const DWARFDie &CUDie, const DWARFUnit &Unit,
                                DebugMap &ModuleMap, const DebugMapObject &DMO,
                                RangesTy &Ranges,
                                OffsetsStringPool &OffsetsStringPool,
                                UniquingStringPool &UniquingStringPoolStringPool,
                                DeclContextTree &ODRContexts, unsigned &UnitID,
-                               unsigned Indent = 0);
+                               unsigned Indent = 0, bool Quiet = false);
 
   /// Recursively add the debug info in this clang module .pcm
   /// file (and all the modules imported by it in a bottom-up fashion)
   /// to Units.
   Error loadClangModule(StringRef Filename, StringRef ModulePath,
                         StringRef ModuleName, uint64_t DwoId,
                         DebugMap &ModuleMap, const DebugMapObject &DMO,
                         RangesTy &Ranges, OffsetsStringPool &OffsetsStringPool,
                         UniquingStringPool &UniquingStringPool,
                         DeclContextTree &ODRContexts, unsigned &UnitID,
-                        unsigned Indent = 0);
+                        unsigned Indent = 0, bool Quiet = false);
 
   /// Flags passed to DwarfLinker::lookForDIEsToKeep
   enum TraversalFlags {
     TF_Keep = 1 << 0,            ///< Mark the traversed DIEs as kept.
     TF_InFunctionScope = 1 << 1, ///< Current scope is a function scope.
     TF_DependencyWalk = 1 << 2,  ///< Walking the dependencies of a kept DIE.
     TF_ParentWalk = 1 << 3,      ///< Walking up the parents of a kept DIE.
     TF_ODR = 1 << 4,             ///< Use the ODR while keeping dependents.

tools/dsymutil/DwarfLinker.cpp

if (DwoId)
     return *DwoId;
   return 0;
 }
 
 bool DwarfLinker::registerModuleReference(
     const DWARFDie &CUDie, const DWARFUnit &Unit, DebugMap &ModuleMap,
     const DebugMapObject &DMO, RangesTy &Ranges, OffsetsStringPool &StringPool,
     UniquingStringPool &UniquingStringPool, DeclContextTree &ODRContexts,
-    unsigned &UnitID, unsigned Indent) {
+    unsigned &UnitID, unsigned Indent, bool Quiet) {
   std::string PCMfile = dwarf::toString(
       CUDie.find({dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}), "");
   if (PCMfile.empty())
     return false;
 
   // Clang module DWARF skeleton CUs abuse this for the path to the module.
   std::string PCMpath = dwarf::toString(CUDie.find(dwarf::DW_AT_comp_dir), "");
   uint64_t DwoId = getDwoId(CUDie, Unit);
 
   std::string Name = dwarf::toString(CUDie.find(dwarf::DW_AT_name), "");
   if (Name.empty()) {
+    if (!Quiet)
       reportWarning("Anonymous module skeleton CU for " + PCMfile, DMO);
     return true;
   }
 
-  if (Options.Verbose) {
+  if (!Quiet && Options.Verbose) {
     outs().indent(Indent);
     outs() << "Found clang module reference " << PCMfile;
   }
 
   auto Cached = ClangModules.find(PCMfile);
   if (Cached != ClangModules.end()) {
     // FIXME: Until PR27449 (https://llvm.org/bugs/show_bug.cgi?id=27449) is
     // fixed in clang, only warn about DWO_id mismatches in verbose mode.
     // ASTFileSignatures will change randomly when a module is rebuilt.
-    if (Options.Verbose && (Cached->second != DwoId))
+    if (!Quiet && Options.Verbose && (Cached->second != DwoId))
       reportWarning(Twine("hash mismatch: this object file was built against a "
                           "different version of the module ") +
                         PCMfile,
                     DMO);
-    if (Options.Verbose)
+    if (!Quiet && Options.Verbose)
       outs() << " [cached].\n";
     return true;
   }
-  if (Options.Verbose)
+  if (!Quiet && Options.Verbose)
     outs() << " ...\n";
 
   // Cyclic dependencies are disallowed by Clang, but we still
   // shouldn't run into an infinite loop, so mark it as processed now.
   ClangModules.insert({PCMfile, DwoId});
   if (Error E = loadClangModule(PCMfile, PCMpath, Name, DwoId, ModuleMap, DMO,
                                 Ranges, StringPool, UniquingStringPool,
-                                ODRContexts, UnitID, Indent + 2)) {
+                                ODRContexts, UnitID, Indent + 2, Quiet)) {
     consumeError(std::move(E));
     return false;
   }
   return true;
 }
 
 ErrorOr<const object::ObjectFile &>
 DwarfLinker::loadObject(const DebugMapObject &Obj, const DebugMap &Map) {
if (!Object) {
     reportWarning(
         Twine(Obj.getObjectFilename()) + ": " + toString(std::move(Err)), Obj);
     return errorToErrorCode(std::move(Err));
   }
 
   return *Object;
 }
 
-Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath,
-                                   StringRef ModuleName, uint64_t DwoId,
-                                   DebugMap &ModuleMap,
-                                   const DebugMapObject &DMO, RangesTy &Ranges,
-                                   OffsetsStringPool &StringPool,
-                                   UniquingStringPool &UniquingStringPool,
-                                   DeclContextTree &ODRContexts,
-                                   unsigned &UnitID, unsigned Indent) {
+Error DwarfLinker::loadClangModule(
+    StringRef Filename, StringRef ModulePath, StringRef ModuleName,
+    uint64_t DwoId, DebugMap &ModuleMap, const DebugMapObject &DMO,
+    RangesTy &Ranges, OffsetsStringPool &StringPool,
+    UniquingStringPool &UniquingStringPool, DeclContextTree &ODRContexts,
+    unsigned &UnitID, unsigned Indent, bool Quiet) {
   SmallString<80> Path(Options.PrependPath);
   if (sys::path::is_relative(Filename))
     sys::path::append(Path, ModulePath, Filename);
   else
     sys::path::append(Path, Filename);
   // Don't use the cached binary holder because we have no thread-safety
   // guarantee and the lifetime is limited.
   auto &Obj = ModuleMap.addDebugMapObject(
Error DwarfLinker::loadClangModule(
   for (const auto &CU : DwarfContext->compile_units()) {
     updateDwarfVersion(CU->getVersion());
     // Recursively get all modules imported by this one.
     auto CUDie = CU->getUnitDIE(false);
     if (!CUDie)
       continue;
     if (!registerModuleReference(CUDie, *CU, ModuleMap, DMO, Ranges, StringPool,
                                  UniquingStringPool, ODRContexts, UnitID,
-                                 Indent)) {
+                                 Indent, Quiet)) {
       if (Unit) {
         std::string Err =
             (Filename +
              ": Clang modules are expected to have exactly 1 compile unit.\n")
                 .str();
         error(Err);
         return make_error<StringError>(Err, inconvertibleErrorCode());
       }
       // FIXME: Until PR27449 (https://llvm.org/bugs/show_bug.cgi?id=27449) is
       // fixed in clang, only warn about DWO_id mismatches in verbose mode.
       // ASTFileSignatures will change randomly when a module is rebuilt.
       uint64_t PCMDwoId = getDwoId(CUDie, *CU);
       if (PCMDwoId != DwoId) {
-        if (Options.Verbose)
+        if (!Quiet && Options.Verbose)
           reportWarning(
               Twine("hash mismatch: this object file was built against a "
                     "different version of the module ") +
                   Filename,
               DMO);
         // Update the cache entry with the DwoId of the module loaded from disk.
         ClangModules[Filename] = PCMDwoId;
       }
 
       // Add this module.
       Unit = llvm::make_unique<CompileUnit>(*CU, UnitID++, !Options.NoODR,
                                             ModuleName);
       Unit->setHasInterestingContent();
       analyzeContextInfo(CUDie, 0, *Unit, &ODRContexts.getRoot(),
                          UniquingStringPool, ODRContexts);
       // Keep everything.
       Unit->markEverythingAsKept();
     }
   }
   if (!Unit->getOrigUnit().getUnitDIE().hasChildren())
     return Error::success();
-  if (Options.Verbose) {
+  if (!Quiet && Options.Verbose) {
     outs().indent(Indent);
     outs() << "cloning .debug_info from " << Filename << "\n";
   }
 
   UnitListTy CompileUnits;
   CompileUnits.push_back(std::move(Unit));
   DIECloner(*this, RelocMgr, DIEAlloc, CompileUnits, Options)
       .cloneAllCompileUnits(*DwarfContext, DMO, Ranges, StringPool);
for (const auto &CU : LinkContext.DwarfContext->compile_units()) {
       auto CUDie = CU->getUnitDIE(false);
       if (Options.Verbose) {
         outs() << "Input compilation unit:";
         DIDumpOptions DumpOpts;
         DumpOpts.RecurseDepth = 0;
         DumpOpts.Verbose = Options.Verbose;
         CUDie.dump(outs(), 0, DumpOpts);
       }
-
-      if (!CUDie || LLVM_UNLIKELY(Options.Update) ||
-          !registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
-                                   LinkContext.Ranges, OffsetsStringPool,
-                                   UniquingStringPool, ODRContexts, UnitID)) {
-        LinkContext.CompileUnits.push_back(llvm::make_unique<CompileUnit>(
-            *CU, UnitID++, !Options.NoODR && !Options.Update, ""));
-      }
+      if (CUDie && !LLVM_UNLIKELY(Options.Update))
+        registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
+                                UniquingStringPool, ODRContexts, UnitID);
     }
   }
 
   // If we haven't seen any CUs, pick an arbitrary valid Dwarf version anyway.
   if (MaxDwarfVersion == 0)
     MaxDwarfVersion = 3;
 
   // These variables manage the list of processed object files.
   // The mutex and condition variable are to ensure that this is thread safe.
   std::mutex ProcessedFilesMutex;
   std::condition_variable ProcessedFilesConditionVariable;
   BitVector ProcessedFiles(NumObjects, false);
 
   // Now do analyzeContextInfo in parallel as it is particularly expensive.
   auto AnalyzeLambda = [&]() {
     for (unsigned i = 0, e = NumObjects; i != e; ++i) {
       auto &LinkContext = ObjectContexts[i];
 
-      if (!LinkContext.ObjectFile) {
+      if (!LinkContext.ObjectFile || !LinkContext.DwarfContext) {
         std::unique_lock<std::mutex> LockGuard(ProcessedFilesMutex);
         ProcessedFiles.set(i);
         ProcessedFilesConditionVariable.notify_one();
         continue;
       }
 
+      for (const auto &CU : LinkContext.DwarfContext->compile_units()) {
+        updateDwarfVersion(CU->getVersion());
+        // The !registerModuleReference() condition effectively skips
+        // over fully resolved skeleton units. This second pass of
+        // registerModuleReferences doesn't do any new work, but it
+        // will collect top-level errors, which are suppressed. Module
+        // warnings were already displayed in the first iteration.
+        bool Quiet = true;
+        auto CUDie = CU->getUnitDIE(false);
+        if (!CUDie || LLVM_UNLIKELY(Options.Update) ||
+            !registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
+                                     LinkContext.Ranges, OffsetsStringPool,
+                                     UniquingStringPool, ODRContexts, UnitID,
+                                     Quiet)) {
+          LinkContext.CompileUnits.push_back(llvm::make_unique<CompileUnit>(
+              *CU, UnitID++, !Options.NoODR && !Options.Update, ""));
+        }
+      }
+      
       // Now build the DIE parent links that we will use during the next phase.
       for (auto &CurrentUnit : LinkContext.CompileUnits) {
         auto CUDie = CurrentUnit->getOrigUnit().getUnitDIE();
         if (!CUDie)
           continue;
         analyzeContextInfo(CurrentUnit->getOrigUnit().getUnitDIE(), 0,
                            *CurrentUnit, &ODRContexts.getRoot(),
                            UniquingStringPool, ODRContexts);