diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/DWARFLinker.h --- a/llvm/include/llvm/DWARFLinker/DWARFLinker.h +++ b/llvm/include/llvm/DWARFLinker/DWARFLinker.h @@ -220,6 +220,8 @@ typedef std::map swiftInterfacesMap; typedef std::map objectPrefixMap; +typedef function_ref CompileUnitHandler; + /// The core of the Dwarf linking logic. /// /// The generation of the dwarf information from the object files will be @@ -240,8 +242,15 @@ DwarfLinkerClient ClientID = DwarfLinkerClient::General) : TheDwarfEmitter(Emitter), DwarfLinkerClientID(ClientID) {} - /// Add object file to be linked. - void addObjectFile(DWARFFile &File); + /// Add object file to be linked. Pre-load compile unit die. Call + /// \p OnCUDieLoaded for each compile unit die. If specified \p File + /// has reference to the Clang module then such module would be + /// pre-loaded by \p Loader for !Update case. + /// + /// \pre NoODR, Update options should be set before call to addObjectFile. + void addObjectFile( + DWARFFile &File, objFileLoader Loader = nullptr, + CompileUnitHandler OnCUDieLoaded = [](const DWARFUnit &) {}); /// Link debug info for added objFiles. Object /// files are linked all together. @@ -304,12 +313,6 @@ Options.ErrorHandler = Handler; } - /// Set object files loader which would be used to load - /// additional objects for splitted dwarf. - void setObjFileLoader(objFileLoader Loader) { - Options.ObjFileLoader = Loader; - } - /// Set map for Swift interfaces. void setSwiftInterfacesMap(swiftInterfacesMap *Map) { Options.ParseableSwiftInterfaces = Map; @@ -410,10 +413,25 @@ void copyInvariantDebugSection(DWARFContext &Dwarf); + /// Keep information for referenced clang module: already loaded DWARF info + /// of the clang module and a CompileUnit of the module. + struct RefModuleUnit { + RefModuleUnit(DWARFFile &File, std::unique_ptr Unit) + : File(File), Unit(std::move(Unit)) {} + RefModuleUnit(RefModuleUnit &&Other) + : File(Other.File), Unit(std::move(Other.Unit)) {} + RefModuleUnit(const RefModuleUnit &) = delete; + + DWARFFile &File; + std::unique_ptr Unit; + }; + using ModuleUnitListTy = std::vector; + /// Keeps track of data associated with one object during linking. struct LinkContext { DWARFFile &File; UnitListTy CompileUnits; + ModuleUnitListTy ModuleUnits; bool Skip = false; LinkContext(DWARFFile &File) : File(File) {} @@ -464,30 +482,38 @@ const DWARFFile &File, CompileUnit &CU, unsigned Flags); + /// Check whether specified \p CUDie is a Clang module reference. + /// if \p Quiet is false then display error messages. + /// \return first == true if CUDie is a Clang module reference. + /// second == true if module is already loaded. + std::pair isClangModuleRef(const DWARFDie &CUDie, + std::string &PCMFile, + LinkContext &Context, unsigned Indent, + bool Quiet); + /// If this compile unit is really a skeleton CU that points to a /// clang module, register it in ClangModules and return true. /// /// A skeleton CU is a CU without children, a DW_AT_gnu_dwo_name /// pointing to the module, and a DW_AT_gnu_dwo_id with the module /// hash. - bool registerModuleReference(DWARFDie CUDie, const DWARFUnit &Unit, - const DWARFFile &File, - OffsetsStringPool &OffsetsStringPool, - DeclContextTree &ODRContexts, - uint64_t ModulesEndOffset, unsigned &UnitID, - bool IsLittleEndian, unsigned Indent = 0, - bool Quiet = false); + bool registerModuleReference(const DWARFDie &CUDie, LinkContext &Context, + objFileLoader Loader, + CompileUnitHandler OnCUDieLoaded, + unsigned Indent = 0); /// Recursively add the debug info in this clang module .pcm /// file (and all the modules imported by it in a bottom-up fashion) - /// to Units. - Error loadClangModule(DWARFDie CUDie, StringRef FilePath, - StringRef ModuleName, uint64_t DwoId, - const DWARFFile &File, + /// to ModuleUnits. + Error loadClangModule(objFileLoader Loader, const DWARFDie &CUDie, + const std::string &PCMFile, LinkContext &Context, + CompileUnitHandler OnCUDieLoaded, unsigned Indent = 0); + + /// Clone specified Clang module unit \p Unit. + Error cloneModuleUnit(LinkContext &Context, RefModuleUnit &Unit, + DeclContextTree &ODRContexts, OffsetsStringPool &OffsetsStringPool, - DeclContextTree &ODRContexts, uint64_t ModulesEndOffset, - unsigned &UnitID, bool IsLittleEndian, - unsigned Indent = 0, bool Quiet = false); + unsigned Indent = 0); /// Mark the passed DIE as well as all the ones it depends on as kept. void keepDIEAndDependencies(AddressesMap &RelocMgr, RangesTy &Ranges, @@ -762,6 +788,9 @@ std::function StringsTranslator = nullptr; + /// A unique ID that identifies each compile unit. + unsigned UniqueUnitID = 0; + /// linking options struct DWARFLinkerOptions { /// Generate processing log to the standard output. @@ -802,8 +831,6 @@ // error handler messageHandler ErrorHandler = nullptr; - objFileLoader ObjFileLoader = nullptr; - /// A list of all .swiftinterface files referenced by the debug /// info, mapping Module name to path on disk. The entries need to /// be uniqued and sorted and there are only few entries expected diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp --- a/llvm/lib/DWARFLinker/DWARFLinker.cpp +++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp @@ -2000,7 +2000,7 @@ hashFullyQualifiedName(Die, *CU, File, ++ChildRecurseDepth))); } -static uint64_t getDwoId(const DWARFDie &CUDie, const DWARFUnit &Unit) { +static uint64_t getDwoId(const DWARFDie &CUDie) { auto DwoId = dwarf::toUnsigned( CUDie.find({dwarf::DW_AT_dwo_id, dwarf::DW_AT_GNU_dwo_id})); if (DwoId) @@ -2020,36 +2020,45 @@ return p.str().str(); } -bool DWARFLinker::registerModuleReference(DWARFDie CUDie, const DWARFUnit &Unit, - const DWARFFile &File, - OffsetsStringPool &StringPool, - DeclContextTree &ODRContexts, - uint64_t ModulesEndOffset, - unsigned &UnitID, bool IsLittleEndian, - unsigned Indent, bool Quiet) { - std::string PCMfile = dwarf::toString( +static std::string getPCMFile(const DWARFDie &CUDie, + objectPrefixMap *ObjectPrefixMap) { + std::string PCMFile = dwarf::toString( CUDie.find({dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}), ""); - if (PCMfile.empty()) - return false; - if (Options.ObjectPrefixMap) - PCMfile = remapPath(PCMfile, *Options.ObjectPrefixMap); + + if (PCMFile.empty()) + return PCMFile; + + if (ObjectPrefixMap) + PCMFile = remapPath(PCMFile, *ObjectPrefixMap); + + return PCMFile; +} + +std::pair DWARFLinker::isClangModuleRef(const DWARFDie &CUDie, + std::string &PCMFile, + LinkContext &Context, + unsigned Indent, + bool Quiet) { + if (PCMFile.empty()) + return std::make_pair(false, false); // Clang module DWARF skeleton CUs abuse this for the path to the module. - uint64_t DwoId = getDwoId(CUDie, Unit); + uint64_t DwoId = getDwoId(CUDie); std::string Name = dwarf::toString(CUDie.find(dwarf::DW_AT_name), ""); if (Name.empty()) { if (!Quiet) - reportWarning("Anonymous module skeleton CU for " + PCMfile, File); - return true; + reportWarning("Anonymous module skeleton CU for " + PCMFile, + Context.File); + return std::make_pair(true, true); } if (!Quiet && Options.Verbose) { outs().indent(Indent); - outs() << "Found clang module reference " << PCMfile; + outs() << "Found clang module reference " << PCMFile; } - auto Cached = ClangModules.find(PCMfile); + auto Cached = ClangModules.find(PCMFile); if (Cached != ClangModules.end()) { // FIXME: Until PR27449 (https://llvm.org/bugs/show_bug.cgi?id=27449) is // fixed in clang, only warn about DWO_id mismatches in verbose mode. @@ -2057,109 +2066,114 @@ if (!Quiet && Options.Verbose && (Cached->second != DwoId)) reportWarning(Twine("hash mismatch: this object file was built against a " "different version of the module ") + - PCMfile, - File); + PCMFile, + Context.File); if (!Quiet && Options.Verbose) outs() << " [cached].\n"; - return true; + return std::make_pair(true, true); } - if (!Quiet && Options.Verbose) + + return std::make_pair(true, false); +} + +bool DWARFLinker::registerModuleReference(const DWARFDie &CUDie, + LinkContext &Context, + objFileLoader Loader, + CompileUnitHandler OnCUDieLoaded, + unsigned Indent) { + std::string PCMFile = getPCMFile(CUDie, Options.ObjectPrefixMap); + std::pair IsClangModuleRef = + isClangModuleRef(CUDie, PCMFile, Context, Indent, false); + + if (!IsClangModuleRef.first) + return false; + + if (IsClangModuleRef.second) + return true; + + if (Options.Verbose) outs() << " ...\n"; // Cyclic dependencies are disallowed by Clang, but we still // shouldn't run into an infinite loop, so mark it as processed now. - ClangModules.insert({PCMfile, DwoId}); + ClangModules.insert({PCMFile, getDwoId(CUDie)}); - if (Error E = loadClangModule(CUDie, PCMfile, Name, DwoId, File, StringPool, - ODRContexts, ModulesEndOffset, UnitID, - IsLittleEndian, Indent + 2, Quiet)) { + if (Error E = loadClangModule(Loader, CUDie, PCMFile, Context, OnCUDieLoaded, + Indent + 2)) { consumeError(std::move(E)); return false; } return true; } -Error DWARFLinker::loadClangModule( - DWARFDie CUDie, StringRef Filename, StringRef ModuleName, uint64_t DwoId, - const DWARFFile &File, OffsetsStringPool &StringPool, - DeclContextTree &ODRContexts, uint64_t ModulesEndOffset, unsigned &UnitID, - bool IsLittleEndian, unsigned Indent, bool Quiet) { +Error DWARFLinker::loadClangModule(objFileLoader Loader, const DWARFDie &CUDie, + const std::string &PCMFile, + LinkContext &Context, + CompileUnitHandler OnCUDieLoaded, + unsigned Indent) { + + uint64_t DwoId = getDwoId(CUDie); + std::string ModuleName = dwarf::toString(CUDie.find(dwarf::DW_AT_name), ""); + /// Using a SmallString<0> because loadClangModule() is recursive. SmallString<0> Path(Options.PrependPath); - if (sys::path::is_relative(Filename)) + if (sys::path::is_relative(PCMFile)) resolveRelativeObjectPath(Path, CUDie); - sys::path::append(Path, Filename); + sys::path::append(Path, PCMFile); // Don't use the cached binary holder because we have no thread-safety // guarantee and the lifetime is limited. - if (Options.ObjFileLoader == nullptr) + if (Loader == nullptr) { + reportError("Could not load clang module: loader is not specified.\n", + Context.File); return Error::success(); + } - auto ErrOrObj = Options.ObjFileLoader(File.FileName, Path); + auto ErrOrObj = Loader(Context.File.FileName, Path); if (!ErrOrObj) return Error::success(); std::unique_ptr Unit; - for (const auto &CU : ErrOrObj->Dwarf->compile_units()) { + OnCUDieLoaded(*CU); updateDwarfVersion(CU->getVersion()); // Recursively get all modules imported by this one. - auto CUDie = CU->getUnitDIE(false); - if (!CUDie) + auto ChildCUDie = CU->getUnitDIE(); + if (!ChildCUDie) continue; - if (!registerModuleReference(CUDie, *CU, File, StringPool, ODRContexts, - ModulesEndOffset, UnitID, IsLittleEndian, - Indent, Quiet)) { + if (!registerModuleReference(ChildCUDie, Context, Loader, OnCUDieLoaded, + Indent)) { if (Unit) { std::string Err = - (Filename + - ": Clang modules are expected to have exactly 1 compile unit.\n") - .str(); - reportError(Err, File); + (PCMFile + + ": Clang modules are expected to have exactly 1 compile unit.\n"); + reportError(Err, Context.File); return make_error(Err, inconvertibleErrorCode()); } // FIXME: Until PR27449 (https://llvm.org/bugs/show_bug.cgi?id=27449) is // fixed in clang, only warn about DWO_id mismatches in verbose mode. // ASTFileSignatures will change randomly when a module is rebuilt. - uint64_t PCMDwoId = getDwoId(CUDie, *CU); + uint64_t PCMDwoId = getDwoId(ChildCUDie); if (PCMDwoId != DwoId) { - if (!Quiet && Options.Verbose) + if (Options.Verbose) reportWarning( Twine("hash mismatch: this object file was built against a " "different version of the module ") + - Filename, - File); + PCMFile, + Context.File); // Update the cache entry with the DwoId of the module loaded from disk. - ClangModules[Filename] = PCMDwoId; + ClangModules[PCMFile] = PCMDwoId; } // Add this module. - Unit = std::make_unique(*CU, UnitID++, !Options.NoODR, + Unit = std::make_unique(*CU, UniqueUnitID++, !Options.NoODR, ModuleName); - analyzeContextInfo(CUDie, 0, *Unit, &ODRContexts.getRoot(), ODRContexts, - ModulesEndOffset, Options.ParseableSwiftInterfaces, - [&](const Twine &Warning, const DWARFDie &DIE) { - reportWarning(Warning, File, &DIE); - }); - // Keep everything. - Unit->markEverythingAsKept(); } } - assert(Unit && "CompileUnit is not set!"); - if (!Unit->getOrigUnit().getUnitDIE().hasChildren()) - return Error::success(); - if (!Quiet && Options.Verbose) { - outs().indent(Indent); - outs() << "cloning .debug_info from " << Filename << "\n"; - } - UnitListTy CompileUnits; - CompileUnits.push_back(std::move(Unit)); - assert(TheDwarfEmitter); - DIECloner(*this, TheDwarfEmitter, *ErrOrObj, DIEAlloc, CompileUnits, - Options.Update) - .cloneAllCompileUnits(*(ErrOrObj->Dwarf), File, StringPool, - IsLittleEndian); + if (Unit) + Context.ModuleUnits.emplace_back(RefModuleUnit{*ErrOrObj, std::move(Unit)}); + return Error::success(); } @@ -2339,19 +2353,32 @@ "debug_aranges"); } -void DWARFLinker::addObjectFile(DWARFFile &File) { +void DWARFLinker::addObjectFile(DWARFFile &File, objFileLoader Loader, + CompileUnitHandler OnCUDieLoaded) { ObjectContexts.emplace_back(LinkContext(File)); - if (ObjectContexts.back().File.Dwarf) + if (ObjectContexts.back().File.Dwarf) { updateAccelKind(*ObjectContexts.back().File.Dwarf); + + for (const std::unique_ptr &CU : + ObjectContexts.back().File.Dwarf->compile_units()) { + DWARFDie CUDie = CU->getUnitDIE(); + + if (!CUDie) + continue; + + OnCUDieLoaded(*CU); + + if (!LLVM_UNLIKELY(Options.Update)) + registerModuleReference(CUDie, ObjectContexts.back(), Loader, + OnCUDieLoaded); + } + } } Error DWARFLinker::link() { assert(Options.NoOutput || TheDwarfEmitter); - // A unique ID that identifies each compile unit. - unsigned UnitID = 0; - // First populate the data structure we need for each iteration of the // parallel loop. unsigned NumObjects = ObjectContexts.size(); @@ -2479,10 +2506,12 @@ DumpOpts.Verbose = Options.Verbose; CUDie.dump(outs(), 0, DumpOpts); } - if (CUDie && !LLVM_UNLIKELY(Options.Update)) - registerModuleReference(CUDie, *CU, OptContext.File, OffsetsStringPool, - ODRContexts, 0, UnitID, - OptContext.File.Dwarf->isLittleEndian()); + } + + for (auto &CU : OptContext.ModuleUnits) { + if (Error Err = + cloneModuleUnit(OptContext, CU, ODRContexts, OffsetsStringPool)) + reportWarning(toString(std::move(Err)), CU.File); } } @@ -2514,19 +2543,15 @@ for (const auto &CU : Context.File.Dwarf->compile_units()) { updateDwarfVersion(CU->getVersion()); - // The !registerModuleReference() condition effectively skips - // over fully resolved skeleton units. This second pass of - // registerModuleReferences doesn't do any new work, but it - // will collect top-level errors, which are suppressed. Module - // warnings were already displayed in the first iteration. - bool Quiet = true; - auto CUDie = CU->getUnitDIE(false); + // The !isClangModuleRef condition effectively skips over fully resolved + // skeleton units. + auto CUDie = CU->getUnitDIE(); + std::string PCMFile = getPCMFile(CUDie, Options.ObjectPrefixMap); + if (!CUDie || LLVM_UNLIKELY(Options.Update) || - !registerModuleReference(CUDie, *CU, Context.File, OffsetsStringPool, - ODRContexts, ModulesEndOffset, UnitID, - Quiet)) { + !isClangModuleRef(CUDie, PCMFile, Context, 0, true).first) { Context.CompileUnits.push_back(std::make_unique( - *CU, UnitID++, !Options.NoODR && !Options.Update, "")); + *CU, UniqueUnitID++, !Options.NoODR && !Options.Update, "")); } } @@ -2718,6 +2743,41 @@ return Error::success(); } +Error DWARFLinker::cloneModuleUnit(LinkContext &Context, RefModuleUnit &Unit, + DeclContextTree &ODRContexts, + OffsetsStringPool &OffsetsStringPool, + unsigned Indent) { + assert(Unit.Unit.get() != nullptr); + + if (!Unit.Unit->getOrigUnit().getUnitDIE().hasChildren()) + return Error::success(); + + if (Options.Verbose) { + outs().indent(Indent); + outs() << "cloning .debug_info from " << Unit.File.FileName << "\n"; + } + + // Analyze context for the module. + analyzeContextInfo(Unit.Unit->getOrigUnit().getUnitDIE(), 0, *(Unit.Unit), + &ODRContexts.getRoot(), ODRContexts, 0, + Options.ParseableSwiftInterfaces, + [&](const Twine &Warning, const DWARFDie &DIE) { + reportWarning(Warning, Context.File, &DIE); + }); + // Keep everything. + Unit.Unit->markEverythingAsKept(); + + // Clone unit. + UnitListTy CompileUnits; + CompileUnits.emplace_back(std::move(Unit.Unit)); + assert(TheDwarfEmitter); + DIECloner(*this, TheDwarfEmitter, Unit.File, DIEAlloc, CompileUnits, + Options.Update) + .cloneAllCompileUnits(*Unit.File.Dwarf, Unit.File, OffsetsStringPool, + Unit.File.Dwarf->isLittleEndian()); + return Error::success(); +} + bool DWARFLinker::verify(const DWARFFile &File) { assert(File.Dwarf); diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp --- a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp +++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp @@ -591,58 +591,58 @@ [&](const Twine &Error, StringRef Context, const DWARFDie *) { error(Error, Context); }); - GeneralLinker.setObjFileLoader( - [&DebugMap, &RL, this](StringRef ContainerName, - StringRef Path) -> ErrorOr { - auto &Obj = DebugMap.addDebugMapObject( - Path, sys::TimePoint(), MachO::N_OSO); - - if (auto ErrorOrObj = loadObject(Obj, DebugMap, RL)) { - return *ErrorOrObj; - } else { - // Try and emit more helpful warnings by applying some heuristics. - StringRef ObjFile = ContainerName; - bool IsClangModule = sys::path::extension(Path).equals(".pcm"); - bool IsArchive = ObjFile.endswith(")"); - - if (IsClangModule) { - StringRef ModuleCacheDir = sys::path::parent_path(Path); - if (sys::fs::exists(ModuleCacheDir)) { - // If the module's parent directory exists, we assume that the - // module cache has expired and was pruned by clang. A more - // adventurous dsymutil would invoke clang to rebuild the module - // now. - if (!ModuleCacheHintDisplayed) { - WithColor::note() - << "The clang module cache may have expired since " - "this object file was built. Rebuilding the " - "object file will rebuild the module cache.\n"; - ModuleCacheHintDisplayed = true; - } - } else if (IsArchive) { - // If the module cache directory doesn't exist at all and the - // object file is inside a static library, we assume that the - // static library was built on a different machine. We don't want - // to discourage module debugging for convenience libraries within - // a project though. - if (!ArchiveHintDisplayed) { - WithColor::note() - << "Linking a static library that was built with " - "-gmodules, but the module cache was not found. " - "Redistributable static libraries should never be " - "built with module debugging enabled. The debug " - "experience will be degraded due to incomplete " - "debug information.\n"; - ArchiveHintDisplayed = true; - } - } + objFileLoader Loader = [&DebugMap, &RL, + this](StringRef ContainerName, + StringRef Path) -> ErrorOr { + auto &Obj = DebugMap.addDebugMapObject( + Path, sys::TimePoint(), MachO::N_OSO); + + if (auto ErrorOrObj = loadObject(Obj, DebugMap, RL)) { + return *ErrorOrObj; + } else { + // Try and emit more helpful warnings by applying some heuristics. + StringRef ObjFile = ContainerName; + bool IsClangModule = sys::path::extension(Path).equals(".pcm"); + bool IsArchive = ObjFile.endswith(")"); + + if (IsClangModule) { + StringRef ModuleCacheDir = sys::path::parent_path(Path); + if (sys::fs::exists(ModuleCacheDir)) { + // If the module's parent directory exists, we assume that the + // module cache has expired and was pruned by clang. A more + // adventurous dsymutil would invoke clang to rebuild the module + // now. + if (!ModuleCacheHintDisplayed) { + WithColor::note() + << "The clang module cache may have expired since " + "this object file was built. Rebuilding the " + "object file will rebuild the module cache.\n"; + ModuleCacheHintDisplayed = true; + } + } else if (IsArchive) { + // If the module cache directory doesn't exist at all and the + // object file is inside a static library, we assume that the + // static library was built on a different machine. We don't want + // to discourage module debugging for convenience libraries within + // a project though. + if (!ArchiveHintDisplayed) { + WithColor::note() + << "Linking a static library that was built with " + "-gmodules, but the module cache was not found. " + "Redistributable static libraries should never be " + "built with module debugging enabled. The debug " + "experience will be degraded due to incomplete " + "debug information.\n"; + ArchiveHintDisplayed = true; } - - return ErrorOrObj.getError(); } + } - llvm_unreachable("Unhandled DebugMap object"); - }); + return ErrorOrObj.getError(); + } + + llvm_unreachable("Unhandled DebugMap object"); + }; GeneralLinker.setSwiftInterfacesMap(&ParseableSwiftInterfaces); bool ReflectionSectionsPresentInBinary = false; // If there is no output specified, no point in checking the binary for swift5 @@ -702,8 +702,9 @@ continue; } + if (auto ErrorOrObj = loadObject(*Obj, Map, RL)) - GeneralLinker.addObjectFile(*ErrorOrObj); + GeneralLinker.addObjectFile(*ErrorOrObj, Loader); else { ObjectsForLinking.push_back(std::make_unique( Obj->getObjectFilename(), nullptr, nullptr,