diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -1503,10 +1503,7 @@ // we should only invoke this using the individual indexes written out // via a WriteIndexesThinBackend. FunctionImporter::ImportMapTy ImportList; - std::vector> OwnedImports; - MapVector ModuleMap; - if (!lto::loadReferencedModules(*M, *CombinedIndex, ImportList, ModuleMap, - OwnedImports)) + if (!lto::initImportList(*M, *CombinedIndex, ImportList)) return; auto AddStream = [&](size_t Task) { @@ -1583,7 +1580,7 @@ if (Error E = thinBackend(Conf, -1, AddStream, *M, *CombinedIndex, ImportList, ModuleToDefinedGVSummaries[M->getModuleIdentifier()], - ModuleMap, CGOpts.CmdArgs)) { + /* ModuleMap */ nullptr, CGOpts.CmdArgs)) { handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) { errs() << "Error running ThinLTO backend: " << EIB.message() << '\n'; }); diff --git a/clang/test/CodeGen/thinlto_backend.ll b/clang/test/CodeGen/thinlto_backend.ll --- a/clang/test/CodeGen/thinlto_backend.ll +++ b/clang/test/CodeGen/thinlto_backend.ll @@ -47,7 +47,7 @@ ; Ensure we get expected error for input files without summaries ; RUN: opt -o %t2.o %s ; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -o %t3.o -x ir %t1.o -c -fthinlto-index=%t.thinlto.bc 2>&1 | FileCheck %s -check-prefix=CHECK-ERROR2 -; CHECK-ERROR2: Error loading imported file '{{.*}}': Could not find module summary +; CHECK-ERROR2: Error loading imported file {{.*}}: Could not find module summary target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/include/llvm/LTO/LTOBackend.h b/llvm/include/llvm/LTO/LTOBackend.h --- a/llvm/include/llvm/LTO/LTOBackend.h +++ b/llvm/include/llvm/LTO/LTOBackend.h @@ -46,11 +46,16 @@ ModuleSummaryIndex &CombinedIndex); /// Runs a ThinLTO backend. +/// If \p ModuleMap is not nullptr, all the module files to be imported have +/// already been mapped to memory and the corresponding BitcodeModule objects +/// are saved in the ModuleMap. If \p ModuleMap is nullptr, module files will +/// be mapped to memory on demand and at any given time during importing, only +/// one source module will be kept open at the most. Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, Module &M, const ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, - MapVector &ModuleMap, + MapVector *ModuleMap, const std::vector &CmdArgs = std::vector()); Error finalizeOptimizationRemarks( @@ -62,15 +67,11 @@ /// Variant of the above. Expected findThinLTOModule(MemoryBufferRef MBRef); -/// Distributed ThinLTO: load the referenced modules, keeping their buffers -/// alive in the provided OwnedImportLifetimeManager. Returns false if the +/// Distributed ThinLTO: collect the referenced modules based on +/// module summary and initialize ImportList. Returns false if the /// operation failed. -bool loadReferencedModules( - const Module &M, const ModuleSummaryIndex &CombinedIndex, - FunctionImporter::ImportMapTy &ImportList, - MapVector &ModuleMap, - std::vector> - &OwnedImportsLifetimeManager); +bool initImportList(const Module &M, const ModuleSummaryIndex &CombinedIndex, + FunctionImporter::ImportMapTy &ImportList); } } diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1215,7 +1215,7 @@ return MOrErr.takeError(); return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex, - ImportList, DefinedGlobals, ModuleMap); + ImportList, DefinedGlobals, &ModuleMap); }; auto ModuleID = BM.getModuleIdentifier(); diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -539,7 +539,7 @@ Module &Mod, const ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, - MapVector &ModuleMap, + MapVector *ModuleMap, const std::vector &CmdArgs) { Expected TOrErr = initAndLookupTarget(Conf, Mod); if (!TOrErr) @@ -608,11 +608,35 @@ auto ModuleLoader = [&](StringRef Identifier) { assert(Mod.getContext().isODRUniquingDebugTypes() && "ODR Type uniquing should be enabled on the context"); - auto I = ModuleMap.find(Identifier); - assert(I != ModuleMap.end()); - return I->second.getLazyModule(Mod.getContext(), - /*ShouldLazyLoadMetadata=*/true, - /*IsImporting*/ true); + if (ModuleMap) { + auto I = ModuleMap->find(Identifier); + assert(I != ModuleMap->end()); + return I->second.getLazyModule(Mod.getContext(), + /*ShouldLazyLoadMetadata=*/true, + /*IsImporting*/ true); + } + + ErrorOr> MBOrErr = + llvm::MemoryBuffer::getFile(Identifier); + if (!MBOrErr) + return Expected>(make_error( + Twine("Error loading imported file ") + Identifier + " : ", + MBOrErr.getError())); + + Expected BMOrErr = findThinLTOModule(**MBOrErr); + if (!BMOrErr) + return Expected>(make_error( + Twine("Error loading imported file ") + Identifier + " : " + + toString(BMOrErr.takeError()), + inconvertibleErrorCode())); + + Expected> MOrErr = + BMOrErr->getLazyModule(Mod.getContext(), + /*ShouldLazyLoadMetadata=*/true, + /*IsImporting*/ true); + if (MOrErr) + (*MOrErr)->setOwnedMemoryBuffer(std::move(*MBOrErr)); + return MOrErr; }; FunctionImporter Importer(CombinedIndex, ModuleLoader, @@ -652,12 +676,9 @@ inconvertibleErrorCode()); } -bool lto::loadReferencedModules( - const Module &M, const ModuleSummaryIndex &CombinedIndex, - FunctionImporter::ImportMapTy &ImportList, - MapVector &ModuleMap, - std::vector> - &OwnedImportsLifetimeManager) { +bool lto::initImportList(const Module &M, + const ModuleSummaryIndex &CombinedIndex, + FunctionImporter::ImportMapTy &ImportList) { if (ThinLTOAssumeMerged) return true; // We can simply import the values mentioned in the combined index, since @@ -678,26 +699,5 @@ ImportList[Summary->modulePath()].insert(GUID); } } - - for (auto &I : ImportList) { - ErrorOr> MBOrErr = - llvm::MemoryBuffer::getFile(I.first()); - if (!MBOrErr) { - errs() << "Error loading imported file '" << I.first() - << "': " << MBOrErr.getError().message() << "\n"; - return false; - } - - Expected BMOrErr = findThinLTOModule(**MBOrErr); - if (!BMOrErr) { - handleAllErrors(BMOrErr.takeError(), [&](ErrorInfoBase &EIB) { - errs() << "Error loading imported file '" << I.first() - << "': " << EIB.message() << '\n'; - }); - return false; - } - ModuleMap.insert({I.first(), *BMOrErr}); - OwnedImportsLifetimeManager.push_back(std::move(*MBOrErr)); - } return true; }