Index: llvm/trunk/include/llvm/Bitcode/ReaderWriter.h =================================================================== --- llvm/trunk/include/llvm/Bitcode/ReaderWriter.h +++ llvm/trunk/include/llvm/Bitcode/ReaderWriter.h @@ -97,8 +97,12 @@ /// Write the specified module summary index to the given raw output stream, /// where it will be written in a new bitcode block. This is used when - /// writing the combined index file for ThinLTO. - void WriteIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out); + /// writing the combined index file for ThinLTO. When writing a subset of the + /// index for a distributed backend, provide the \p ModuleToSummariesForIndex + /// map. + void WriteIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, + std::map + *ModuleToSummariesForIndex = nullptr); /// isBitcodeWrapper - Return true if the given bytes are the magic bytes /// for an LLVM IR bitcode wrapper. Index: llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h =================================================================== --- llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h +++ llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h @@ -19,6 +19,7 @@ #include "llvm-c/lto.h" #include "llvm/ADT/StringSet.h" #include "llvm/ADT/Triple.h" +#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Target/TargetOptions.h" @@ -27,7 +28,6 @@ namespace llvm { class StringRef; -class ModuleSummaryIndex; class LLVMContext; class TargetMachine; @@ -201,6 +201,13 @@ void crossModuleImport(Module &Module, ModuleSummaryIndex &Index); /** + * Compute the list of summaries needed for importing into module. + */ + static void gatherImportedSummariesForModule( + StringRef ModulePath, ModuleSummaryIndex &Index, + std::map &ModuleToSummariesForIndex); + + /** * Perform internalization. */ void internalize(Module &Module, ModuleSummaryIndex &Index); Index: llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h =================================================================== --- llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h +++ llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h @@ -87,6 +87,22 @@ void ComputeCrossModuleImportForModule( StringRef ModulePath, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList); + +/// Compute the set of summaries needed for a ThinLTO backend compilation of +/// \p ModulePath. +// +/// This includes summaries from that module (in case any global summary based +/// optimizations were recorded) and from any definitions in other modules that +/// should be imported. +// +/// \p ModuleToSummariesForIndex will be populated with the needed summaries +/// from each required module path. Use a std::map instead of StringMap to get +/// stable order for bitcode emission. +void gatherImportedSummariesForModule( + StringRef ModulePath, + const StringMap &ModuleToDefinedGVSummaries, + const StringMap &ImportLists, + std::map &ModuleToSummariesForIndex); } #endif // LLVM_FUNCTIONIMPORT_H Index: llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp +++ llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -263,6 +263,10 @@ /// The combined index to write to bitcode. const ModuleSummaryIndex &Index; + /// When writing a subset of the index for distributed backends, client + /// provides a map of modules to the corresponding GUIDs/summaries to write. + std::map *ModuleToSummariesForIndex; + /// Map that holds the correspondence between the GUID used in the combined /// index and a value id generated by this class to use in references. std::map GUIDToValueIdMap; @@ -272,18 +276,142 @@ public: /// Constructs a IndexBitcodeWriter object for the given combined index, - /// writing to the provided \p Buffer. + /// writing to the provided \p Buffer. When writing a subset of the index + /// for a distributed backend, provide a \p ModuleToSummariesForIndex map. IndexBitcodeWriter(SmallVectorImpl &Buffer, - const ModuleSummaryIndex &Index) - : BitcodeWriter(Buffer), Index(Index) { - // Assign unique value ids to all functions in the index for use + const ModuleSummaryIndex &Index, + std::map + *ModuleToSummariesForIndex = nullptr) + : BitcodeWriter(Buffer), Index(Index), + ModuleToSummariesForIndex(ModuleToSummariesForIndex) { + // Assign unique value ids to all summaries to be written, for use // in writing out the call graph edges. Save the mapping from GUID // to the new global value id to use when writing those edges, which // are currently saved in the index in terms of GUID. - for (auto &II : Index) - GUIDToValueIdMap[II.first] = ++GlobalValueId; + for (const auto &I : *this) + GUIDToValueIdMap[I.first] = ++GlobalValueId; } + /// The below iterator returns the GUID and associated summary. + typedef std::pair GVInfo; + + /// Iterator over the value GUID and summaries to be written to bitcode, + /// hides the details of whether they are being pulled from the entire + /// index or just those in a provided ModuleToSummariesForIndex map. + class iterator + : public llvm::iterator_facade_base { + /// Enables access to parent class. + const IndexBitcodeWriter &Writer; + + // Iterators used when writing only those summaries in a provided + // ModuleToSummariesForIndex map: + + /// Points to the last element in outer ModuleToSummariesForIndex map. + std::map::iterator ModuleSummariesBack; + /// Iterator on outer ModuleToSummariesForIndex map. + std::map::iterator ModuleSummariesIter; + /// Iterator on an inner global variable summary map. + GVSummaryMapTy::iterator ModuleGVSummariesIter; + + // Iterators used when writing all summaries in the index: + + /// Points to the last element in the Index outer GlobalValueMap. + const_gvsummary_iterator IndexSummariesBack; + /// Iterator on outer GlobalValueMap. + const_gvsummary_iterator IndexSummariesIter; + /// Iterator on an inner GlobalValueSummaryList. + GlobalValueSummaryList::const_iterator IndexGVSummariesIter; + + public: + /// Construct iterator from parent \p Writer and indicate if we are + /// constructing the end iterator. + iterator(const IndexBitcodeWriter &Writer, bool IsAtEnd) : Writer(Writer) { + // Set up the appropriate set of iterators given whether we are writing + // the full index or just a subset. + // Can't setup the Back or inner iterators if the corresponding map + // is empty. This will be handled specially in operator== as well. + if (Writer.ModuleToSummariesForIndex && + !Writer.ModuleToSummariesForIndex->empty()) { + ModuleSummariesBack = + std::prev(Writer.ModuleToSummariesForIndex->end()); + ModuleSummariesIter = Writer.ModuleToSummariesForIndex->begin(); + ModuleGVSummariesIter = !IsAtEnd ? ModuleSummariesIter->second.begin() + : ModuleSummariesBack->second.end(); + } else if (!Writer.ModuleToSummariesForIndex && + Writer.Index.begin() != Writer.Index.end()) { + IndexSummariesBack = std::prev(Writer.Index.end()); + IndexSummariesIter = Writer.Index.begin(); + IndexGVSummariesIter = !IsAtEnd ? IndexSummariesIter->second.begin() + : IndexSummariesBack->second.end(); + } + } + + /// Increment the appropriate set of iterators. + iterator &operator++() { + // First the inner iterator is incremented, then if it is at the end + // and there are more outer iterations to go, the inner is reset to + // the start of the next inner list. + if (Writer.ModuleToSummariesForIndex) { + ++ModuleGVSummariesIter; + if (ModuleGVSummariesIter == ModuleSummariesIter->second.end() && + ModuleSummariesIter != ModuleSummariesBack) { + ++ModuleSummariesIter; + ModuleGVSummariesIter = ModuleSummariesIter->second.begin(); + } + } else { + ++IndexGVSummariesIter; + if (IndexGVSummariesIter == IndexSummariesIter->second.end() && + IndexSummariesIter != IndexSummariesBack) { + ++IndexSummariesIter; + IndexGVSummariesIter = IndexSummariesIter->second.begin(); + } + } + return *this; + } + + /// Access the pair corresponding to the current + /// outer and inner iterator positions. + GVInfo operator*() { + if (Writer.ModuleToSummariesForIndex) + return std::make_pair(ModuleGVSummariesIter->first, + ModuleGVSummariesIter->second); + return std::make_pair(IndexSummariesIter->first, + IndexGVSummariesIter->get()); + } + + /// Checks if the iterators are equal, with special handling for empty + /// indexes. + bool operator==(const iterator &RHS) const { + if (Writer.ModuleToSummariesForIndex) { + // First ensure that both are writing the same subset. + if (Writer.ModuleToSummariesForIndex != + RHS.Writer.ModuleToSummariesForIndex) + return false; + // Already determined above that maps are the same, so if one is + // empty, they both are. + if (Writer.ModuleToSummariesForIndex->empty()) + return true; + return ModuleGVSummariesIter == RHS.ModuleGVSummariesIter; + } + // First ensure RHS also writing the full index, and that both are + // writing the same full index. + if (RHS.Writer.ModuleToSummariesForIndex || + &Writer.Index != &RHS.Writer.Index) + return false; + // Already determined above that maps are the same, so if one is + // empty, they both are. + if (Writer.Index.begin() == Writer.Index.end()) + return true; + return IndexGVSummariesIter == RHS.IndexGVSummariesIter; + } + }; + + /// Obtain the start iterator over the summaries to be written. + iterator begin() { return iterator(*this, /*IsAtEnd=*/false); } + /// Obtain the end iterator over the summaries to be written. + iterator end() { return iterator(*this, /*IsAtEnd=*/true); } + private: /// Main entry point for writing a combined index to bitcode, invoked by /// BitcodeWriter::write() after it writes the header. @@ -294,6 +422,14 @@ void writeCombinedValueSymbolTable(); void writeCombinedGlobalValueSummary(); + /// Indicates whether the provided \p ModulePath should be written into + /// the module string table, e.g. if full index written or if it is in + /// the provided subset. + bool doIncludeModule(StringRef ModulePath) { + return !ModuleToSummariesForIndex || + ModuleToSummariesForIndex->count(ModulePath); + } + bool hasValueId(GlobalValue::GUID ValGUID) { const auto &VMI = GUIDToValueIdMap.find(ValGUID); return VMI != GUIDToValueIdMap.end(); @@ -2963,6 +3099,8 @@ SmallVector Vals; for (const auto &MPSE : Index.modulePaths()) { + if (!doIncludeModule(MPSE.getKey())) + continue; StringEncoding Bits = getStringEncoding(MPSE.getKey().data(), MPSE.getKey().size()); unsigned AbbrevToUse = Abbrev8Bit; @@ -3218,78 +3356,75 @@ NameVals.clear(); }; - for (const auto &GSI : Index) { - for (auto &SI : GSI.second) { - GlobalValueSummary *S = SI.get(); - assert(S); - - assert(hasValueId(GSI.first)); - unsigned ValueId = getValueId(GSI.first); - SummaryToValueIdMap[S] = ValueId; - - if (auto *AS = dyn_cast(S)) { - // Will process aliases as a post-pass because the reader wants all - // global to be loaded first. - Aliases.push_back(AS); - continue; - } - - if (auto *VS = dyn_cast(S)) { - NameVals.push_back(ValueId); - NameVals.push_back(Index.getModuleId(VS->modulePath())); - NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); - for (auto &RI : VS->refs()) { - NameVals.push_back(getValueId(RI.getGUID())); - } - - // Emit the finished record. - Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals, - FSModRefsAbbrev); - NameVals.clear(); - MaybeEmitOriginalName(*S); - continue; - } + for (const auto &I : *this) { + GlobalValueSummary *S = I.second; + assert(S); + + assert(hasValueId(I.first)); + unsigned ValueId = getValueId(I.first); + SummaryToValueIdMap[S] = ValueId; + + if (auto *AS = dyn_cast(S)) { + // Will process aliases as a post-pass because the reader wants all + // global to be loaded first. + Aliases.push_back(AS); + continue; + } - auto *FS = cast(S); + if (auto *VS = dyn_cast(S)) { NameVals.push_back(ValueId); - NameVals.push_back(Index.getModuleId(FS->modulePath())); - NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); - NameVals.push_back(FS->instCount()); - NameVals.push_back(FS->refs().size()); - - for (auto &RI : FS->refs()) { + NameVals.push_back(Index.getModuleId(VS->modulePath())); + NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); + for (auto &RI : VS->refs()) { NameVals.push_back(getValueId(RI.getGUID())); } - bool HasProfileData = false; - for (auto &EI : FS->calls()) { - HasProfileData |= EI.second.ProfileCount != 0; - if (HasProfileData) - break; - } - - for (auto &EI : FS->calls()) { - // If this GUID doesn't have a value id, it doesn't have a function - // summary and we don't need to record any calls to it. - if (!hasValueId(EI.first.getGUID())) - continue; - NameVals.push_back(getValueId(EI.first.getGUID())); - assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite"); - NameVals.push_back(EI.second.CallsiteCount); - if (HasProfileData) - NameVals.push_back(EI.second.ProfileCount); - } - - unsigned FSAbbrev = - (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); - unsigned Code = - (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED); - // Emit the finished record. - Stream.EmitRecord(Code, NameVals, FSAbbrev); + Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals, + FSModRefsAbbrev); NameVals.clear(); MaybeEmitOriginalName(*S); + continue; + } + + auto *FS = cast(S); + NameVals.push_back(ValueId); + NameVals.push_back(Index.getModuleId(FS->modulePath())); + NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); + NameVals.push_back(FS->instCount()); + NameVals.push_back(FS->refs().size()); + + for (auto &RI : FS->refs()) { + NameVals.push_back(getValueId(RI.getGUID())); } + + bool HasProfileData = false; + for (auto &EI : FS->calls()) { + HasProfileData |= EI.second.ProfileCount != 0; + if (HasProfileData) + break; + } + + for (auto &EI : FS->calls()) { + // If this GUID doesn't have a value id, it doesn't have a function + // summary and we don't need to record any calls to it. + if (!hasValueId(EI.first.getGUID())) + continue; + NameVals.push_back(getValueId(EI.first.getGUID())); + assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite"); + NameVals.push_back(EI.second.CallsiteCount); + if (HasProfileData) + NameVals.push_back(EI.second.ProfileCount); + } + + unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); + unsigned Code = + (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED); + + // Emit the finished record. + Stream.EmitRecord(Code, NameVals, FSAbbrev); + NameVals.clear(); + MaybeEmitOriginalName(*S); } for (auto *AS : Aliases) { @@ -3563,12 +3698,15 @@ // Write the specified module summary index to the given raw output stream, // where it will be written in a new bitcode block. This is used when -// writing the combined index file for ThinLTO. -void llvm::WriteIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out) { +// writing the combined index file for ThinLTO. When writing a subset of the +// index for a distributed backend, provide a \p ModuleToSummariesForIndex map. +void llvm::WriteIndexToFile( + const ModuleSummaryIndex &Index, raw_ostream &Out, + std::map *ModuleToSummariesForIndex) { SmallVector Buffer; Buffer.reserve(256 * 1024); - IndexBitcodeWriter IndexWriter(Buffer, Index); + IndexBitcodeWriter IndexWriter(Buffer, Index, ModuleToSummariesForIndex); IndexWriter.write(); Out.write((char *)&Buffer.front(), Buffer.size()); Index: llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp +++ llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp @@ -720,6 +720,29 @@ } /** + * Compute the list of summaries needed for importing into module. + */ +void ThinLTOCodeGenerator::gatherImportedSummariesForModule( + StringRef ModulePath, ModuleSummaryIndex &Index, + std::map &ModuleToSummariesForIndex) { + auto ModuleCount = Index.modulePaths().size(); + + // Collect for each module the list of function it defines (GUID -> Summary). + StringMap ModuleToDefinedGVSummaries(ModuleCount); + Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); + + // Generate import/export list + StringMap ImportLists(ModuleCount); + StringMap ExportLists(ModuleCount); + ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, + ExportLists); + + llvm::gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, + ImportLists, + ModuleToSummariesForIndex); +} + +/** * Perform internalization. */ void ThinLTOCodeGenerator::internalize(Module &TheModule, Index: llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp +++ llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp @@ -418,6 +418,33 @@ #endif } +/// Compute the set of summaries needed for a ThinLTO backend compilation of +/// \p ModulePath. +void llvm::gatherImportedSummariesForModule( + StringRef ModulePath, + const StringMap &ModuleToDefinedGVSummaries, + const StringMap &ImportLists, + std::map &ModuleToSummariesForIndex) { + // Include all summaries from the importing module. + ModuleToSummariesForIndex[ModulePath] = + ModuleToDefinedGVSummaries.lookup(ModulePath); + auto ModuleImports = ImportLists.find(ModulePath); + if (ModuleImports != ImportLists.end()) { + // Include summaries for imports. + for (auto &ILI : ModuleImports->second) { + auto &SummariesForIndex = ModuleToSummariesForIndex[ILI.first()]; + const auto &DefinedGVSummaries = + ModuleToDefinedGVSummaries.lookup(ILI.first()); + for (auto &GI : ILI.second) { + const auto &DS = DefinedGVSummaries.find(GI.first); + assert(DS != DefinedGVSummaries.end() && + "Expected a defined summary for imported global value"); + SummariesForIndex[GI.first] = DS->second; + } + } + } +} + // Automatically import functions in Module \p DestModule based on the summaries // index. // Index: llvm/trunk/test/ThinLTO/X86/Inputs/distributed_indexes.ll =================================================================== --- llvm/trunk/test/ThinLTO/X86/Inputs/distributed_indexes.ll +++ llvm/trunk/test/ThinLTO/X86/Inputs/distributed_indexes.ll @@ -0,0 +1,4 @@ +define void @g() { +entry: + ret void +} Index: llvm/trunk/test/ThinLTO/X86/distributed_indexes.ll =================================================================== --- llvm/trunk/test/ThinLTO/X86/distributed_indexes.ll +++ llvm/trunk/test/ThinLTO/X86/distributed_indexes.ll @@ -0,0 +1,47 @@ +; RUN: opt -module-summary %s -o %t1.bc +; RUN: opt -module-summary %p/Inputs/distributed_indexes.ll -o %t2.bc +; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc +; RUN: llvm-lto -thinlto-action=distributedindexes -thinlto-index %t.index.bc %t1.bc %t2.bc +; RUN: llvm-bcanalyzer -dump %t1.bc.thinlto.bc | FileCheck %s --check-prefix=BACKEND1 +; RUN: llvm-bcanalyzer -dump %t2.bc.thinlto.bc | FileCheck %s --check-prefix=BACKEND2 + +; The backend index for this module contains summaries from itself and +; Inputs/distributed_indexes.ll, as it imports from the latter. +; BACKEND1: Index = + getModuleSummaryIndexForFile(F, InputFile.file()); - // If we are doing ThinLTO compilation, simply build the combined - // module index/summary and emit it. We don't need to parse the modules - // and link them in this case. - if (options::thinlto) { - ModuleSummaryIndex CombinedIndex; - uint64_t NextModuleId = 0; + // Skip files without a module summary. + if (Index) + CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId); + } + + if (options::thinlto_index_only) { + // Collect for each module the list of function it defines (GUID -> + // Summary). + StringMap> + ModuleToDefinedGVSummaries(NextModuleId); + CombinedIndex.collectDefinedGVSummariesPerModule( + ModuleToDefinedGVSummaries); + + // FIXME: We want to do this for the case where the threads are launched + // from gold as well, in which case this will be moved out of the + // thinlto_index_only handling, and the function importer will be invoked + // directly using the Lists. + StringMap ImportLists(NextModuleId); + StringMap ExportLists(NextModuleId); + ComputeCrossModuleImport(CombinedIndex, ModuleToDefinedGVSummaries, + ImportLists, ExportLists); + + // For each input bitcode file, generate an individual index that + // contains summaries only for its own global values, and for any that + // should be imported. for (claimed_file &F : Modules) { PluginInputFile InputFile(F.handle); - - std::unique_ptr Index = - getModuleSummaryIndexForFile(F, InputFile.file()); - - // Skip files without a module summary. - if (Index) - CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId); + std::error_code EC; + raw_fd_ostream OS((Twine(InputFile.file().name) + ".thinlto.bc").str(), + EC, sys::fs::OpenFlags::F_None); + if (EC) + message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s", + InputFile.file().name, EC.message().c_str()); + // Build a map of module to the GUIDs and summary objects that should + // be written to its index. + std::map ModuleToSummariesForIndex; + gatherImportedSummariesForModule(InputFile.file().name, + ModuleToDefinedGVSummaries, ImportLists, + ModuleToSummariesForIndex); + WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); } + cleanup_hook(); + exit(0); + } + + // Create OS in nested scope so that it will be closed on destruction. + { std::error_code EC; raw_fd_ostream OS(output_name + ".thinlto.bc", EC, sys::fs::OpenFlags::F_None); @@ -1224,16 +1259,24 @@ message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s", output_name.data(), EC.message().c_str()); WriteIndexToFile(CombinedIndex, OS); - OS.close(); + } - if (options::thinlto_index_only) { - cleanup_hook(); - exit(0); - } + thinLTOBackends(ApiFile, CombinedIndex); + return LDPS_OK; +} - thinLTOBackends(ApiFile, CombinedIndex); +/// gold informs us that all symbols have been read. At this point, we use +/// get_symbols to see if any of our definitions have been overridden by a +/// native object file. Then, perform optimization and codegen. +static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) { + if (Modules.empty()) return LDPS_OK; - } + + if (unsigned NumOpts = options::extra.size()) + cl::ParseCommandLineOptions(NumOpts, &options::extra[0]); + + if (options::thinlto) + return thinLTOLink(ApiFile); LLVMContext Context; Context.setDiscardValueNames(options::TheOutputType != Index: llvm/trunk/tools/llvm-lto/llvm-lto.cpp =================================================================== --- llvm/trunk/tools/llvm-lto/llvm-lto.cpp +++ llvm/trunk/tools/llvm-lto/llvm-lto.cpp @@ -66,6 +66,7 @@ enum ThinLTOModes { THINLINK, + THINDISTRIBUTE, THINPROMOTE, THINIMPORT, THININTERNALIZE, @@ -80,6 +81,8 @@ clEnumValN( THINLINK, "thinlink", "ThinLink: produces the index by linking only the summaries."), + clEnumValN(THINDISTRIBUTE, "distributedindexes", + "Produces individual indexes for distributed backends."), clEnumValN(THINPROMOTE, "promote", "Perform pre-import promotion (requires -thinlto-index)."), clEnumValN(THINIMPORT, "import", "Perform both promotion and " @@ -343,6 +346,8 @@ switch (ThinLTOMode) { case THINLINK: return thinLink(); + case THINDISTRIBUTE: + return distributedIndexes(); case THINPROMOTE: return promote(); case THINIMPORT: @@ -385,6 +390,36 @@ return; } + /// Load the combined index from disk, then compute and generate + /// individual index files suitable for ThinLTO distributed backend builds + /// on the files mentioned on the command line (these must match the index + /// content). + void distributedIndexes() { + if (InputFilenames.size() != 1 && !OutputFilename.empty()) + report_fatal_error("Can't handle a single output filename and multiple " + "input files, do not provide an output filename and " + "the output files will be suffixed from the input " + "ones."); + + auto Index = loadCombinedIndex(); + for (auto &Filename : InputFilenames) { + // Build a map of module to the GUIDs and summary objects that should + // be written to its index. + std::map ModuleToSummariesForIndex; + ThinLTOCodeGenerator::gatherImportedSummariesForModule( + Filename, *Index, ModuleToSummariesForIndex); + + std::string OutputName = OutputFilename; + if (OutputName.empty()) { + OutputName = Filename + ".thinlto.bc"; + } + std::error_code EC; + raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::F_None); + error(EC, "error opening the file '" + OutputName + "'"); + WriteIndexToFile(*Index, OS, &ModuleToSummariesForIndex); + } + } + /// Load the combined index from disk, then load every file referenced by /// the index and add them to the generator, finally perform the promotion /// on the files mentioned on the command line (these must match the index