Index: llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h =================================================================== --- llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h +++ llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h @@ -202,6 +202,12 @@ void promote(Module &Module, ModuleSummaryIndex &Index); /** + * Compute and emit the imported files for module at \p ModulePath. + */ + static void emitImports(StringRef ModulePath, StringRef OutputName, + ModuleSummaryIndex &Index); + + /** * Perform cross-module importing for the module identified by * ModuleIdentifier. */ Index: llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h =================================================================== --- llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h +++ llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h @@ -103,6 +103,10 @@ const StringMap &ModuleToDefinedGVSummaries, const StringMap &ImportLists, std::map &ModuleToSummariesForIndex); + +std::error_code +EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, + const StringMap &ImportLists); } #endif // LLVM_FUNCTIONIMPORT_H Index: llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp +++ llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp @@ -743,6 +743,30 @@ } /** + * Emit the list of files needed for importing into module. + */ +void ThinLTOCodeGenerator::emitImports(StringRef ModulePath, + StringRef OutputName, + ModuleSummaryIndex &Index) { + auto ModuleCount = Index.modulePaths().size(); + + // Collect for each module the list of function it defines (GUID -> Summary). + StringMap ModuleToDefinedGVSummaries(ModuleCount); + Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); + + // Generate import/export list + StringMap ImportLists(ModuleCount); + StringMap ExportLists(ModuleCount); + ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, + ExportLists); + + std::error_code EC; + if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists))) + report_fatal_error(Twine("Failed to open ") + OutputName + + " to save imports lists\n"); +} + +/** * Perform internalization. */ void ThinLTOCodeGenerator::internalize(Module &TheModule, Index: llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp +++ llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp @@ -445,6 +445,21 @@ } } +/// Emit the files \p ModulePath will import from into \p OutputFilename. +std::error_code llvm::EmitImportsFiles( + StringRef ModulePath, StringRef OutputFilename, + const StringMap &ImportLists) { + auto ModuleImports = ImportLists.find(ModulePath); + std::error_code EC; + raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::F_None); + if (EC) + return EC; + if (ModuleImports != ImportLists.end()) + for (auto &ILI : ModuleImports->second) + ImportsOS << ILI.first() << "\n"; + return std::error_code(); +} + // Automatically import functions in Module \p DestModule based on the summaries // index. // Index: llvm/trunk/test/ThinLTO/X86/Inputs/emit_imports.ll =================================================================== --- llvm/trunk/test/ThinLTO/X86/Inputs/emit_imports.ll +++ llvm/trunk/test/ThinLTO/X86/Inputs/emit_imports.ll @@ -0,0 +1,4 @@ +define void @g() { +entry: + ret void +} Index: llvm/trunk/test/ThinLTO/X86/emit_imports.ll =================================================================== --- llvm/trunk/test/ThinLTO/X86/emit_imports.ll +++ llvm/trunk/test/ThinLTO/X86/emit_imports.ll @@ -0,0 +1,21 @@ +; RUN: opt -module-summary %s -o %t1.bc +; RUN: opt -module-summary %p/Inputs/emit_imports.ll -o %t2.bc +; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc +; RUN: llvm-lto -thinlto-action=emitimports -thinlto-index %t.index.bc %t1.bc %t2.bc + +; The imports file for this module contains the bitcode file for +; Inputs/emit_imports.ll +; RUN: cat %t1.bc.imports | count 1 +; RUN: cat %t1.bc.imports | FileCheck %s --check-prefix=IMPORTS1 +; IMPORTS1: emit_imports.ll.tmp2.bc + +; The imports file for Input/emit_imports.ll is empty as it does not import anything. +; RUN: cat %t2.bc.imports | count 0 + +declare void @g(...) + +define void @f() { +entry: + call void (...) @g() + ret void +} Index: llvm/trunk/test/tools/gold/X86/thinlto_emit_imports.ll =================================================================== --- llvm/trunk/test/tools/gold/X86/thinlto_emit_imports.ll +++ llvm/trunk/test/tools/gold/X86/thinlto_emit_imports.ll @@ -0,0 +1,27 @@ +; Generate summary sections and test gold handling. +; RUN: opt -module-summary %s -o %t.o +; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o + +; Ensure gold generates imports files if requested for distributed backends. +; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ +; RUN: --plugin-opt=thinlto \ +; RUN: --plugin-opt=thinlto-index-only \ +; RUN: --plugin-opt=thinlto-emit-imports-files \ +; RUN: -shared %t.o %t2.o -o %t3 + +; The imports file for this module contains the bitcode file for +; Inputs/thinlto.ll +; RUN: cat %t.o.imports | count 1 +; RUN: cat %t.o.imports | FileCheck %s --check-prefix=IMPORTS1 +; IMPORTS1: test/tools/gold/X86/Output/thinlto_emit_imports.ll.tmp2.o + +; The imports file for Input/thinlto.ll is empty as it does not import anything. +; RUN: cat %t2.o.imports | count 0 + +declare void @g(...) + +define void @f() { +entry: + call void (...) @g() + ret void +} Index: llvm/trunk/tools/gold/gold-plugin.cpp =================================================================== --- llvm/trunk/tools/gold/gold-plugin.cpp +++ llvm/trunk/tools/gold/gold-plugin.cpp @@ -184,6 +184,11 @@ // the import decisions, and exit afterwards. The assumption is // that the build system will launch the backend processes. static bool thinlto_index_only = false; + // If true, when generating individual index files for distributed backends, + // also generate a "${bitcodefile}.imports" file at the same location for each + // bitcode file, listing the files it imports from in plain text. This is to + // support distributed build file staging. + static bool thinlto_emit_imports_files = false; // Additional options to pass into the code generator. // Note: This array will contain all plugin options which are not claimed // as plugin exclusive to pass to the code generator. @@ -217,6 +222,8 @@ thinlto = true; } else if (opt == "thinlto-index-only") { thinlto_index_only = true; + } else if (opt == "thinlto-emit-imports-files") { + thinlto_emit_imports_files = true; } else if (opt.size() == 2 && opt[0] == 'O') { if (opt[1] < '0' || opt[1] > '3') message(LDPL_FATAL, "Optimization level must be between 0 and 3"); @@ -1209,6 +1216,10 @@ CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId); } + if (options::thinlto_emit_imports_files && !options::thinlto_index_only) + message(LDPL_WARNING, + "thinlto-emit-imports-files ignored unless thinlto-index-only"); + if (options::thinlto_index_only) { // Collect for each module the list of function it defines (GUID -> // Summary). @@ -1244,6 +1255,15 @@ ModuleToDefinedGVSummaries, ImportLists, ModuleToSummariesForIndex); WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); + + if (options::thinlto_emit_imports_files) { + if ((EC = EmitImportsFiles( + InputFile.file().name, + (Twine(InputFile.file().name) + ".imports").str(), + ImportLists))) + message(LDPL_FATAL, "Unable to open %s.imports", + InputFile.file().name, EC.message().c_str()); + } } cleanup_hook(); Index: llvm/trunk/tools/llvm-lto/llvm-lto.cpp =================================================================== --- llvm/trunk/tools/llvm-lto/llvm-lto.cpp +++ llvm/trunk/tools/llvm-lto/llvm-lto.cpp @@ -67,6 +67,7 @@ enum ThinLTOModes { THINLINK, THINDISTRIBUTE, + THINEMITIMPORTS, THINPROMOTE, THINIMPORT, THININTERNALIZE, @@ -83,6 +84,8 @@ "ThinLink: produces the index by linking only the summaries."), clEnumValN(THINDISTRIBUTE, "distributedindexes", "Produces individual indexes for distributed backends."), + clEnumValN(THINEMITIMPORTS, "emitimports", + "Emit imports files for distributed backends."), clEnumValN(THINPROMOTE, "promote", "Perform pre-import promotion (requires -thinlto-index)."), clEnumValN(THINIMPORT, "import", "Perform both promotion and " @@ -359,6 +362,8 @@ return thinLink(); case THINDISTRIBUTE: return distributedIndexes(); + case THINEMITIMPORTS: + return emitImports(); case THINPROMOTE: return promote(); case THINIMPORT: @@ -431,6 +436,25 @@ } } + /// Load the combined index from disk, compute the imports, and emit + /// the import file lists for each module to disk. + void emitImports() { + if (InputFilenames.size() != 1 && !OutputFilename.empty()) + report_fatal_error("Can't handle a single output filename and multiple " + "input files, do not provide an output filename and " + "the output files will be suffixed from the input " + "ones."); + + auto Index = loadCombinedIndex(); + for (auto &Filename : InputFilenames) { + std::string OutputName = OutputFilename; + if (OutputName.empty()) { + OutputName = Filename + ".imports"; + } + ThinLTOCodeGenerator::emitImports(Filename, OutputName, *Index); + } + } + /// Load the combined index from disk, then load every file referenced by /// the index and add them to the generator, finally perform the promotion /// on the files mentioned on the command line (these must match the index