Index: include/llvm/Bitcode/BitcodeWriter.h =================================================================== --- include/llvm/Bitcode/BitcodeWriter.h +++ include/llvm/Bitcode/BitcodeWriter.h @@ -43,9 +43,15 @@ /// /// \p GenerateHash enables hashing the Module and including the hash in the /// bitcode (currently for use in ThinLTO incremental build). + /// + /// If \p ModHash is non-null and holds a non-zero value, that value + /// is used as the hash instead of computing from the generated bitcode. + /// Can be used to produce the same module hash for a minimized bitcode + /// used just for the thin link as in the regular full bitcode that will + /// be used in the backend. void writeModule(const Module *M, bool ShouldPreserveUseListOrder = false, const ModuleSummaryIndex *Index = nullptr, - bool GenerateHash = false); + bool GenerateHash = false, ModuleHash *ModHash = nullptr); }; /// \brief Write the specified module to the specified raw output stream. @@ -62,10 +68,17 @@ /// /// \p GenerateHash enables hashing the Module and including the hash in the /// bitcode (currently for use in ThinLTO incremental build). + /// + /// If \p ModHash is non-null and holds a non-zero value, that value + /// is used as the hash instead of computing from the generated bitcode. + /// Can be used to produce the same module hash for a minimized bitcode + /// used just for the thin link as in the regular full bitcode that will + /// be used in the backend. void WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder = false, const ModuleSummaryIndex *Index = nullptr, - bool GenerateHash = false); + bool GenerateHash = false, + ModuleHash *ModHash = nullptr); /// Write the specified module summary index to the given raw output stream, /// where it will be written in a new bitcode block. This is used when Index: include/llvm/Object/ModuleSummaryIndexObjectFile.h =================================================================== --- include/llvm/Object/ModuleSummaryIndexObjectFile.h +++ include/llvm/Object/ModuleSummaryIndexObjectFile.h @@ -88,9 +88,12 @@ } /// Parse the module summary index out of an IR file and return the module -/// summary index object if found, or nullptr if not. +/// summary index object if found, or nullptr if not. If Identifier is +/// non-empty, it is used as the module ID (module path) in the resulting +/// index. This can be used when the index is being read from a file +/// containing minimized bitcode just for the thin link. Expected> -getModuleSummaryIndexForFile(StringRef Path); +getModuleSummaryIndexForFile(StringRef Path, StringRef Identifier = ""); } #endif Index: include/llvm/Transforms/IPO.h =================================================================== --- include/llvm/Transforms/IPO.h +++ include/llvm/Transforms/IPO.h @@ -249,7 +249,8 @@ ModulePass *createSampleProfileLoaderPass(StringRef Name); /// Write ThinLTO-ready bitcode to Str. -ModulePass *createWriteThinLTOBitcodePass(raw_ostream &Str); +ModulePass *createWriteThinLTOBitcodePass(raw_ostream &Str, + std::string ThinLinkBitcodeFile = ""); } // End llvm namespace Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -108,6 +108,11 @@ /// True if a module hash record should be written. bool GenerateHash; + /// If non-zero, use as the module hash instead of computing from the written + /// bitcode. In either case, the written module hash value is stored in + /// this field. + ModuleHash ModHash; + /// The start bit of the identification block. uint64_t BitcodeStartBit; @@ -124,10 +129,12 @@ /// writing to the provided \p Buffer. ModuleBitcodeWriter(const Module *M, SmallVectorImpl &Buffer, BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, - const ModuleSummaryIndex *Index, bool GenerateHash) + const ModuleSummaryIndex *Index, bool GenerateHash, + ModuleHash ModHash = ModuleHash{{0}}) : BitcodeWriterBase(Stream), Buffer(Buffer), M(*M), VE(*M, ShouldPreserveUseListOrder), Index(Index), - GenerateHash(GenerateHash), BitcodeStartBit(Stream.GetCurrentBitNo()) { + GenerateHash(GenerateHash), ModHash(ModHash), + BitcodeStartBit(Stream.GetCurrentBitNo()) { // Assign ValueIds to any callee values in the index that came from // indirect call profiles and were recorded as a GUID not a Value* // (which would have been assigned an ID by the ValueEnumerator). @@ -152,6 +159,10 @@ /// Emit the current module to the bitstream. void write(); + /// Return the module hash value, used to retrieve the written hash + /// after write() is called. + const ModuleHash &getModuleHash() const { return ModHash; } + private: uint64_t bitcodeStartBit() { return BitcodeStartBit; } @@ -3771,14 +3782,23 @@ void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { // Emit the module's hash. // MODULE_CODE_HASH: [5*i32] + + // If a non-zero module hash was provided, use it instead of computing + // the hash. + if (ModHash != ModuleHash{{0}}) { + Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef(ModHash)); + return; + } SHA1 Hasher; + uint32_t Vals[5]; Hasher.update(ArrayRef((const uint8_t *)&(Buffer)[BlockStartPos], Buffer.size() - BlockStartPos)); StringRef Hash = Hasher.result(); - uint32_t Vals[5]; for (int Pos = 0; Pos < 20; Pos += 4) { Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); } + // Save the written hash value. + std::copy(std::begin(Vals), std::end(Vals), std::begin(ModHash)); // Emit the finished record. Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); @@ -3935,10 +3955,13 @@ void BitcodeWriter::writeModule(const Module *M, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, - bool GenerateHash) { + bool GenerateHash, ModuleHash *ModHash) { ModuleBitcodeWriter ModuleWriter( - M, Buffer, *Stream, ShouldPreserveUseListOrder, Index, GenerateHash); + M, Buffer, *Stream, ShouldPreserveUseListOrder, Index, GenerateHash, + ModHash ? *ModHash : ModuleHash{{0}}); ModuleWriter.write(); + if (ModHash) + *ModHash = ModuleWriter.getModuleHash(); } /// WriteBitcodeToFile - Write the specified module to the specified output @@ -3946,7 +3969,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, - bool GenerateHash) { + bool GenerateHash, ModuleHash *ModHash) { SmallVector Buffer; Buffer.reserve(256*1024); @@ -3957,7 +3980,8 @@ Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); BitcodeWriter Writer(Buffer); - Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash); + Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash, + ModHash); if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) emitDarwinBCHeaderAndTrailer(Buffer, TT); Index: lib/Object/ModuleSummaryIndexObjectFile.cpp =================================================================== --- lib/Object/ModuleSummaryIndexObjectFile.cpp +++ lib/Object/ModuleSummaryIndexObjectFile.cpp @@ -96,13 +96,18 @@ // Parse the module summary index out of an IR file and return the summary // index object if found, or nullptr if not. Expected> -llvm::getModuleSummaryIndexForFile(StringRef Path) { +llvm::getModuleSummaryIndexForFile(StringRef Path, StringRef Identifier) { ErrorOr> FileOrErr = MemoryBuffer::getFileOrSTDIN(Path); std::error_code EC = FileOrErr.getError(); if (EC) return errorCodeToError(EC); - MemoryBufferRef BufferRef = (FileOrErr.get())->getMemBufferRef(); + std::unique_ptr MemBuffer = std::move(FileOrErr.get()); + // If Identifier is non-empty, use it as the buffer identifier, which + // will become the module path in the index. + if (Identifier.empty()) + Identifier = MemBuffer->getBufferIdentifier(); + MemoryBufferRef BufferRef(MemBuffer->getBuffer(), Identifier); if (IgnoreEmptyThinLTOIndexFile && !BufferRef.getBufferSize()) return nullptr; Expected> ObjOrErr = Index: lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp =================================================================== --- lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -14,7 +14,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Transforms/IPO.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Analysis/TypeMetadataUtils.h" @@ -25,7 +24,10 @@ #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/FileSystem.h" #include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/Utils/Cloning.h" using namespace llvm; @@ -251,8 +253,8 @@ // a multi-module bitcode file with the two parts to OS. Otherwise, write only a // regular LTO bitcode file to OS. void splitAndWriteThinLTOBitcode( - raw_ostream &OS, function_ref AARGetter, - Module &M) { + raw_ostream &OS, raw_ostream *ThinLinkOS, + function_ref AARGetter, Module &M) { std::string ModuleId = getModuleId(&M); if (ModuleId.empty()) { // We couldn't generate a module ID for this module, just write it out as a @@ -334,17 +336,34 @@ simplifyExternals(*MergedM); - SmallVector Buffer; - BitcodeWriter W(Buffer); // FIXME: Try to re-use BSI and PFI from the original module here. ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, nullptr); - W.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index, - /*GenerateHash=*/true); - W.writeModule(MergedM.get()); + SmallVector Buffer; + BitcodeWriter W(Buffer); + // Save the module hash produced for the full bitcode, which will + // be used in the backends, and use that in the minimized bitcode + // produced for the full link. + ModuleHash ModHash; + W.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index, + /*GenerateHash=*/true, &ModHash); + W.writeModule(MergedM.get()); OS << Buffer; + + // If a minimized bitcode module was requeted for the thin link, + // strip the debug info (the merged module was already stripped above) + // and write it to the given OS. + if (ThinLinkOS) { + Buffer.clear(); + BitcodeWriter W2(Buffer); + StripDebugInfo(M); + W2.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index, + /*GenerateHash=*/true, &ModHash); + W2.writeModule(MergedM.get()); + *ThinLinkOS << Buffer; + } } // Returns whether this module needs to be split because it uses type metadata. @@ -359,20 +378,36 @@ return false; } -void writeThinLTOBitcode(raw_ostream &OS, +void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS, function_ref AARGetter, Module &M, const ModuleSummaryIndex *Index) { // See if this module has any type metadata. If so, we need to split it. if (requiresSplit(M)) - return splitAndWriteThinLTOBitcode(OS, AARGetter, M); + return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M); // Otherwise we can just write it out as a regular module. + + // Save the module hash produced for the full bitcode, which will + // be used in the backends, and use that in the minimized bitcode + // produced for the full link. + ModuleHash ModHash; WriteBitcodeToFile(&M, OS, /*ShouldPreserveUseListOrder=*/false, Index, - /*GenerateHash=*/true); + /*GenerateHash=*/true, &ModHash); + // If a minimized bitcode module was requeted for the thin link, + // strip the debug info and write it to the given OS. + if (ThinLinkOS) { + StripDebugInfo(M); + WriteBitcodeToFile(&M, *ThinLinkOS, /*ShouldPreserveUseListOrder=*/false, + Index, + /*GenerateHash=*/true, &ModHash); + } } class WriteThinLTOBitcode : public ModulePass { raw_ostream &OS; // raw_ostream to print on + // The output stream on which to emit a minimized module for use + // just in the thin link, if requested. + std::unique_ptr ThinLinkOS; public: static char ID; // Pass identification, replacement for typeid @@ -380,9 +415,24 @@ initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry()); } - explicit WriteThinLTOBitcode(raw_ostream &o) + explicit WriteThinLTOBitcode(raw_ostream &o, std::string ThinLinkBitcodeFile) : ModulePass(ID), OS(o) { initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry()); + setUpThinLtoBitcodeFile(ThinLinkBitcodeFile); + } + + // The thin link only needs the module symbol table and summary sections. + // If requested, create the OS for a minimized module to be written. + // For now we simply strip debug metadata. + void setUpThinLtoBitcodeFile(std::string ThinLinkBitcodeFile) { + if (!ThinLinkBitcodeFile.empty()) { + std::error_code Error; + ThinLinkOS.reset( + new raw_fd_ostream(ThinLinkBitcodeFile, Error, sys::fs::F_None)); + if (Error) + report_fatal_error(Twine("Failed to open ") + ThinLinkBitcodeFile + + " to save thin link bitcode\n"); + } } StringRef getPassName() const override { return "ThinLTO Bitcode Writer"; } @@ -390,7 +440,7 @@ bool runOnModule(Module &M) override { const ModuleSummaryIndex *Index = &(getAnalysis().getIndex()); - writeThinLTOBitcode(OS, LegacyAARGetter(*this), M, Index); + writeThinLTOBitcode(OS, ThinLinkOS.get(), LegacyAARGetter(*this), M, Index); return true; } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -411,6 +461,8 @@ INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode", "Write ThinLTO Bitcode", false, true) -ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str) { - return new WriteThinLTOBitcode(Str); +ModulePass * +llvm::createWriteThinLTOBitcodePass(raw_ostream &Str, + std::string ThinLinkBitcodeFile) { + return new WriteThinLTOBitcode(Str, ThinLinkBitcodeFile); } Index: test/ThinLTO/X86/distributed_import.ll =================================================================== --- test/ThinLTO/X86/distributed_import.ll +++ test/ThinLTO/X86/distributed_import.ll @@ -1,15 +1,42 @@ -; RUN: opt -module-summary %s -o %t1.bc -; RUN: opt -module-summary %p/Inputs/distributed_import.ll -o %t2.bc +; Test distributed build thin link output from llvm-lto2 +; Generate bitcode files with summary, as well as minimized bitcode without +; the debug metadata for the thin link. +; RUN: opt -thinlto-bc %s -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc +; RUN: opt -thinlto-bc %p/Inputs/distributed_import.ll -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc + +; First perform the thin link on the normal bitcode file. ; RUN: llvm-lto2 %t1.bc %t2.bc -o %t.o -save-temps \ ; RUN: -thinlto-distributed-indexes \ ; RUN: -r=%t1.bc,g, \ ; RUN: -r=%t1.bc,f,px \ ; RUN: -r=%t2.bc,g,px -; RUN: opt -function-import -summary-file %t1.bc.thinlto.bc %t1.bc -o %t1.out +; RUN: opt -function-import -summary-file %t1.bc.thinlto.bc %t1.bc -o %t1.out ; RUN: opt -function-import -summary-file %t2.bc.thinlto.bc %t2.bc -o %t2.out ; RUN: llvm-dis -o - %t2.out | FileCheck %s -; CHECK: @G.llvm.0 + +; Save dumps of the generated index files. +; RUN: llvm-bcanalyzer -dump %t1.bc.thinlto.bc | grep -v "Summary of " >%t1.dump.orig +; RUN: llvm-bcanalyzer -dump %t2.bc.thinlto.bc | grep -v "Summary of " >%t2.dump.orig + +; Next perform the thin link on the minimized bitcode files, and compare dumps +; of the resulting indexes to the above dumps to ensure they are identical. +; RUN: rm -f %t1.bc.thinlto.bc %t2.bc.thinlto.bc +; RUN: llvm-lto2 %t1.thinlink.bc %t2.thinlink.bc -o %t.o -save-temps \ +; RUN: -thinlto-distributed-indexes \ +; RUN: -thinlto-object-suffix-replace="thinlink.bc;bc" \ +; RUN: -r=%t1.thinlink.bc,g, \ +; RUN: -r=%t1.thinlink.bc,f,px \ +; RUN: -r=%t2.thinlink.bc,g,px +; RUN: llvm-bcanalyzer -dump %t1.bc.thinlto.bc | grep -v "Summary of " >%t1.dump.new +; RUN: llvm-bcanalyzer -dump %t2.bc.thinlto.bc | grep -v "Summary of " >%t2.dump.new +; RUN: diff %t1.dump.orig %t1.dump.new +; RUN: diff %t2.dump.orig %t2.dump.new +; Make sure importing occurs as expected +; RUN: opt -function-import -summary-file %t2.bc.thinlto.bc %t2.bc -o %t2.out +; RUN: llvm-dis -o - %t2.out | FileCheck %s + +; CHECK: @G.llvm. target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @@ -20,3 +47,8 @@ call i32 (...) @g() ret void } + +!llvm.dbg.cu = !{} + +!1 = !{i32 2, !"Debug Info Version", i32 3} +!llvm.module.flags = !{!1} Index: test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll =================================================================== --- test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll +++ test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll @@ -1,6 +1,33 @@ -; RUN: opt -thinlto-bc -o %t %s -; RUN: llvm-dis -o - %t | FileCheck %s -; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s +; Generate bitcode files with summary, as well as minimized bitcode without +; the debug metadata for the thin link. +; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t.thinlink.bc -o %t.bc %s +; RUN: llvm-dis -o - %t.bc | FileCheck %s +; RUN: llvm-dis -o - %t.thinlink.bc | FileCheck --check-prefix=NODEBUG %s +; RUN: llvm-bcanalyzer -dump %t.bc | FileCheck --check-prefix=BCA %s + +; Make sure the combined index files produced by both the normal and the +; thin link bitcode files are identical +; RUN: llvm-lto -thinlto -o %t3 %t.bc +; Make sure we don't inadvertently read the regular bitcode file (save +; and restore so we can use it again further down). +; RUN: mv %t.bc %t.bc.sv +; RUN: llvm-lto -thinlto -thinlto-object-suffix-replace="thinlink.bc;bc" -o %t4 %t.thinlink.bc +; RUN: mv %t.bc.sv %t.bc +; Need to grep out input filename +; RUN: llvm-bcanalyzer -dump %t3.thinlto.bc | grep -v "Summary of " >%t3.dump +; RUN: llvm-bcanalyzer -dump %t4.thinlto.bc | grep -v "Summary of " >%t4.dump +; RUN: diff %t3.dump %t4.dump + +; Try again using -thinlto-action to produce combined index +; RUN: rm -f %t3.thinlto.bc %t4.thinlto.bc +; RUN: llvm-lto -thinlto-action=thinlink -o %t3.thinlto.bc %t.bc +; Make sure we don't inadvertently read the regular bitcode file. +; RUN: rm -f %t.bc +; RUN: llvm-lto -thinlto-action=thinlink -thinlto-object-suffix-replace="thinlink.bc;bc" -o %t4.thinlto.bc %t.thinlink.bc +; Need to grep out input filename +; RUN: llvm-bcanalyzer -dump %t3.thinlto.bc | grep -v "Summary of " >%t3.dump +; RUN: llvm-bcanalyzer -dump %t4.thinlto.bc | grep -v "Summary of " >%t4.dump +; RUN: diff %t3.dump %t4.dump ; BCA: &1 | FileCheck --check-prefix=ERROR %s -; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=M0 %s -; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=M1 %s -; RUN: llvm-bcanalyzer -dump %t0 | FileCheck --check-prefix=BCA0 %s -; RUN: llvm-bcanalyzer -dump %t1 | FileCheck --check-prefix=BCA1 %s +; RUN: llvm-dis -o - %t0.bc | FileCheck --check-prefix=M0 %s +; RUN: llvm-dis -o - %t1.bc | FileCheck --check-prefix=M1 %s +; RUN: llvm-dis -o - %t0.thinlink.bc | FileCheck --check-prefix=NODEBUG %s +; RUN: llvm-dis -o - %t1.thinlink.bc | FileCheck --check-prefix=NODEBUG %s +; RUN: llvm-bcanalyzer -dump %t0.bc | FileCheck --check-prefix=BCA0 %s +; RUN: llvm-bcanalyzer -dump %t1.bc | FileCheck --check-prefix=BCA1 %s + +; Make sure the combined index files produced by both the normal and the +; thin link bitcode files are identical +; RUN: llvm-lto -thinlto -o %t3 %t0.bc +; RUN: llvm-lto -thinlto -thinlto-object-suffix-replace="thinlink.bc;bc" -o %t4 %t0.thinlink.bc +; Need to grep out input filename +; RUN: llvm-bcanalyzer -dump %t3.thinlto.bc | grep -v "Summary of " >%t3.dump +; RUN: llvm-bcanalyzer -dump %t4.thinlto.bc | grep -v "Summary of " >%t4.dump +; RUN: diff %t3.dump %t4.dump ; ERROR: llvm-modextract: error: module index out of range; bitcode file contains 2 module(s) @@ -29,6 +44,7 @@ ; M0: !llvm.dbg.cu ; M1-NOT: !llvm.dbg.cu +; NODEBUG-NOT: !llvm.dbg.cu !llvm.dbg.cu = !{} !1 = !{i32 2, !"Debug Info Version", i32 3} Index: test/tools/gold/X86/thinlto_object_suffix_replace.ll =================================================================== --- /dev/null +++ test/tools/gold/X86/thinlto_object_suffix_replace.ll @@ -0,0 +1,42 @@ +; Test to make sure the thinlto-object-suffix-replace option is handled +; correctly. + +; Generate bitcode file with summary, as well as a minimized bitcode without +; the debug metadata for the thin link. +; RUN: opt -thinlto-bc %s -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.o + +; First perform the thin link on the normal bitcode file, and save dump +; of the resulting index. +; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ +; RUN: -m elf_x86_64 \ +; RUN: --plugin-opt=thinlto \ +; RUN: --plugin-opt=thinlto-index-only \ +; RUN: -shared %t1.o -o %t3 +; RUN: llvm-bcanalyzer -dump %t1.o.thinlto.bc | grep -v "Summary of " >%t1.dump.orig + +; Next perform the thin link on the minimized bitcode file, and compare dump +; of the resulting index to the above dump to ensure they are identical. +; RUN: rm -f %t1.o.thinlto.bc +; Make sure it isn't inadvertently using the regular bitcode file. +; RUN: rm -f %t1.o +; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \ +; RUN: -m elf_x86_64 \ +; RUN: --plugin-opt=thinlto \ +; RUN: --plugin-opt=thinlto-index-only \ +; RUN: --plugin-opt=thinlto-object-suffix-replace=".thinlink.bc;.o" \ +; RUN: -shared %t1.thinlink.bc -o %t3 +; RUN: llvm-bcanalyzer -dump %t1.o.thinlto.bc | grep -v "Summary of " >%t1.dump.new +; RUN: diff %t1.dump.orig %t1.dump.new + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @f() { +entry: + ret void +} + +!llvm.dbg.cu = !{} + +!1 = !{i32 2, !"Debug Info Version", i32 3} +!llvm.module.flags = !{!1} Index: tools/gold/gold-plugin.cpp =================================================================== --- tools/gold/gold-plugin.cpp +++ tools/gold/gold-plugin.cpp @@ -164,6 +164,12 @@ // corresponding bitcode file, will use a path formed by replacing the // bitcode file's path prefix matching oldprefix with newprefix. static std::string thinlto_prefix_replace; + // Option to control the name of modules encoded in the individual index + // files for a distributed backend. This enables the use of minimized + // bitcode files for the thin link, assuming the name of the full bitcode + // file used in the backend differs just in some part of the file suffix. + // If specified, expects a string of the form "oldsuffix:newsuffix". + static std::string thinlto_object_suffix_replace; // Optional path to a directory for caching ThinLTO objects. static std::string cache_dir; // Additional options to pass into the code generator. @@ -206,6 +212,12 @@ thinlto_prefix_replace = opt.substr(strlen("thinlto-prefix-replace=")); if (thinlto_prefix_replace.find(';') == std::string::npos) message(LDPL_FATAL, "thinlto-prefix-replace expects 'old;new' format"); + } else if (opt.startswith("thinlto-object-suffix-replace=")) { + thinlto_object_suffix_replace = + opt.substr(strlen("thinlto-object-suffix-replace=")); + if (thinlto_object_suffix_replace.find(';') == std::string::npos) + message(LDPL_FATAL, + "thinlto-object-suffix-replace expects 'old;new' format"); } else if (opt.startswith("cache-dir=")) { cache_dir = opt.substr(strlen("cache-dir=")); } else if (opt.size() == 2 && opt[0] == 'O') { @@ -566,8 +578,35 @@ return View; } -static void addModule(LTO &Lto, claimed_file &F, const void *View) { - MemoryBufferRef BufferRef(StringRef((const char *)View, F.filesize), F.name); +/// Parse the thinlto-object-suffix-replace option into the \p OldSuffix and +/// \p NewSuffix strings, if it was specified. +static void getThinLTOOldAndNewSuffix(std::string &OldSuffix, + std::string &NewSuffix) { + assert(options::thinlto_object_suffix_replace.empty() || + options::thinlto_object_suffix_replace.find(";") != StringRef::npos); + StringRef SuffixReplace = options::thinlto_object_suffix_replace; + std::pair Split = SuffixReplace.split(";"); + OldSuffix = Split.first.str(); + NewSuffix = Split.second.str(); +} + +/// Given the original \p Path to an output file, replace any filename +/// suffix matching \p OldSuffix with \p NewSuffix. +static std::string getThinLTOObjectFileName(const std::string Path, + const std::string &OldSuffix, + const std::string &NewSuffix) { + if (OldSuffix.empty() && NewSuffix.empty()) + return Path; + StringRef NewPath = Path; + NewPath.consume_back(OldSuffix); + std::string NewNewPath = NewPath.str() + NewSuffix; + return NewPath.str() + NewSuffix; +} + +static void addModule(LTO &Lto, claimed_file &F, const void *View, + StringRef Filename) { + MemoryBufferRef BufferRef(StringRef((const char *)View, F.filesize), + Filename); Expected> ObjOrErr = InputFile::create(BufferRef); if (!ObjOrErr) @@ -789,19 +828,31 @@ if (options::thinlto_index_only) getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix); + std::string OldSuffix, NewSuffix; + getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); + // Set for owning string objects used as buffer identifiers. + StringSet<> ObjectFilenames; + for (claimed_file &F : Modules) { if (options::thinlto && !HandleToInputFile.count(F.leader_handle)) HandleToInputFile.insert(std::make_pair( F.leader_handle, llvm::make_unique(F.handle))); const void *View = getSymbolsAndView(F); + // In case we are thin linking with a minimized bitcode file, ensure + // the module paths encoded in the index reflect where the backends + // will locate the full bitcode files for compiling/importing. + std::string Identifier = + getThinLTOObjectFileName(F.name, OldSuffix, NewSuffix); + auto ObjFilename = ObjectFilenames.insert(Identifier); + assert(ObjFilename.second); if (!View) { if (options::thinlto_index_only) // Write empty output files that may be expected by the distributed // build system. - writeEmptyDistributedBuildOutputs(F.name, OldPrefix, NewPrefix); + writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix); continue; } - addModule(*Lto, F, View); + addModule(*Lto, F, View, ObjFilename.first->first()); } SmallString<128> Filename; Index: tools/llvm-lto/llvm-lto.cpp =================================================================== --- tools/llvm-lto/llvm-lto.cpp +++ tools/llvm-lto/llvm-lto.cpp @@ -117,6 +117,12 @@ "prefix of output file is oldprefix it will be " "replaced with newprefix.")); +static cl::opt ThinLTOObjectSuffixReplace( + "thinlto-object-suffix-replace", + cl::desc("Control the module paths encoded in the combined index. " + "Expects 'oldsuffix;newsuffix' and if suffix of thin link input " + "file is oldsuffix it will be replaced with newsuffix.")); + static cl::opt ThinLTOModuleId( "thinlto-module-id", cl::desc("For the module ID for the file to process, useful to " @@ -319,6 +325,31 @@ } } +/// Parse the thinlto-object-suffix-replace option into the \p OldSuffix and +/// \p NewSuffix strings, if it was specified. +static void getThinLTOOldAndNewSuffix(std::string &OldSuffix, + std::string &NewSuffix) { + assert(ThinLTOObjectSuffixReplace.empty() || + ThinLTOObjectSuffixReplace.find(";") != StringRef::npos); + StringRef SuffixReplace = ThinLTOObjectSuffixReplace; + std::pair Split = SuffixReplace.split(";"); + OldSuffix = Split.first.str(); + NewSuffix = Split.second.str(); +} + +/// Given the original \p Path to an output file, replace any filename +/// suffix matching \p OldSuffix with \p NewSuffix. +static std::string getThinLTOObjectFileName(const std::string Path, + const std::string &OldSuffix, + const std::string &NewSuffix) { + if (OldSuffix.empty() && NewSuffix.empty()) + return Path; + StringRef NewPath = Path; + NewPath.consume_back(OldSuffix); + std::string NewNewPath = NewPath.str() + NewSuffix; + return NewPath.str() + NewSuffix; +} + /// Create a combined index file from the input IR files and write it. /// /// This is meant to enable testing of ThinLTO combined index generation, @@ -326,10 +357,22 @@ static void createCombinedModuleSummaryIndex() { ModuleSummaryIndex CombinedIndex; uint64_t NextModuleId = 0; + std::string OldSuffix, NewSuffix; + getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); + // Set for owning string objects used as buffer identifiers. + StringSet<> ObjectFilenames; for (auto &Filename : InputFilenames) { ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename + "': "); + // In case we are thin linking with a minimized bitcode file, ensure + // the module paths encoded in the index reflect where the backends + // will locate the full bitcode files for compiling/importing. + std::string Identifier = + getThinLTOObjectFileName(Filename, OldSuffix, NewSuffix); + auto ObjFilename = ObjectFilenames.insert(Identifier); + assert(ObjFilename.second); std::unique_ptr Index = - ExitOnErr(llvm::getModuleSummaryIndexForFile(Filename)); + ExitOnErr(llvm::getModuleSummaryIndexForFile( + Filename, ObjFilename.first->first())); // Skip files without a module summary. if (!Index) continue; @@ -472,13 +515,25 @@ LLVMContext Ctx; std::vector> InputBuffers; + std::string OldSuffix, NewSuffix; + getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); + // Set for owning string objects used as buffer identifiers. + StringSet<> ObjectFilenames; for (unsigned i = 0; i < InputFilenames.size(); ++i) { auto &Filename = InputFilenames[i]; StringRef CurrentActivity = "loading file '" + Filename + "'"; auto InputOrErr = MemoryBuffer::getFile(Filename); error(InputOrErr, "error " + CurrentActivity); InputBuffers.push_back(std::move(*InputOrErr)); - ThinGenerator.addModule(Filename, InputBuffers.back()->getBuffer()); + // In case we are thin linking with a minimized bitcode file, ensure + // the module paths encoded in the index reflect where the backends + // will locate the full bitcode files for compiling/importing. + std::string Identifier = + getThinLTOObjectFileName(Filename, OldSuffix, NewSuffix); + auto ObjFilename = ObjectFilenames.insert(Identifier); + assert(ObjFilename.second); + ThinGenerator.addModule(ObjFilename.first->first(), + InputBuffers.back()->getBuffer()); } auto CombinedIndex = ThinGenerator.linkCombinedIndex(); Index: tools/llvm-lto2/llvm-lto2.cpp =================================================================== --- tools/llvm-lto2/llvm-lto2.cpp +++ tools/llvm-lto2/llvm-lto2.cpp @@ -64,6 +64,13 @@ "import files for the " "distributed backend case")); +static cl::opt ThinLTOObjectSuffixReplace( + "thinlto-object-suffix-replace", + cl::desc("Control where files for distributed backends are " + "created. Expects 'oldprefix;newprefix' and if path " + "prefix of output file is oldprefix it will be " + "replaced with newprefix.")); + static cl::opt Threads("thinlto-threads", cl::init(llvm::heavyweight_hardware_concurrency())); @@ -126,6 +133,35 @@ return T(); } +/// Parse the thinlto-object-suffix-replace option into the \p OldSuffix and +/// \p NewSuffix strings, if it was specified. +static void getThinLTOOldAndNewSuffix(std::string &OldSuffix, + std::string &NewSuffix) { + if (ThinLTOObjectSuffixReplace.empty()) + return; + assert(ThinLTOObjectSuffixReplace.find(";") != StringRef::npos); + assert(ThinLTODistributedIndexes && "Using different files for thin link and " + "backends only makes sense when writing " + "out index"); + StringRef SuffixReplace = ThinLTOObjectSuffixReplace; + std::pair Split = SuffixReplace.split(";"); + OldSuffix = Split.first.str(); + NewSuffix = Split.second.str(); +} + +/// Given the original \p Path to an output file, replace any filename +/// suffix matching \p OldSuffix with \p NewSuffix. +static std::string getThinLTOObjectFileName(const std::string Path, + const std::string &OldSuffix, + const std::string &NewSuffix) { + if (OldSuffix.empty() && NewSuffix.empty()) + return Path; + StringRef NewPath = Path; + NewPath.consume_back(OldSuffix); + std::string NewNewPath = NewPath.str() + NewSuffix; + return NewPath.str() + NewSuffix; +} + int main(int argc, char **argv) { InitializeAllTargets(); InitializeAllTargetMCs(); @@ -227,11 +263,22 @@ Backend = createInProcessThinBackend(Threads); LTO Lto(std::move(Conf), std::move(Backend)); + std::string OldSuffix, NewSuffix; + getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); + // Set for owning string objects used as buffer identifiers. + StringSet<> ObjectFilenames; + bool HasErrors = false; for (std::string F : InputFilenames) { std::unique_ptr MB = check(MemoryBuffer::getFile(F), F); - std::unique_ptr Input = - check(InputFile::create(MB->getMemBufferRef()), F); + // In case we are thin linking with a minimized bitcode file, ensure + // the module paths encoded in the index reflect where the backends + // will locate the full bitcode files for compiling/importing. + std::string Identifier = getThinLTOObjectFileName(F, OldSuffix, NewSuffix); + auto ObjFilename = ObjectFilenames.insert(Identifier); + assert(ObjFilename.second); + MemoryBufferRef BufferRef(MB->getBuffer(), ObjFilename.first->first()); + std::unique_ptr Input = check(InputFile::create(BufferRef), F); std::vector Res; for (const InputFile::Symbol &Sym : Input->symbols()) { Index: tools/opt/opt.cpp =================================================================== --- tools/opt/opt.cpp +++ tools/opt/opt.cpp @@ -102,6 +102,11 @@ OutputThinLTOBC("thinlto-bc", cl::desc("Write output as ThinLTO-ready bitcode")); +static cl::opt ThinLinkBitcodeFile( + "thin-link-bitcode-file", cl::value_desc("filename"), + cl::desc( + "A file in which to write minimized bitcode for the thin link only")); + static cl::opt NoVerify("disable-verify", cl::desc("Do not run the verifier"), cl::Hidden); @@ -700,7 +705,7 @@ report_fatal_error("Text output is incompatible with -module-hash"); Passes.add(createPrintModulePass(*OS, "", PreserveAssemblyUseListOrder)); } else if (OutputThinLTOBC) - Passes.add(createWriteThinLTOBitcodePass(*OS)); + Passes.add(createWriteThinLTOBitcodePass(*OS, ThinLinkBitcodeFile)); else Passes.add(createBitcodeWriterPass(*OS, PreserveBitcodeUseListOrder, EmitSummaryIndex, EmitModuleHash));