Index: include/llvm/Bitcode/BitcodeWriter.h =================================================================== --- include/llvm/Bitcode/BitcodeWriter.h +++ include/llvm/Bitcode/BitcodeWriter.h @@ -43,9 +43,16 @@ /// /// \p GenerateHash enables hashing the Module and including the hash in the /// bitcode (currently for use in ThinLTO incremental build). + /// + /// If \p ModHash is non-null, when GenerateHash is true, the resulting + /// hash is written into ModHash. When GenerateHash is false, that value + /// is used as the hash instead of computing from the generated bitcode. + /// Can be used to produce the same module hash for a minimized bitcode + /// used just for the thin link as in the regular full bitcode that will + /// be used in the backend. void writeModule(const Module *M, bool ShouldPreserveUseListOrder = false, const ModuleSummaryIndex *Index = nullptr, - bool GenerateHash = false); + bool GenerateHash = false, ModuleHash *ModHash = nullptr); }; /// \brief Write the specified module to the specified raw output stream. @@ -62,10 +69,18 @@ /// /// \p GenerateHash enables hashing the Module and including the hash in the /// bitcode (currently for use in ThinLTO incremental build). + /// + /// If \p ModHash is non-null, when GenerateHash is true, the resulting + /// hash is written into ModHash. When GenerateHash is false, that value + /// is used as the hash instead of computing from the generated bitcode. + /// Can be used to produce the same module hash for a minimized bitcode + /// used just for the thin link as in the regular full bitcode that will + /// be used in the backend. void WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder = false, const ModuleSummaryIndex *Index = nullptr, - bool GenerateHash = false); + bool GenerateHash = false, + ModuleHash *ModHash = nullptr); /// Write the specified module summary index to the given raw output stream, /// where it will be written in a new bitcode block. This is used when Index: include/llvm/Object/ModuleSummaryIndexObjectFile.h =================================================================== --- include/llvm/Object/ModuleSummaryIndexObjectFile.h +++ include/llvm/Object/ModuleSummaryIndexObjectFile.h @@ -88,9 +88,12 @@ } /// Parse the module summary index out of an IR file and return the module -/// summary index object if found, or nullptr if not. +/// summary index object if found, or nullptr if not. If Identifier is +/// non-empty, it is used as the module ID (module path) in the resulting +/// index. This can be used when the index is being read from a file +/// containing minimized bitcode just for the thin link. Expected> -getModuleSummaryIndexForFile(StringRef Path); +getModuleSummaryIndexForFile(StringRef Path, StringRef Identifier = ""); } #endif Index: include/llvm/Transforms/IPO.h =================================================================== --- include/llvm/Transforms/IPO.h +++ include/llvm/Transforms/IPO.h @@ -249,7 +249,8 @@ ModulePass *createSampleProfileLoaderPass(StringRef Name); /// Write ThinLTO-ready bitcode to Str. -ModulePass *createWriteThinLTOBitcodePass(raw_ostream &Str); +ModulePass *createWriteThinLTOBitcodePass(raw_ostream &Str, + std::string ThinLinkBitcodeFile = ""); } // End llvm namespace Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -108,6 +108,14 @@ /// True if a module hash record should be written. bool GenerateHash; + /// If non-null, when GenerateHash is true, the resulting hash is written + /// into ModHash. When GenerateHash is false, that specified value + /// is used as the hash instead of computing from the generated bitcode. + /// Can be used to produce the same module hash for a minimized bitcode + /// used just for the thin link as in the regular full bitcode that will + /// be used in the backend. + ModuleHash *ModHash; + /// The start bit of the identification block. uint64_t BitcodeStartBit; @@ -124,10 +132,12 @@ /// writing to the provided \p Buffer. ModuleBitcodeWriter(const Module *M, SmallVectorImpl &Buffer, BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, - const ModuleSummaryIndex *Index, bool GenerateHash) + const ModuleSummaryIndex *Index, bool GenerateHash, + ModuleHash *ModHash = nullptr) : BitcodeWriterBase(Stream), Buffer(Buffer), M(*M), VE(*M, ShouldPreserveUseListOrder), Index(Index), - GenerateHash(GenerateHash), BitcodeStartBit(Stream.GetCurrentBitNo()) { + GenerateHash(GenerateHash), ModHash(ModHash), + BitcodeStartBit(Stream.GetCurrentBitNo()) { // Assign ValueIds to any callee values in the index that came from // indirect call profiles and were recorded as a GUID not a Value* // (which would have been assigned an ID by the ValueEnumerator). @@ -3771,17 +3781,26 @@ void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { // Emit the module's hash. // MODULE_CODE_HASH: [5*i32] - SHA1 Hasher; - Hasher.update(ArrayRef((const uint8_t *)&(Buffer)[BlockStartPos], - Buffer.size() - BlockStartPos)); - StringRef Hash = Hasher.result(); - uint32_t Vals[5]; - for (int Pos = 0; Pos < 20; Pos += 4) { - Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); - } + if (GenerateHash) { + SHA1 Hasher; + uint32_t Vals[5]; + Hasher.update(ArrayRef((const uint8_t *)&(Buffer)[BlockStartPos], + Buffer.size() - BlockStartPos)); + StringRef Hash = Hasher.result(); + for (int Pos = 0; Pos < 20; Pos += 4) { + Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); + } - // Emit the finished record. - Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); + + if (ModHash) + // Save the written hash value. + std::copy(std::begin(Vals), std::end(Vals), std::begin(*ModHash)); + } else if (ModHash) { + Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef(*ModHash)); + return; + } } void ModuleBitcodeWriter::write() { @@ -3842,9 +3861,7 @@ writeValueSymbolTable(M.getValueSymbolTable(), /* IsModuleLevel */ true, &FunctionToBitcodeIndex); - if (GenerateHash) { - writeModuleHash(BlockStartPos); - } + writeModuleHash(BlockStartPos); Stream.ExitBlock(); } @@ -3935,9 +3952,10 @@ void BitcodeWriter::writeModule(const Module *M, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, - bool GenerateHash) { - ModuleBitcodeWriter ModuleWriter( - M, Buffer, *Stream, ShouldPreserveUseListOrder, Index, GenerateHash); + bool GenerateHash, ModuleHash *ModHash) { + ModuleBitcodeWriter ModuleWriter(M, Buffer, *Stream, + ShouldPreserveUseListOrder, Index, + GenerateHash, ModHash); ModuleWriter.write(); } @@ -3946,7 +3964,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, - bool GenerateHash) { + bool GenerateHash, ModuleHash *ModHash) { SmallVector Buffer; Buffer.reserve(256*1024); @@ -3957,7 +3975,8 @@ Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); BitcodeWriter Writer(Buffer); - Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash); + Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash, + ModHash); if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) emitDarwinBCHeaderAndTrailer(Buffer, TT); Index: lib/Object/ModuleSummaryIndexObjectFile.cpp =================================================================== --- lib/Object/ModuleSummaryIndexObjectFile.cpp +++ lib/Object/ModuleSummaryIndexObjectFile.cpp @@ -96,13 +96,18 @@ // Parse the module summary index out of an IR file and return the summary // index object if found, or nullptr if not. Expected> -llvm::getModuleSummaryIndexForFile(StringRef Path) { +llvm::getModuleSummaryIndexForFile(StringRef Path, StringRef Identifier) { ErrorOr> FileOrErr = MemoryBuffer::getFileOrSTDIN(Path); std::error_code EC = FileOrErr.getError(); if (EC) return errorCodeToError(EC); - MemoryBufferRef BufferRef = (FileOrErr.get())->getMemBufferRef(); + std::unique_ptr MemBuffer = std::move(FileOrErr.get()); + // If Identifier is non-empty, use it as the buffer identifier, which + // will become the module path in the index. + if (Identifier.empty()) + Identifier = MemBuffer->getBufferIdentifier(); + MemoryBufferRef BufferRef(MemBuffer->getBuffer(), Identifier); if (IgnoreEmptyThinLTOIndexFile && !BufferRef.getBufferSize()) return nullptr; Expected> ObjOrErr = Index: lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp =================================================================== --- lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -14,7 +14,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Transforms/IPO.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Analysis/TypeMetadataUtils.h" @@ -25,7 +24,10 @@ #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/FileSystem.h" #include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/Utils/Cloning.h" using namespace llvm; @@ -251,13 +253,17 @@ // a multi-module bitcode file with the two parts to OS. Otherwise, write only a // regular LTO bitcode file to OS. void splitAndWriteThinLTOBitcode( - raw_ostream &OS, function_ref AARGetter, - Module &M) { + raw_ostream &OS, raw_ostream *ThinLinkOS, + function_ref AARGetter, Module &M) { std::string ModuleId = getModuleId(&M); if (ModuleId.empty()) { // We couldn't generate a module ID for this module, just write it out as a // regular LTO module. WriteBitcodeToFile(&M, OS); + if (ThinLinkOS) + // We don't have a ThinLTO part, but still write the module to the + // ThinLinkOS if requested so that the expected output file is produced. + WriteBitcodeToFile(&M, *ThinLinkOS); return; } @@ -334,17 +340,34 @@ simplifyExternals(*MergedM); - SmallVector Buffer; - BitcodeWriter W(Buffer); // FIXME: Try to re-use BSI and PFI from the original module here. ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, nullptr); - W.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index, - /*GenerateHash=*/true); - W.writeModule(MergedM.get()); + SmallVector Buffer; + BitcodeWriter W(Buffer); + // Save the module hash produced for the full bitcode, which will + // be used in the backends, and use that in the minimized bitcode + // produced for the full link. + ModuleHash ModHash = {{0}}; + W.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index, + /*GenerateHash=*/true, &ModHash); + W.writeModule(MergedM.get()); OS << Buffer; + + // If a minimized bitcode module was requeted for the thin link, + // strip the debug info (the merged module was already stripped above) + // and write it to the given OS. + if (ThinLinkOS) { + Buffer.clear(); + BitcodeWriter W2(Buffer); + StripDebugInfo(M); + W2.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index, + /*GenerateHash=*/false, &ModHash); + W2.writeModule(MergedM.get()); + *ThinLinkOS << Buffer; + } } // Returns whether this module needs to be split because it uses type metadata. @@ -359,20 +382,36 @@ return false; } -void writeThinLTOBitcode(raw_ostream &OS, +void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS, function_ref AARGetter, Module &M, const ModuleSummaryIndex *Index) { // See if this module has any type metadata. If so, we need to split it. if (requiresSplit(M)) - return splitAndWriteThinLTOBitcode(OS, AARGetter, M); + return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M); // Otherwise we can just write it out as a regular module. + + // Save the module hash produced for the full bitcode, which will + // be used in the backends, and use that in the minimized bitcode + // produced for the full link. + ModuleHash ModHash = {{0}}; WriteBitcodeToFile(&M, OS, /*ShouldPreserveUseListOrder=*/false, Index, - /*GenerateHash=*/true); + /*GenerateHash=*/true, &ModHash); + // If a minimized bitcode module was requeted for the thin link, + // strip the debug info and write it to the given OS. + if (ThinLinkOS) { + StripDebugInfo(M); + WriteBitcodeToFile(&M, *ThinLinkOS, /*ShouldPreserveUseListOrder=*/false, + Index, + /*GenerateHash=*/false, &ModHash); + } } class WriteThinLTOBitcode : public ModulePass { raw_ostream &OS; // raw_ostream to print on + // The output stream on which to emit a minimized module for use + // just in the thin link, if requested. + std::unique_ptr ThinLinkOS; public: static char ID; // Pass identification, replacement for typeid @@ -380,9 +419,24 @@ initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry()); } - explicit WriteThinLTOBitcode(raw_ostream &o) + explicit WriteThinLTOBitcode(raw_ostream &o, std::string ThinLinkBitcodeFile) : ModulePass(ID), OS(o) { initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry()); + setUpThinLtoBitcodeFile(ThinLinkBitcodeFile); + } + + // The thin link only needs the module symbol table and summary sections. + // If requested, create the OS for a minimized module to be written. + // For now we simply strip debug metadata. + void setUpThinLtoBitcodeFile(std::string ThinLinkBitcodeFile) { + if (!ThinLinkBitcodeFile.empty()) { + std::error_code Error; + ThinLinkOS.reset( + new raw_fd_ostream(ThinLinkBitcodeFile, Error, sys::fs::F_None)); + if (Error) + report_fatal_error(Twine("Failed to open ") + ThinLinkBitcodeFile + + " to save thin link bitcode\n"); + } } StringRef getPassName() const override { return "ThinLTO Bitcode Writer"; } @@ -390,7 +444,7 @@ bool runOnModule(Module &M) override { const ModuleSummaryIndex *Index = &(getAnalysis().getIndex()); - writeThinLTOBitcode(OS, LegacyAARGetter(*this), M, Index); + writeThinLTOBitcode(OS, ThinLinkOS.get(), LegacyAARGetter(*this), M, Index); return true; } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -411,6 +465,8 @@ INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode", "Write ThinLTO Bitcode", false, true) -ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str) { - return new WriteThinLTOBitcode(Str); +ModulePass * +llvm::createWriteThinLTOBitcodePass(raw_ostream &Str, + std::string ThinLinkBitcodeFile) { + return new WriteThinLTOBitcode(Str, ThinLinkBitcodeFile); } Index: test/ThinLTO/X86/distributed_import.ll =================================================================== --- test/ThinLTO/X86/distributed_import.ll +++ test/ThinLTO/X86/distributed_import.ll @@ -1,15 +1,40 @@ -; RUN: opt -module-summary %s -o %t1.bc -; RUN: opt -module-summary %p/Inputs/distributed_import.ll -o %t2.bc +; Test distributed build thin link output from llvm-lto2 +; Generate bitcode files with summary, as well as minimized bitcode without +; the debug metadata for the thin link. +; RUN: opt -thinlto-bc %s -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc +; RUN: opt -thinlto-bc %p/Inputs/distributed_import.ll -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc + +; First perform the thin link on the normal bitcode file. ; RUN: llvm-lto2 %t1.bc %t2.bc -o %t.o -save-temps \ ; RUN: -thinlto-distributed-indexes \ ; RUN: -r=%t1.bc,g, \ ; RUN: -r=%t1.bc,f,px \ ; RUN: -r=%t2.bc,g,px -; RUN: opt -function-import -summary-file %t1.bc.thinlto.bc %t1.bc -o %t1.out +; RUN: opt -function-import -summary-file %t1.bc.thinlto.bc %t1.bc -o %t1.out ; RUN: opt -function-import -summary-file %t2.bc.thinlto.bc %t2.bc -o %t2.out ; RUN: llvm-dis -o - %t2.out | FileCheck %s -; CHECK: @G.llvm.0 + +; Save the generated index files. +; RUN: cp %t1.bc.thinlto.bc %t1.bc.thinlto.bc.orig +; RUN: cp %t2.bc.thinlto.bc %t2.bc.thinlto.bc.orig + +; Next perform the thin link on the minimized bitcode files, and compare dumps +; of the resulting indexes to the above dumps to ensure they are identical. +; RUN: rm -f %t1.bc.thinlto.bc %t2.bc.thinlto.bc +; RUN: llvm-lto2 %t1.thinlink.bc %t2.thinlink.bc -o %t.o -save-temps \ +; RUN: -thinlto-distributed-indexes \ +; RUN: -thinlto-object-suffix-replace="thinlink.bc;bc" \ +; RUN: -r=%t1.thinlink.bc,g, \ +; RUN: -r=%t1.thinlink.bc,f,px \ +; RUN: -r=%t2.thinlink.bc,g,px +; RUN: diff %t1.bc.thinlto.bc.orig %t1.bc.thinlto.bc +; RUN: diff %t2.bc.thinlto.bc.orig %t2.bc.thinlto.bc +; Make sure importing occurs as expected +; RUN: opt -function-import -summary-file %t2.bc.thinlto.bc %t2.bc -o %t2.out +; RUN: llvm-dis -o - %t2.out | FileCheck %s + +; CHECK: @G.llvm. target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @@ -20,3 +45,8 @@ call i32 (...) @g() ret void } + +!llvm.dbg.cu = !{} + +!1 = !{i32 2, !"Debug Info Version", i32 3} +!llvm.module.flags = !{!1} Index: test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll =================================================================== --- test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll +++ test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll @@ -1,6 +1,27 @@ -; RUN: opt -thinlto-bc -o %t %s -; RUN: llvm-dis -o - %t | FileCheck %s -; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s +; Generate bitcode files with summary, as well as minimized bitcode without +; the debug metadata for the thin link. +; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t.thinlink.bc -o %t.bc %s +; RUN: llvm-dis -o - %t.bc | FileCheck %s +; RUN: llvm-dis -o - %t.thinlink.bc | FileCheck --check-prefix=NODEBUG %s +; RUN: llvm-bcanalyzer -dump %t.bc | FileCheck --check-prefix=BCA %s + +; Make sure the combined index files produced by both the normal and the +; thin link bitcode files are identical +; RUN: llvm-lto -thinlto -o %t3 %t.bc +; Make sure we don't inadvertently read the regular bitcode file (save +; and restore so we can use it again further down). +; RUN: mv %t.bc %t.bc.sv +; RUN: llvm-lto -thinlto -thinlto-object-suffix-replace="thinlink.bc;bc" -o %t4 %t.thinlink.bc +; RUN: mv %t.bc.sv %t.bc +; RUN: diff %t3.thinlto.bc %t4.thinlto.bc + +; Try again using -thinlto-action to produce combined index +; RUN: rm -f %t3.thinlto.bc %t4.thinlto.bc +; RUN: llvm-lto -thinlto-action=thinlink -o %t3.thinlto.bc %t.bc +; Make sure we don't inadvertently read the regular bitcode file. +; RUN: rm -f %t.bc +; RUN: llvm-lto -thinlto-action=thinlink -thinlto-object-suffix-replace="thinlink.bc;bc" -o %t4.thinlto.bc %t.thinlink.bc +; RUN: diff %t3.thinlto.bc %t4.thinlto.bc ; BCA: &1 | FileCheck --check-prefix=ERROR %s -; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=M0 %s -; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=M1 %s -; RUN: llvm-bcanalyzer -dump %t0 | FileCheck --check-prefix=BCA0 %s -; RUN: llvm-bcanalyzer -dump %t1 | FileCheck --check-prefix=BCA1 %s +; RUN: llvm-dis -o - %t0.bc | FileCheck --check-prefix=M0 %s +; RUN: llvm-dis -o - %t1.bc | FileCheck --check-prefix=M1 %s +; RUN: llvm-dis -o - %t0.thinlink.bc | FileCheck --check-prefix=NODEBUG %s +; RUN: llvm-dis -o - %t1.thinlink.bc | FileCheck --check-prefix=NODEBUG %s +; RUN: llvm-bcanalyzer -dump %t0.bc | FileCheck --check-prefix=BCA0 %s +; RUN: llvm-bcanalyzer -dump %t1.bc | FileCheck --check-prefix=BCA1 %s + +; Make sure the combined index files produced by both the normal and the +; thin link bitcode files are identical +; RUN: llvm-lto -thinlto -o %t3 %t0.bc +; RUN: llvm-lto -thinlto -thinlto-object-suffix-replace="thinlink.bc;bc" -o %t4 %t0.thinlink.bc +; RUN: diff %t3.thinlto.bc %t4.thinlto.bc ; ERROR: llvm-modextract: error: module index out of range; bitcode file contains 2 module(s) @@ -29,6 +41,7 @@ ; M0: !llvm.dbg.cu ; M1-NOT: !llvm.dbg.cu +; NODEBUG-NOT: !llvm.dbg.cu !llvm.dbg.cu = !{} !1 = !{i32 2, !"Debug Info Version", i32 3} Index: test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll =================================================================== --- test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll +++ test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll @@ -1,6 +1,9 @@ -; RUN: opt -thinlto-bc -o %t %s +; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -o %t %s ; RUN: llvm-dis -o - %t | FileCheck %s ; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s +; When not splitting the module, the thin link bitcode file should simply be a +; copy of the regular module. +; RUN: diff %t %t2 ; BCA-NOT: Split = SuffixReplace.split(";"); + OldSuffix = Split.first.str(); + NewSuffix = Split.second.str(); +} + +/// Given the original \p Path to an output file, replace any filename +/// suffix matching \p OldSuffix with \p NewSuffix. +static std::string getThinLTOObjectFileName(const std::string Path, + const std::string &OldSuffix, + const std::string &NewSuffix) { + if (OldSuffix.empty() && NewSuffix.empty()) + return Path; + StringRef NewPath = Path; + NewPath.consume_back(OldSuffix); + std::string NewNewPath = NewPath.str() + NewSuffix; + return NewPath.str() + NewSuffix; +} + +static void addModule(LTO &Lto, claimed_file &F, const void *View, + StringRef Filename) { + MemoryBufferRef BufferRef(StringRef((const char *)View, F.filesize), + Filename); Expected> ObjOrErr = InputFile::create(BufferRef); if (!ObjOrErr) @@ -789,19 +828,31 @@ if (options::thinlto_index_only) getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix); + std::string OldSuffix, NewSuffix; + getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); + // Set for owning string objects used as buffer identifiers. + StringSet<> ObjectFilenames; + for (claimed_file &F : Modules) { if (options::thinlto && !HandleToInputFile.count(F.leader_handle)) HandleToInputFile.insert(std::make_pair( F.leader_handle, llvm::make_unique(F.handle))); const void *View = getSymbolsAndView(F); + // In case we are thin linking with a minimized bitcode file, ensure + // the module paths encoded in the index reflect where the backends + // will locate the full bitcode files for compiling/importing. + std::string Identifier = + getThinLTOObjectFileName(F.name, OldSuffix, NewSuffix); + auto ObjFilename = ObjectFilenames.insert(Identifier); + assert(ObjFilename.second); if (!View) { if (options::thinlto_index_only) // Write empty output files that may be expected by the distributed // build system. - writeEmptyDistributedBuildOutputs(F.name, OldPrefix, NewPrefix); + writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix); continue; } - addModule(*Lto, F, View); + addModule(*Lto, F, View, ObjFilename.first->first()); } SmallString<128> Filename; Index: tools/llvm-lto/llvm-lto.cpp =================================================================== --- tools/llvm-lto/llvm-lto.cpp +++ tools/llvm-lto/llvm-lto.cpp @@ -117,6 +117,12 @@ "prefix of output file is oldprefix it will be " "replaced with newprefix.")); +static cl::opt ThinLTOObjectSuffixReplace( + "thinlto-object-suffix-replace", + cl::desc("Control the module paths encoded in the combined index. " + "Expects 'oldsuffix;newsuffix' and if suffix of thin link input " + "file is oldsuffix it will be replaced with newsuffix.")); + static cl::opt ThinLTOModuleId( "thinlto-module-id", cl::desc("For the module ID for the file to process, useful to " @@ -319,6 +325,31 @@ } } +/// Parse the thinlto-object-suffix-replace option into the \p OldSuffix and +/// \p NewSuffix strings, if it was specified. +static void getThinLTOOldAndNewSuffix(std::string &OldSuffix, + std::string &NewSuffix) { + assert(ThinLTOObjectSuffixReplace.empty() || + ThinLTOObjectSuffixReplace.find(";") != StringRef::npos); + StringRef SuffixReplace = ThinLTOObjectSuffixReplace; + std::pair Split = SuffixReplace.split(";"); + OldSuffix = Split.first.str(); + NewSuffix = Split.second.str(); +} + +/// Given the original \p Path to an output file, replace any filename +/// suffix matching \p OldSuffix with \p NewSuffix. +static std::string getThinLTOObjectFileName(const std::string Path, + const std::string &OldSuffix, + const std::string &NewSuffix) { + if (OldSuffix.empty() && NewSuffix.empty()) + return Path; + StringRef NewPath = Path; + NewPath.consume_back(OldSuffix); + std::string NewNewPath = NewPath.str() + NewSuffix; + return NewPath.str() + NewSuffix; +} + /// Create a combined index file from the input IR files and write it. /// /// This is meant to enable testing of ThinLTO combined index generation, @@ -326,10 +357,22 @@ static void createCombinedModuleSummaryIndex() { ModuleSummaryIndex CombinedIndex; uint64_t NextModuleId = 0; + std::string OldSuffix, NewSuffix; + getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); + // Set for owning string objects used as buffer identifiers. + StringSet<> ObjectFilenames; for (auto &Filename : InputFilenames) { ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename + "': "); + // In case we are thin linking with a minimized bitcode file, ensure + // the module paths encoded in the index reflect where the backends + // will locate the full bitcode files for compiling/importing. + std::string Identifier = + getThinLTOObjectFileName(Filename, OldSuffix, NewSuffix); + auto ObjFilename = ObjectFilenames.insert(Identifier); + assert(ObjFilename.second); std::unique_ptr Index = - ExitOnErr(llvm::getModuleSummaryIndexForFile(Filename)); + ExitOnErr(llvm::getModuleSummaryIndexForFile( + Filename, ObjFilename.first->first())); // Skip files without a module summary. if (!Index) continue; @@ -472,13 +515,25 @@ LLVMContext Ctx; std::vector> InputBuffers; + std::string OldSuffix, NewSuffix; + getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); + // Set for owning string objects used as buffer identifiers. + StringSet<> ObjectFilenames; for (unsigned i = 0; i < InputFilenames.size(); ++i) { auto &Filename = InputFilenames[i]; StringRef CurrentActivity = "loading file '" + Filename + "'"; auto InputOrErr = MemoryBuffer::getFile(Filename); error(InputOrErr, "error " + CurrentActivity); InputBuffers.push_back(std::move(*InputOrErr)); - ThinGenerator.addModule(Filename, InputBuffers.back()->getBuffer()); + // In case we are thin linking with a minimized bitcode file, ensure + // the module paths encoded in the index reflect where the backends + // will locate the full bitcode files for compiling/importing. + std::string Identifier = + getThinLTOObjectFileName(Filename, OldSuffix, NewSuffix); + auto ObjFilename = ObjectFilenames.insert(Identifier); + assert(ObjFilename.second); + ThinGenerator.addModule(ObjFilename.first->first(), + InputBuffers.back()->getBuffer()); } auto CombinedIndex = ThinGenerator.linkCombinedIndex(); Index: tools/llvm-lto2/llvm-lto2.cpp =================================================================== --- tools/llvm-lto2/llvm-lto2.cpp +++ tools/llvm-lto2/llvm-lto2.cpp @@ -64,6 +64,13 @@ "import files for the " "distributed backend case")); +static cl::opt ThinLTOObjectSuffixReplace( + "thinlto-object-suffix-replace", + cl::desc("Control where files for distributed backends are " + "created. Expects 'oldprefix;newprefix' and if path " + "prefix of output file is oldprefix it will be " + "replaced with newprefix.")); + static cl::opt Threads("thinlto-threads", cl::init(llvm::heavyweight_hardware_concurrency())); @@ -126,6 +133,35 @@ return T(); } +/// Parse the thinlto-object-suffix-replace option into the \p OldSuffix and +/// \p NewSuffix strings, if it was specified. +static void getThinLTOOldAndNewSuffix(std::string &OldSuffix, + std::string &NewSuffix) { + if (ThinLTOObjectSuffixReplace.empty()) + return; + assert(ThinLTOObjectSuffixReplace.find(";") != StringRef::npos); + assert(ThinLTODistributedIndexes && "Using different files for thin link and " + "backends only makes sense when writing " + "out index"); + StringRef SuffixReplace = ThinLTOObjectSuffixReplace; + std::pair Split = SuffixReplace.split(";"); + OldSuffix = Split.first.str(); + NewSuffix = Split.second.str(); +} + +/// Given the original \p Path to an output file, replace any filename +/// suffix matching \p OldSuffix with \p NewSuffix. +static std::string getThinLTOObjectFileName(const std::string Path, + const std::string &OldSuffix, + const std::string &NewSuffix) { + if (OldSuffix.empty() && NewSuffix.empty()) + return Path; + StringRef NewPath = Path; + NewPath.consume_back(OldSuffix); + std::string NewNewPath = NewPath.str() + NewSuffix; + return NewPath.str() + NewSuffix; +} + int main(int argc, char **argv) { InitializeAllTargets(); InitializeAllTargetMCs(); @@ -227,11 +263,22 @@ Backend = createInProcessThinBackend(Threads); LTO Lto(std::move(Conf), std::move(Backend)); + std::string OldSuffix, NewSuffix; + getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix); + // Set for owning string objects used as buffer identifiers. + StringSet<> ObjectFilenames; + bool HasErrors = false; for (std::string F : InputFilenames) { std::unique_ptr MB = check(MemoryBuffer::getFile(F), F); - std::unique_ptr Input = - check(InputFile::create(MB->getMemBufferRef()), F); + // In case we are thin linking with a minimized bitcode file, ensure + // the module paths encoded in the index reflect where the backends + // will locate the full bitcode files for compiling/importing. + std::string Identifier = getThinLTOObjectFileName(F, OldSuffix, NewSuffix); + auto ObjFilename = ObjectFilenames.insert(Identifier); + assert(ObjFilename.second); + MemoryBufferRef BufferRef(MB->getBuffer(), ObjFilename.first->first()); + std::unique_ptr Input = check(InputFile::create(BufferRef), F); std::vector Res; for (const InputFile::Symbol &Sym : Input->symbols()) { Index: tools/opt/opt.cpp =================================================================== --- tools/opt/opt.cpp +++ tools/opt/opt.cpp @@ -102,6 +102,11 @@ OutputThinLTOBC("thinlto-bc", cl::desc("Write output as ThinLTO-ready bitcode")); +static cl::opt ThinLinkBitcodeFile( + "thin-link-bitcode-file", cl::value_desc("filename"), + cl::desc( + "A file in which to write minimized bitcode for the thin link only")); + static cl::opt NoVerify("disable-verify", cl::desc("Do not run the verifier"), cl::Hidden); @@ -700,7 +705,7 @@ report_fatal_error("Text output is incompatible with -module-hash"); Passes.add(createPrintModulePass(*OS, "", PreserveAssemblyUseListOrder)); } else if (OutputThinLTOBC) - Passes.add(createWriteThinLTOBitcodePass(*OS)); + Passes.add(createWriteThinLTOBitcodePass(*OS, ThinLinkBitcodeFile)); else Passes.add(createBitcodeWriterPass(*OS, PreserveBitcodeUseListOrder, EmitSummaryIndex, EmitModuleHash));