Index: include/llvm/Bitcode/BitcodeReader.h =================================================================== --- include/llvm/Bitcode/BitcodeReader.h +++ include/llvm/Bitcode/BitcodeReader.h @@ -61,9 +61,10 @@ friend Expected> getBitcodeModuleList(MemoryBufferRef Buffer); - Expected> - getModuleImpl(LLVMContext &Context, bool MaterializeAll, - bool ShouldLazyLoadMetadata); + Expected> getModuleImpl(LLVMContext &Context, + bool MaterializeAll, + bool ShouldLazyLoadMetadata, + bool IsImporting); public: StringRef getBuffer() const { @@ -72,8 +73,11 @@ /// Read the bitcode module and prepare for lazy deserialization of function /// bodies. If ShouldLazyLoadMetadata is true, lazily load metadata as well. - Expected> - getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata); + /// If IsImporting is true, this module is being parsed for ThinLTO + /// importing into another module. + Expected> getLazyModule(LLVMContext &Context, + bool ShouldLazyLoadMetadata, + bool IsImporting); /// Read the entire bitcode module and return it. Expected> parseModule(LLVMContext &Context); @@ -91,18 +95,20 @@ /// Read the header of the specified bitcode buffer and prepare for lazy /// deserialization of function bodies. If ShouldLazyLoadMetadata is true, - /// lazily load metadata as well. + /// lazily load metadata as well. If IsImporting is true, this module is + /// being parsed for ThinLTO importing into another module. Expected> getLazyBitcodeModule(MemoryBufferRef Buffer, LLVMContext &Context, - bool ShouldLazyLoadMetadata = false); + bool ShouldLazyLoadMetadata = false, + bool IsImporting = false); /// Like getLazyBitcodeModule, except that the module takes ownership of /// the memory buffer if successful. If successful, this moves Buffer. On - /// error, this *does not* move Buffer. - Expected> - getOwningLazyBitcodeModule(std::unique_ptr &&Buffer, - LLVMContext &Context, - bool ShouldLazyLoadMetadata = false); + /// error, this *does not* move Buffer. If IsImporting is true, this module is + /// being parsed for ThinLTO importing into another module. + Expected> getOwningLazyBitcodeModule( + std::unique_ptr &&Buffer, LLVMContext &Context, + bool ShouldLazyLoadMetadata = false, bool IsImporting = false); /// Read the header of the specified bitcode buffer and extract just the /// triple information. If successful, this returns a string. On error, this Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -475,7 +475,8 @@ /// \brief Main interface to parsing a bitcode buffer. /// \returns true if an error occurred. - Error parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata = false); + Error parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata = false, + bool IsImporting = false); static uint64_t decodeSignRotatedValue(uint64_t V); @@ -3071,9 +3072,10 @@ } } -Error BitcodeReader::parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata) { +Error BitcodeReader::parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata, + bool IsImporting) { TheModule = M; - MDLoader = MetadataLoader(Stream, *M, ValueList, + MDLoader = MetadataLoader(Stream, *M, ValueList, IsImporting, [&](unsigned ID) { return getTypeByID(ID); }); return parseModule(0, ShouldLazyLoadMetadata); } @@ -5196,7 +5198,7 @@ /// everything. Expected> BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll, - bool ShouldLazyLoadMetadata) { + bool ShouldLazyLoadMetadata, bool IsImporting) { BitstreamCursor Stream(Buffer); std::string ProducerIdentification; @@ -5219,7 +5221,8 @@ M->setMaterializer(R); // Delay parsing Metadata if ShouldLazyLoadMetadata is true. - if (Error Err = R->parseBitcodeInto(M.get(), ShouldLazyLoadMetadata)) + if (Error Err = + R->parseBitcodeInto(M.get(), ShouldLazyLoadMetadata, IsImporting)) return std::move(Err); if (MaterializeAll) { @@ -5235,9 +5238,9 @@ } Expected> -BitcodeModule::getLazyModule(LLVMContext &Context, - bool ShouldLazyLoadMetadata) { - return getModuleImpl(Context, false, ShouldLazyLoadMetadata); +BitcodeModule::getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, + bool IsImporting) { + return getModuleImpl(Context, false, ShouldLazyLoadMetadata, IsImporting); } // Parse the specified bitcode buffer, returning the function info index. @@ -5299,20 +5302,20 @@ } Expected> -llvm::getLazyBitcodeModule(MemoryBufferRef Buffer, - LLVMContext &Context, bool ShouldLazyLoadMetadata) { +llvm::getLazyBitcodeModule(MemoryBufferRef Buffer, LLVMContext &Context, + bool ShouldLazyLoadMetadata, bool IsImporting) { Expected BM = getSingleModule(Buffer); if (!BM) return BM.takeError(); - return BM->getLazyModule(Context, ShouldLazyLoadMetadata); + return BM->getLazyModule(Context, ShouldLazyLoadMetadata, IsImporting); } -Expected> -llvm::getOwningLazyBitcodeModule(std::unique_ptr &&Buffer, - LLVMContext &Context, - bool ShouldLazyLoadMetadata) { - auto MOrErr = getLazyBitcodeModule(*Buffer, Context, ShouldLazyLoadMetadata); +Expected> llvm::getOwningLazyBitcodeModule( + std::unique_ptr &&Buffer, LLVMContext &Context, + bool ShouldLazyLoadMetadata, bool IsImporting) { + auto MOrErr = getLazyBitcodeModule(*Buffer, Context, ShouldLazyLoadMetadata, + IsImporting); if (MOrErr) (*MOrErr)->setOwnedMemoryBuffer(std::move(Buffer)); return MOrErr; @@ -5320,7 +5323,7 @@ Expected> BitcodeModule::parseModule(LLVMContext &Context) { - return getModuleImpl(Context, true, false); + return getModuleImpl(Context, true, false, false); // TODO: Restore the use-lists to the in-memory state when the bitcode was // written. We must defer until the Module has been fully materialized. } Index: lib/Bitcode/Reader/MetadataLoader.h =================================================================== --- lib/Bitcode/Reader/MetadataLoader.h +++ lib/Bitcode/Reader/MetadataLoader.h @@ -36,12 +36,14 @@ class MetadataLoader { class MetadataLoaderImpl; std::unique_ptr Pimpl; + /// True if metadata is being parsed for a module being ThinLTO imported. + bool IsImporting = false; Error parseMetadata(bool ModuleLevel); public: ~MetadataLoader(); MetadataLoader(BitstreamCursor &Stream, Module &TheModule, - BitcodeReaderValueList &ValueList, + BitcodeReaderValueList &ValueList, bool IsImporting, std::function getTypeByID); MetadataLoader &operator=(MetadataLoader &&); MetadataLoader(MetadataLoader &&); Index: lib/Bitcode/Reader/MetadataLoader.cpp =================================================================== --- lib/Bitcode/Reader/MetadataLoader.cpp +++ lib/Bitcode/Reader/MetadataLoader.cpp @@ -86,6 +86,12 @@ using namespace llvm; +/// Flag whether we need to import full type definitions for ThinLTO. +/// Currently needed for Darwin and LLDB. +static cl::opt ImportFullTypeDefinitions( + "import-full-type-definitions", cl::init(false), cl::Hidden, + cl::desc("Import full type definitions for ThinLTO.")); + namespace { static int64_t unrotateSign(uint64_t U) { return U & 1 ? ~(U >> 1) : U >> 1; } @@ -398,7 +404,7 @@ Stream(Stream), Context(TheModule.getContext()), TheModule(TheModule), getTypeByID(getTypeByID) {} - Error parseMetadata(bool ModuleLevel); + Error parseMetadata(bool ModuleLevel, bool IsImporting); bool hasFwdRefs() const { return MetadataList.hasFwdRefs(); } Metadata *getMetadataFwdRef(unsigned Idx) { @@ -431,7 +437,8 @@ /// Parse a METADATA_BLOCK. If ModuleLevel is true then we are parsing /// module level metadata. -Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { +Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel, + bool IsImporting) { if (!ModuleLevel && MetadataList.hasFwdRefs()) return error("Invalid metadata: fwd refs into function blocks"); @@ -717,6 +724,25 @@ Metadata *VTableHolder = getDITypeRefOrNull(Record[13]); Metadata *TemplateParams = getMDOrNull(Record[14]); auto *Identifier = getMDString(Record[15]); + // If this module is being parsed so that it can be ThinLTO imported + // into another module, composite types only need to be imported + // as type declarations (unless full type definitions requested). + // Create type declarations up front to save memory. Also, buildODRType + // handles the case where this is type ODRed with a definition needed + // by the importing module, in which case the existing definition is + // used. + if (IsImporting && !ImportFullTypeDefinitions && + (Tag == dwarf::DW_TAG_enumeration_type || + Tag == dwarf::DW_TAG_class_type || + Tag == dwarf::DW_TAG_structure_type || + Tag == dwarf::DW_TAG_union_type)) { + BaseType = nullptr; + OffsetInBits = 0; + Flags = Flags | DINode::FlagFwdDecl; + Elements = nullptr; + VTableHolder = nullptr; + TemplateParams = nullptr; + } DICompositeType *CT = nullptr; if (Identifier) CT = DICompositeType::buildODRType( @@ -1274,17 +1300,19 @@ return *this; } MetadataLoader::MetadataLoader(MetadataLoader &&RHS) - : Pimpl(std::move(RHS.Pimpl)) {} + : Pimpl(std::move(RHS.Pimpl)), IsImporting(RHS.IsImporting) {} MetadataLoader::~MetadataLoader() = default; MetadataLoader::MetadataLoader(BitstreamCursor &Stream, Module &TheModule, BitcodeReaderValueList &ValueList, + bool IsImporting, std::function getTypeByID) : Pimpl(llvm::make_unique(Stream, TheModule, ValueList, - getTypeByID)) {} + getTypeByID)), + IsImporting(IsImporting) {} Error MetadataLoader::parseMetadata(bool ModuleLevel) { - return Pimpl->parseMetadata(ModuleLevel); + return Pimpl->parseMetadata(ModuleLevel, IsImporting); } bool MetadataLoader::hasFwdRefs() const { return Pimpl->hasFwdRefs(); } Index: lib/LTO/LTOBackend.cpp =================================================================== --- lib/LTO/LTOBackend.cpp +++ lib/LTO/LTOBackend.cpp @@ -354,7 +354,8 @@ assert(Mod.getContext().isODRUniquingDebugTypes() && "ODR Type uniquing should be enabled on the context"); return getLazyBitcodeModule(ModuleMap[Identifier], Mod.getContext(), - /*ShouldLazyLoadMetadata=*/true); + /*ShouldLazyLoadMetadata=*/true, + /*IsImporting*/ true); }; FunctionImporter Importer(CombinedIndex, ModuleLoader); Index: lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- lib/LTO/ThinLTOCodeGenerator.cpp +++ lib/LTO/ThinLTOCodeGenerator.cpp @@ -168,12 +168,13 @@ static std::unique_ptr loadModuleFromBuffer(const MemoryBufferRef &Buffer, LLVMContext &Context, - bool Lazy) { + bool Lazy, bool IsImporting) { SMDiagnostic Err; Expected> ModuleOrErr = - Lazy ? getLazyBitcodeModule(Buffer, Context, - /* ShouldLazyLoadMetadata */ true) - : parseBitcodeFile(Buffer, Context); + Lazy + ? getLazyBitcodeModule(Buffer, Context, + /* ShouldLazyLoadMetadata */ true, IsImporting) + : parseBitcodeFile(Buffer, Context); if (!ModuleOrErr) { handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) { SMDiagnostic Err = SMDiagnostic(Buffer.getBufferIdentifier(), @@ -191,7 +192,7 @@ const FunctionImporter::ImportMapTy &ImportList) { auto Loader = [&](StringRef Identifier) { return loadModuleFromBuffer(ModuleMap[Identifier], TheModule.getContext(), - /*Lazy=*/true); + /*Lazy=*/true, /*IsImporting*/ true); }; FunctionImporter Importer(Index, Loader); @@ -759,7 +760,8 @@ Context.setDiscardValueNames(LTODiscardValueNames); // Parse module now - auto TheModule = loadModuleFromBuffer(ModuleBuffer, Context, false); + auto TheModule = loadModuleFromBuffer(ModuleBuffer, Context, false, + /*IsImporting*/ false); // CodeGen ProducedBinaries[count] = codegen(*TheModule); @@ -890,7 +892,8 @@ } // Parse module now - auto TheModule = loadModuleFromBuffer(ModuleBuffer, Context, false); + auto TheModule = loadModuleFromBuffer(ModuleBuffer, Context, false, + /*IsImporting*/ false); // Save temps: original file. saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc"); Index: test/ThinLTO/X86/Inputs/debuginfo-compositetype-import.ll =================================================================== --- /dev/null +++ test/ThinLTO/X86/Inputs/debuginfo-compositetype-import.ll @@ -0,0 +1,13 @@ +; ModuleID = 'debuginfo-compositetype-import2.c' +source_filename = "debuginfo-compositetype-import2.c" +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define i32 @main() { +entry: + call void (...) @foo() + ret i32 0 +} + +declare void @foo(...) #1 Index: test/ThinLTO/X86/debuginfo-compositetype-import.ll =================================================================== --- /dev/null +++ test/ThinLTO/X86/debuginfo-compositetype-import.ll @@ -0,0 +1,49 @@ +; Test to ensure DICompositeType are imported as type declarations +; for ThinLTO + +; RUN: opt -module-summary %s -o %t1.bc +; RUN: opt -module-summary %p/Inputs/debuginfo-compositetype-import.ll -o %t2.bc +; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc + +; By default, composite types are imported as type declarations +; RUN: llvm-lto -thinlto-action=import %t2.bc -thinlto-index=%t.index.bc -o - | llvm-dis -o - | FileCheck %s +; CHECK: distinct !DICompositeType(tag: DW_TAG_enumeration_type, name: "enum", scope: !{{[0-9]+}}, file: !{{[0-9]+}}, line: 50, size: 32, flags: DIFlagFwdDecl, identifier: "enum") +; CHECK: distinct !DICompositeType(tag: DW_TAG_class_type, name: "class", scope: !{{[0-9]+}}, file: !{{[0-9]+}}, line: 728, size: 448, flags: DIFlagFwdDecl, identifier: "class") +; CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "struct", scope: !{{[0-9]+}}, file: !{{[0-9]+}}, line: 309, size: 128, flags: DIFlagFwdDecl, identifier: "list") +; CHECK: distinct !DICompositeType(tag: DW_TAG_union_type, file: !{{[0-9]+}}, line: 115, size: 384, flags: DIFlagFwdDecl, identifier: "union") + +; Ensure that full type definitions of composite types are imported if requested +; RUN: llvm-lto -import-full-type-definitions -thinlto-action=import %t2.bc -thinlto-index=%t.index.bc -o - | llvm-dis -o - | FileCheck %s --check-prefix=FULL +; FULL: distinct !DICompositeType(tag: DW_TAG_enumeration_type, name: "enum", scope: !{{[0-9]+}}, file: !{{[0-9]+}}, line: 50, size: 32, elements: !{{[0-9]+}}, identifier: "enum") +; FULL: distinct !DICompositeType(tag: DW_TAG_class_type, name: "class", scope: !{{[0-9]+}}, file: !{{[0-9]+}}, line: 728, size: 448, elements: !{{[0-9]+}}, identifier: "class") +; FULL: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "struct", scope: !{{[0-9]+}}, file: !{{[0-9]+}}, line: 309, baseType: !{{[0-9]+}}, size: 128, offset: 64, elements: !{{[0-9]+}}, vtableHolder: !{{[0-9]+}}, templateParams: !{{[0-9]+}}, identifier: "list") +; FULL: distinct !DICompositeType(tag: DW_TAG_union_type, file: !{{[0-9]+}}, line: 115, size: 384, elements: !{{[0-9]+}}, identifier: "union") + +; ModuleID = 'debuginfo-compositetype-import.c' +source_filename = "debuginfo-compositetype-import.c" +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define void @foo() #0 !dbg !6 { +entry: + ret void +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} +!llvm.ident = !{!4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 4.0.0 (trunk 286863) (llvm/trunk 286875)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug) +!1 = !DIFile(filename: "debuginfo-compositetype-import.c", directory: "") +!2 = !{i32 2, !"Dwarf Version", i32 4} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !{!"clang version 4.0.0 (trunk 286863) (llvm/trunk 286875)"} +!5 = !{} +!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: false, unit: !0, variables: !5) +!7 = !DISubroutineType(types: !8) +!8 = !{!9, !10, !11, !12} +!9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "enum", scope: !1, file: !1, line: 50, size: 32, elements: !5, identifier: "enum") +!10 = !DICompositeType(tag: DW_TAG_class_type, name: "class", scope: !1, file: !1, line: 728, size: 448, elements: !5, identifier: "class") +!11 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "struct", scope: !1, file: !1, line: 309, baseType: !10, size: 128, offset: 64, elements: !5, vtableHolder: !10, templateParams: !5, identifier: "list") +!12 = distinct !DICompositeType(tag: DW_TAG_union_type, file: !1, line: 115, size: 384, elements: !5, identifier: "union")