Index: include/llvm/IR/DebugInfoMetadata.h =================================================================== --- include/llvm/IR/DebugInfoMetadata.h +++ include/llvm/IR/DebugInfoMetadata.h @@ -1118,8 +1118,11 @@ } TempDILocation cloneImpl() const { - return getTemporary(getContext(), getLine(), getColumn(), getScope(), - getInlinedAt()); + // Get the raw scope/inlinedAt since it is possible to invoke this on + // a DILocation containing temporary metadata. Specifically, this happens + // when we are linking metadata as a postpass after function importing. + return getTemporary(getContext(), getLine(), getColumn(), getRawScope(), + getRawInlinedAt()); } // Disallow replacing operands. Index: include/llvm/IR/GVMaterializer.h =================================================================== --- include/llvm/IR/GVMaterializer.h +++ include/llvm/IR/GVMaterializer.h @@ -18,6 +18,7 @@ #ifndef LLVM_IR_GVMATERIALIZER_H #define LLVM_IR_GVMATERIALIZER_H +#include "llvm/ADT/DenseMap.h" #include #include @@ -26,6 +27,7 @@ class GlobalValue; class Module; class StructType; +class TrackingMDRef; class GVMaterializer { protected: @@ -56,6 +58,14 @@ virtual std::error_code materializeMetadata() = 0; virtual void setStripDebugInfo() = 0; + /// Client should define this interface if the mapping between metadata + /// values and value ids needs to be preserved, e.g. across materializer + /// instantiations. If OnlyTempMD is true, only those that have remained + /// temporary metadata are recorded in the map. + virtual void + saveMDValueList(DenseMap &MDValueToValIDMap, + bool OnlyTempMD) {} + virtual std::vector getIdentifiedStructTypes() const = 0; }; Index: include/llvm/IR/Metadata.h =================================================================== --- include/llvm/IR/Metadata.h +++ include/llvm/IR/Metadata.h @@ -830,10 +830,11 @@ /// \brief Resolve cycles. /// /// Once all forward declarations have been resolved, force cycles to be - /// resolved. + /// resolved. If \a MDMaterialized is true, then any temporary metadata + /// is ignored, otherwise it asserts when encountering temporary metadata. /// /// \pre No operands (or operands' operands, etc.) have \a isTemporary(). - void resolveCycles(); + void resolveCycles(bool MDMaterialized = true); /// \brief Replace a temporary node with a permanent one. /// Index: include/llvm/IR/TrackingMDRef.h =================================================================== --- include/llvm/IR/TrackingMDRef.h +++ include/llvm/IR/TrackingMDRef.h @@ -14,6 +14,7 @@ #ifndef LLVM_IR_TRACKINGMDREF_H #define LLVM_IR_TRACKINGMDREF_H +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/IR/MetadataTracking.h" #include "llvm/Support/Casting.h" @@ -81,17 +82,24 @@ bool operator!=(const TrackingMDRef &X) const { return MD != X.MD; } private: + /// Enabler for DenseMapInfo of TrackingMDRefs, ensure the MD is non-null + /// and not a special DenseMapInfo key. + bool isValid() { + return MD && MD != DenseMapInfo::getEmptyKey() && + MD != DenseMapInfo::getTombstoneKey(); + } + void track() { - if (MD) + if (isValid()) MetadataTracking::track(MD); } void untrack() { - if (MD) + if (isValid()) MetadataTracking::untrack(MD); } void retrack(TrackingMDRef &X) { assert(MD == X.MD && "Expected values to match"); - if (X.MD) { + if (X.isValid()) { MetadataTracking::retrack(X.MD, MD); X.MD = nullptr; } @@ -165,6 +173,22 @@ } }; +/// Provide DenseMapInfo for TrackingMDRefs. +template <> struct DenseMapInfo { + static inline TrackingMDRef getEmptyKey() { + return TrackingMDRef(DenseMapInfo::getEmptyKey()); + } + static inline TrackingMDRef getTombstoneKey() { + return TrackingMDRef(DenseMapInfo::getTombstoneKey()); + } + static unsigned getHashValue(const TrackingMDRef Val) { + return DenseMapInfo::getHashValue(Val.get()); + } + static bool isEqual(const TrackingMDRef LHS, const TrackingMDRef RHS) { + return LHS == RHS; + } +}; + } // end namespace llvm #endif Index: include/llvm/IRReader/IRReader.h =================================================================== --- include/llvm/IRReader/IRReader.h +++ include/llvm/IRReader/IRReader.h @@ -27,10 +27,11 @@ /// If the given file holds a bitcode image, return a Module /// for it which does lazy deserialization of function bodies. Otherwise, /// attempt to parse it as LLVM Assembly and return a fully populated -/// Module. -std::unique_ptr getLazyIRFileModule(StringRef Filename, - SMDiagnostic &Err, - LLVMContext &Context); +/// Module. The ShouldLazyLoadMetadata flag is passed down to the bitcode +/// reader to optionally enable lazy metadata loading. +std::unique_ptr +getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, + bool ShouldLazyLoadMetadata = false); /// If the given MemoryBuffer holds a bitcode image, return a Module /// for it. Otherwise, attempt to parse it as LLVM Assembly and return Index: include/llvm/Linker/Linker.h =================================================================== --- include/llvm/Linker/Linker.h +++ include/llvm/Linker/Linker.h @@ -65,7 +65,11 @@ None = 0, OverrideFromSrc = (1 << 0), LinkOnlyNeeded = (1 << 1), - InternalizeLinkedSymbols = (1 << 2) + InternalizeLinkedSymbols = (1 << 2), + /// Flag indicating whether this is simply a metadata linking postpass + /// when the functions and other global values were parsed and linked + /// by an earlier module link (e.g. during function importing). + MetadataLinkingPostpass = (1 << 3) }; Linker(Module *M, DiagnosticHandlerFunction DiagnosticHandler); @@ -79,11 +83,14 @@ /// shadow those in the Dest. /// For ThinLTO function importing/exporting the \p FunctionInfoIndex /// is passed. If a \p FuncToImport is provided, only that single - /// function is imported from the source module. + /// function is imported from the source module. The ValIDToTempMDMap + /// is populated by the linker when function importing is performed, and + /// is passed as input along with the MetadataLinkingPostpass flag. /// Returns true on error. bool linkInModule(Module *Src, unsigned Flags = Flags::None, FunctionInfoIndex *Index = nullptr, - Function *FuncToImport = nullptr); + Function *FuncToImport = nullptr, + DenseMap *ValIDToTempMDMap = nullptr); /// \brief Set the composite to the passed-in module. void setModule(Module *Dst); Index: include/llvm/Transforms/Utils/ValueMapper.h =================================================================== --- include/llvm/Transforms/Utils/ValueMapper.h +++ include/llvm/Transforms/Utils/ValueMapper.h @@ -50,6 +50,12 @@ /// want to generate a mapped Value on demand. For example, if linking /// lazily. virtual Value *materializeValueFor(Value *V) = 0; + + /// If the client needs to handle temporary metadata it must implement + /// these methods. + virtual Metadata *mapTemporaryMetadata(Metadata *MD) { return nullptr; } + virtual void replaceTemporaryMetadata(const Metadata *OrigMD, + Metadata *NewMD) {} }; /// RemapFlags - These are flags that the value mapping APIs allow. @@ -73,6 +79,11 @@ /// Any global values not in value map are mapped to null instead of /// mapping to self. Illegal if RF_IgnoreMissingEntries is also set. RF_NullMapMissingGlobalValues = 8, + + /// Set when there is still temporary metadata that must be handled, + /// such as when we are doing function importing and will materialize + /// and link metadata as a postpass. + RF_HaveUnmaterializedMetadata = 16, }; static inline RemapFlags operator|(RemapFlags LHS, RemapFlags RHS) { Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -106,6 +106,7 @@ class BitcodeReaderMDValueList { unsigned NumFwdRefs; bool AnyFwdRefs; + bool SavedFwdRefs; unsigned MinFwdRef; unsigned MaxFwdRef; std::vector MDValuePtrs; @@ -113,7 +114,12 @@ LLVMContext &Context; public: BitcodeReaderMDValueList(LLVMContext &C) - : NumFwdRefs(0), AnyFwdRefs(false), Context(C) {} + : NumFwdRefs(0), AnyFwdRefs(false), SavedFwdRefs(false), Context(C) {} + ~BitcodeReaderMDValueList() { + // Assert that we either replaced all forward references, or saved + // them for later replacement. + assert(!NumFwdRefs || SavedFwdRefs); + } // vector compatibility methods unsigned size() const { return MDValuePtrs.size(); } @@ -124,6 +130,8 @@ void pop_back() { MDValuePtrs.pop_back(); } bool empty() const { return MDValuePtrs.empty(); } + void savedFwdRefs() { SavedFwdRefs = true; } + Metadata *operator[](unsigned i) const { assert(i < MDValuePtrs.size()); return MDValuePtrs[i]; @@ -286,6 +294,14 @@ void setStripDebugInfo() override; + /// Save the mapping between the metadata values and the corresponding + /// value id that were recorded in the MDValueList during parsing. If + /// OnlyTempMD is true, then only record those entries that are still + /// temporary metadata. This interface is used when metadata linking is + /// performed as a postpass, such as during function importing. + void saveMDValueList(DenseMap &MDValueToValIDMap, + bool OnlyTempMD) override; + private: /// Parse the "IDENTIFICATION_BLOCK_ID" block, populate the // ProducerIdentification data member, and do some basic enforcement on the @@ -3095,6 +3111,31 @@ void BitcodeReader::setStripDebugInfo() { StripDebugInfo = true; } +void BitcodeReader::saveMDValueList( + DenseMap &MDValueToValIDMap, bool OnlyTempMD) { + for (unsigned i = 0; i < MDValueList.size(); ++i) { + Metadata *MD = MDValueList[i]; + auto *N = dyn_cast_or_null(MD); + // Save all values if !OnlyTempMD, otherwise just the temporary metadata. + if (!OnlyTempMD || (N && N->isTemporary())) { + TrackingMDRef MDRef(MD); + // Will call this after materializing each function, in order to + // handle remapping of the function's instructions/metadata. + // See if we already have an entry in that case. + if (OnlyTempMD && MDValueToValIDMap.count(MDRef)) { + assert(MDValueToValIDMap[MDRef] == i && + "Inconsistent metadata value id"); + return; + } + MDValueToValIDMap[MDRef] = i; + // Flag that we saved the forward refs (temporary metadata) for error + // checking during MDValueList destruction. + if (OnlyTempMD) + MDValueList.savedFwdRefs(); + } + } +} + /// When we see the block for a function body, remember where it is and then /// skip it. This lets us lazily deserialize the functions. std::error_code BitcodeReader::rememberAndSkipFunctionBody() { Index: lib/IR/Metadata.cpp =================================================================== --- lib/IR/Metadata.cpp +++ lib/IR/Metadata.cpp @@ -517,7 +517,7 @@ resolve(); } -void MDNode::resolveCycles() { +void MDNode::resolveCycles(bool MDMaterialized) { if (isResolved()) return; @@ -530,6 +530,8 @@ if (!N) continue; + if (N->isTemporary() && !MDMaterialized) + continue; assert(!N->isTemporary() && "Expected all forward declarations to be resolved"); if (!N->isResolved()) Index: lib/IRReader/IRReader.cpp =================================================================== --- lib/IRReader/IRReader.cpp +++ lib/IRReader/IRReader.cpp @@ -31,11 +31,11 @@ static std::unique_ptr getLazyIRModule(std::unique_ptr Buffer, SMDiagnostic &Err, - LLVMContext &Context) { + LLVMContext &Context, bool ShouldLazyLoadMetadata) { if (isBitcode((const unsigned char *)Buffer->getBufferStart(), (const unsigned char *)Buffer->getBufferEnd())) { - ErrorOr> ModuleOrErr = - getLazyBitcodeModule(std::move(Buffer), Context); + ErrorOr> ModuleOrErr = getLazyBitcodeModule( + std::move(Buffer), Context, nullptr, ShouldLazyLoadMetadata); if (std::error_code EC = ModuleOrErr.getError()) { Err = SMDiagnostic(Buffer->getBufferIdentifier(), SourceMgr::DK_Error, EC.message()); @@ -49,7 +49,8 @@ std::unique_ptr llvm::getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, - LLVMContext &Context) { + LLVMContext &Context, + bool ShouldLazyLoadMetadata) { ErrorOr> FileOrErr = MemoryBuffer::getFileOrSTDIN(Filename); if (std::error_code EC = FileOrErr.getError()) { @@ -58,7 +59,8 @@ return nullptr; } - return getLazyIRModule(std::move(FileOrErr.get()), Err, Context); + return getLazyIRModule(std::move(FileOrErr.get()), Err, Context, + ShouldLazyLoadMetadata); } std::unique_ptr llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, Index: lib/Linker/LinkModules.cpp =================================================================== --- lib/Linker/LinkModules.cpp +++ lib/Linker/LinkModules.cpp @@ -23,6 +23,7 @@ #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/GVMaterializer.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/TypeFinder.h" @@ -379,6 +380,17 @@ LazilyLinkGlobalValues(LazilyLinkGlobalValues), ModLinker(ModLinker) {} Value *materializeValueFor(Value *V) override; + + /// Invoke ModuleLinker callback to save the mapping between the + /// given temporary metadata and its metadata value id. Used to support + /// metadata linking as a postpass for function importing. + Metadata *mapTemporaryMetadata(Metadata *MD) override; + + /// Invoke ModuleLinker callback to replace any temporary metadata saved + /// for the source metadata's id with the new non-temporary metadata. + /// Used when metadata linking as a postpass for function importing. + void replaceTemporaryMetadata(const Metadata *OrigMD, + Metadata *NewMD) override; }; class LinkDiagnosticInfo : public DiagnosticInfo { @@ -426,6 +438,9 @@ /// For symbol clashes, prefer those from Src. unsigned Flags; + /// Flags to pass to value mapper invocations. + RemapFlags ValueMapperFlags; + /// Function index passed into ModuleLinker for using in function /// importing/exporting handling. FunctionInfoIndex *ImportIndex; @@ -445,16 +460,29 @@ /// references. bool DoneLinkingBodies; + /// Association between metadata values created during bitcode parsing and + /// the value id. Used to correlate temporary metadata created during + /// function importing with the final metadata parsed during the subsequent + /// metadata linking postpass. + DenseMap MDValueToValIDMap; + + /// Association between metadata value id and temporary metadata that + /// remains unmapped after function importing. Saved during function + /// importing and consumed during the metadata linking postpass. + DenseMap *ValIDToTempMDMap; + public: ModuleLinker(Module *dstM, Linker::IdentifiedStructTypeSet &Set, Module *srcM, DiagnosticHandlerFunction DiagnosticHandler, unsigned Flags, FunctionInfoIndex *Index = nullptr, - Function *FuncToImport = nullptr) + Function *FuncToImport = nullptr, + DenseMap *ValIDToTempMDMap = nullptr) : DstM(dstM), SrcM(srcM), TypeMap(Set), ValMaterializer(TypeMap, DstM, LazilyLinkGlobalValues, this), - DiagnosticHandler(DiagnosticHandler), Flags(Flags), ImportIndex(Index), + DiagnosticHandler(DiagnosticHandler), Flags(Flags), + ValueMapperFlags(RF_MoveDistinctMDs), ImportIndex(Index), ImportFunction(FuncToImport), HasExportedFunctions(false), - DoneLinkingBodies(false) { + DoneLinkingBodies(false), ValIDToTempMDMap(ValIDToTempMDMap) { assert((ImportIndex || !ImportFunction) && "Expect a FunctionInfoIndex when importing"); // If we have a FunctionInfoIndex but no function to import, @@ -463,6 +491,19 @@ // may be exported to another backend compilation. if (ImportIndex && !ImportFunction) HasExportedFunctions = ImportIndex->hasExportedFunctions(SrcM); + + assert(!(isMetadataLinkingPostpass() && (ImportIndex || ImportFunction)) && + "Illegal to pass a function index or function to import into " + "metadata linking postpass"); + assert((ValIDToTempMDMap || + (!ImportFunction && !isMetadataLinkingPostpass())) && + "Function importing and metadata postpass linking must both provide " + "a ValIDToTempMDMap"); + + // If appropriate, tell the value mapper that it can expect to see + // temporary metadata. + if (!shouldLinkMetadata()) + ValueMapperFlags = ValueMapperFlags | RF_HaveUnmaterializedMetadata; } bool run(); @@ -472,6 +513,15 @@ bool shouldInternalizeLinkedSymbols() { return Flags & Linker::InternalizeLinkedSymbols; } + bool isMetadataLinkingPostpass() { + return Flags & Linker::MetadataLinkingPostpass; + } + + bool shouldLinkMetadata() { + // This will be non-null when we are importing or otherwise want to link + // metadata lazily (LTO), and when linking the metadata. + return ValIDToTempMDMap == nullptr || isMetadataLinkingPostpass(); + } /// Handles cloning of a global values from the source module into /// the destination module, including setting the attributes and visibility. @@ -484,6 +534,16 @@ /// Check if all global value body linking is complete. bool doneLinkingBodies() { return DoneLinkingBodies; } + /// Save the mapping between the given temporary metadata and its metadata + /// value id. Used to support metadata linking as a postpass for function + /// importing. + Metadata *mapTemporaryMetadata(Metadata *MD); + + /// Replace any temporary metadata saved for the source metadata's id with + /// the new non-temporary metadata. Used when metadata linking as a postpass + /// for function importing. + void replaceTemporaryMetadata(const Metadata *OrigMD, Metadata *NewMD); + private: bool shouldLinkFromSource(bool &LinkFromSrc, const GlobalValue &Dest, const GlobalValue &Src); @@ -561,8 +621,8 @@ /// Helper methods to check if we are importing from or potentially /// exporting from the current source module. - bool isPerformingImport() { return ImportFunction != nullptr; } - bool isModuleExporting() { return HasExportedFunctions; } + bool isPerformingImport() const { return ImportFunction != nullptr; } + bool isModuleExporting() const { return HasExportedFunctions; } /// If we are importing from the source module, checks if we should /// import SGV as a definition, otherwise import as a declaration. @@ -922,6 +982,57 @@ return DGV; } +Metadata *ValueMaterializerTy::mapTemporaryMetadata(Metadata *MD) { + return ModLinker->mapTemporaryMetadata(MD); +} + +void ValueMaterializerTy::replaceTemporaryMetadata(const Metadata *OrigMD, + Metadata *NewMD) { + ModLinker->replaceTemporaryMetadata(OrigMD, NewMD); +} + +Metadata *ModuleLinker::mapTemporaryMetadata(Metadata *MD) { + MDNode *Node = cast(MD); + assert(Node->isTemporary()); + TrackingMDRef MDRef(MD); + // If this temporary metadata has a value id recorded during function + // parsing, record that in the ValIDToTempMDMap if one was provided. + if (ValIDToTempMDMap && MDValueToValIDMap.count(MDRef)) { + unsigned Idx = MDValueToValIDMap[MDRef]; + // Check if we created a temp MD when importing a different function from + // this module. If so, reuse it the same temporary metadata, otherwise + // add this temporary metadata to the map. + if (!ValIDToTempMDMap->count(Idx)) + (*ValIDToTempMDMap)[Idx] = Node; + return (*ValIDToTempMDMap)[Idx]; + } + return nullptr; +} + +void ModuleLinker::replaceTemporaryMetadata(const Metadata *OrigMD, + Metadata *NewMD) { +#ifndef NDEBUG + auto *N = dyn_cast_or_null(NewMD); + assert(!N || !N->isTemporary()); +#endif + TrackingMDRef MDRef(const_cast(OrigMD)); + // If a mapping between metadata value ids and temporary metadata + // created during function importing was provided, and the source + // metadata has a value id recorded during metadata parsing, replace + // the temporary metadata with the final mapped metadata now. + if (ValIDToTempMDMap && MDValueToValIDMap.count(MDRef)) { + unsigned Idx = MDValueToValIDMap[MDRef]; + // Nothing to do if we didn't need to create a temporary metadata during + // function importing. + if (!ValIDToTempMDMap->count(Idx)) + return; + MDNode *TempMD = (*ValIDToTempMDMap)[Idx]; + TempMD->replaceAllUsesWith(NewMD); + MDNode::deleteTemporary(TempMD); + ValIDToTempMDMap->erase(Idx); + } +} + bool ModuleLinker::getComdatLeader(Module *M, StringRef ComdatName, const GlobalVariable *&GVar) { const GlobalValue *GVal = M->getNamedValue(ComdatName); @@ -1035,6 +1146,13 @@ bool ModuleLinker::shouldLinkFromSource(bool &LinkFromSrc, const GlobalValue &Dest, const GlobalValue &Src) { + // Already imported all the values. Just map to the Dest value + // in case it is referenced in the metadata. + if (isMetadataLinkingPostpass()) { + LinkFromSrc = false; + return false; + } + // Should we unconditionally use the Src? if (shouldOverrideFromSrc()) { LinkFromSrc = true; @@ -1499,7 +1617,7 @@ continue; } DstElements.push_back( - MapValue(V, ValueMap, RF_MoveDistinctMDs, &TypeMap, &ValMaterializer)); + MapValue(V, ValueMap, ValueMapperFlags, &TypeMap, &ValMaterializer)); } if (IsNewStructor) { NewType = ArrayType::get(NewType->getElementType(), DstElements.size()); @@ -1513,8 +1631,8 @@ /// referenced are in Dest. void ModuleLinker::linkGlobalInit(GlobalVariable &Dst, GlobalVariable &Src) { // Figure out what the initializer looks like in the dest module. - Dst.setInitializer(MapValue(Src.getInitializer(), ValueMap, - RF_MoveDistinctMDs, &TypeMap, &ValMaterializer)); + Dst.setInitializer(MapValue(Src.getInitializer(), ValueMap, ValueMapperFlags, + &TypeMap, &ValMaterializer)); } /// Copy the source function over into the dest function and fix up references @@ -1527,21 +1645,27 @@ if (std::error_code EC = Src.materialize()) return emitError(EC.message()); + if (!shouldLinkMetadata()) + // This is only supported for lazy links. Do after materialization of + // a function and before remapping metadata on instructions below + // in RemapInstruction, as the saved mapping is used to handle + // the temporary metadata hanging off instructions. + SrcM->getMaterializer()->saveMDValueList(MDValueToValIDMap, true); + // Link in the prefix data. if (Src.hasPrefixData()) - Dst.setPrefixData(MapValue(Src.getPrefixData(), ValueMap, - RF_MoveDistinctMDs, &TypeMap, &ValMaterializer)); + Dst.setPrefixData(MapValue(Src.getPrefixData(), ValueMap, ValueMapperFlags, + &TypeMap, &ValMaterializer)); // Link in the prologue data. if (Src.hasPrologueData()) Dst.setPrologueData(MapValue(Src.getPrologueData(), ValueMap, - RF_MoveDistinctMDs, &TypeMap, - &ValMaterializer)); + ValueMapperFlags, &TypeMap, &ValMaterializer)); // Link in the personality function. if (Src.hasPersonalityFn()) Dst.setPersonalityFn(MapValue(Src.getPersonalityFn(), ValueMap, - RF_MoveDistinctMDs, &TypeMap, + ValueMapperFlags, &TypeMap, &ValMaterializer)); // Go through and convert function arguments over, remembering the mapping. @@ -1558,7 +1682,7 @@ SmallVector, 8> MDs; Src.getAllMetadata(MDs); for (const auto &I : MDs) - Dst.setMetadata(I.first, MapMetadata(I.second, ValueMap, RF_MoveDistinctMDs, + Dst.setMetadata(I.first, MapMetadata(I.second, ValueMap, ValueMapperFlags, &TypeMap, &ValMaterializer)); // Splice the body of the source function into the dest function. @@ -1570,9 +1694,8 @@ // functions and patch them up to point to the local versions. for (BasicBlock &BB : Dst) for (Instruction &I : BB) - RemapInstruction(&I, ValueMap, - RF_IgnoreMissingEntries | RF_MoveDistinctMDs, &TypeMap, - &ValMaterializer); + RemapInstruction(&I, ValueMap, RF_IgnoreMissingEntries | ValueMapperFlags, + &TypeMap, &ValMaterializer); // There is no need to map the arguments anymore. for (Argument &Arg : Src.args()) @@ -1584,8 +1707,8 @@ void ModuleLinker::linkAliasBody(GlobalAlias &Dst, GlobalAlias &Src) { Constant *Aliasee = Src.getAliasee(); - Constant *Val = MapValue(Aliasee, ValueMap, RF_MoveDistinctMDs, &TypeMap, - &ValMaterializer); + Constant *Val = + MapValue(Aliasee, ValueMap, ValueMapperFlags, &TypeMap, &ValMaterializer); Dst.setAliasee(Val); } @@ -1629,7 +1752,7 @@ // Add Src elements into Dest node. for (const MDNode *op : NMD.operands()) DestNMD->addOperand(MapMetadata( - op, ValueMap, RF_MoveDistinctMDs | RF_NullMapMissingGlobalValues, + op, ValueMap, ValueMapperFlags | RF_NullMapMissingGlobalValues, &TypeMap, &ValMaterializer)); } } @@ -1893,6 +2016,34 @@ if (linkGlobalValueProto(&GA)) return true; + if (isMetadataLinkingPostpass()) { + // Ensure metadata materialized + if (SrcM->getMaterializer()->materializeMetadata()) + return true; + SrcM->getMaterializer()->saveMDValueList(MDValueToValIDMap, false); + + linkNamedMDNodes(); + + // Handle anything left in the ValIDToTempMDMap, such as metadata nodes + // not reached by the dbg.cu NamedMD (i.e. only reached from instructions). + // Walk the MDValueToValIDMap once to find the set of new (imported) MD + // that still has corresponding temporary metadata, and invoke metadata + // mapping on each one. + for (auto MDI : MDValueToValIDMap) { + if (!ValIDToTempMDMap->count(MDI.second)) + continue; + MapMetadata(MDI.first, ValueMap, ValueMapperFlags, &TypeMap, + &ValMaterializer); + } + assert(ValIDToTempMDMap->empty()); + + // Merge the module flags into the DstM module. + if (linkModuleFlagsMetadata()) + return true; + + return false; + } + for (const AppendingVarInfo &AppendingVar : AppendingVars) linkAppendingVarInit(AppendingVar); @@ -1902,7 +2053,7 @@ continue; const GlobalValue *GV = SrcM->getNamedValue(C.getName()); if (GV) - MapValue(GV, ValueMap, RF_MoveDistinctMDs, &TypeMap, &ValMaterializer); + MapValue(GV, ValueMap, ValueMapperFlags, &TypeMap, &ValMaterializer); } // Link in the function bodies that are defined in the source module into @@ -1960,11 +2111,13 @@ // Remap all of the named MDNodes in Src into the DstM module. We do this // after linking GlobalValues so that MDNodes that reference GlobalValues // are properly remapped. - linkNamedMDNodes(); + if (shouldLinkMetadata()) { + linkNamedMDNodes(); - // Merge the module flags into the DstM module. - if (linkModuleFlagsMetadata()) - return true; + // Merge the module flags into the DstM module. + if (linkModuleFlagsMetadata()) + return true; + } return false; } @@ -2089,11 +2242,14 @@ } bool Linker::linkInModule(Module *Src, unsigned Flags, FunctionInfoIndex *Index, - Function *FuncToImport) { + Function *FuncToImport, + DenseMap *ValIDToTempMDMap) { ModuleLinker TheLinker(Composite, IdentifiedStructTypes, Src, - DiagnosticHandler, Flags, Index, FuncToImport); + DiagnosticHandler, Flags, Index, FuncToImport, + ValIDToTempMDMap); bool RetCode = TheLinker.run(); - Composite->dropTriviallyDeadConstantArrays(); + if (!(Flags & Linker::MetadataLinkingPostpass)) + Composite->dropTriviallyDeadConstantArrays(); return RetCode; } Index: lib/Transforms/Utils/ValueMapper.cpp =================================================================== --- lib/Transforms/Utils/ValueMapper.cpp +++ lib/Transforms/Utils/ValueMapper.cpp @@ -159,13 +159,21 @@ } static Metadata *mapToMetadata(ValueToValueMapTy &VM, const Metadata *Key, - Metadata *Val) { + Metadata *Val, ValueMaterializer *Materializer, + RemapFlags Flags) { VM.MD()[Key].reset(Val); + if (Materializer && !(Flags & RF_HaveUnmaterializedMetadata)) { + auto *N = dyn_cast_or_null(Val); + // Need to invoke this once we have non-temporary MD. + if (!N || !N->isTemporary()) + Materializer->replaceTemporaryMetadata(Key, Val); + } return Val; } -static Metadata *mapToSelf(ValueToValueMapTy &VM, const Metadata *MD) { - return mapToMetadata(VM, MD, const_cast(MD)); +static Metadata *mapToSelf(ValueToValueMapTy &VM, const Metadata *MD, + ValueMaterializer *Materializer, RemapFlags Flags) { + return mapToMetadata(VM, MD, const_cast(MD), Materializer, Flags); } static Metadata *MapMetadataImpl(const Metadata *MD, @@ -198,10 +206,13 @@ } /// Resolve uniquing cycles involving the given metadata. -static void resolveCycles(Metadata *MD) { - if (auto *N = dyn_cast_or_null(MD)) +static void resolveCycles(Metadata *MD, bool MDMaterialized) { + if (auto *N = dyn_cast_or_null(MD)) { + if (!MDMaterialized && N->isTemporary()) + return; if (!N->isResolved()) - N->resolveCycles(); + N->resolveCycles(MDMaterialized); + } } /// Remap the operands of an MDNode. @@ -230,7 +241,7 @@ // Resolve uniquing cycles underneath distinct nodes on the fly so they // don't infect later operands. if (IsDistinct) - resolveCycles(New); + resolveCycles(New, !(Flags & RF_HaveUnmaterializedMetadata)); } } @@ -258,7 +269,7 @@ // Remap operands later. DistinctWorklist.push_back(NewMD); - return mapToMetadata(VM, Node, NewMD); + return mapToMetadata(VM, Node, NewMD, Materializer, Flags); } /// \brief Map a uniqued MDNode. @@ -269,22 +280,29 @@ ValueToValueMapTy &VM, RemapFlags Flags, ValueMapTypeRemapper *TypeMapper, ValueMaterializer *Materializer) { - assert(Node->isUniqued() && "Expected uniqued node"); + assert(((Flags & RF_HaveUnmaterializedMetadata) || Node->isUniqued()) && + "Expected uniqued node"); // Create a temporary node and map it upfront in case we have a uniquing // cycle. If necessary, this mapping will get updated by RAUW logic before // returning. auto ClonedMD = Node->clone(); - mapToMetadata(VM, Node, ClonedMD.get()); + mapToMetadata(VM, Node, ClonedMD.get(), Materializer, Flags); if (!remapOperands(*ClonedMD, DistinctWorklist, VM, Flags, TypeMapper, Materializer)) { // No operands changed, so use the original. ClonedMD->replaceAllUsesWith(const_cast(Node)); - return const_cast(Node); + // Even though replaceAllUsesWith would have replaced the value map + // entry, we need to explictly map with the final non-temporary node + // to replace any temporary metadata via the callback. + return mapToSelf(VM, Node, Materializer, Flags); } - // Uniquify the cloned node. - return MDNode::replaceWithUniqued(std::move(ClonedMD)); + // Uniquify the cloned node. Explicitly map it with the final non-temporary + // node so that replacement of temporary metadata via the callback occurs. + return mapToMetadata(VM, Node, + MDNode::replaceWithUniqued(std::move(ClonedMD)), + Materializer, Flags); } static Metadata *MapMetadataImpl(const Metadata *MD, @@ -297,18 +315,18 @@ return NewMD; if (isa(MD)) - return mapToSelf(VM, MD); + return mapToSelf(VM, MD, Materializer, Flags); if (isa(MD)) if ((Flags & RF_NoModuleLevelChanges)) - return mapToSelf(VM, MD); + return mapToSelf(VM, MD, Materializer, Flags); if (const auto *VMD = dyn_cast(MD)) { Value *MappedV = MapValue(VMD->getValue(), VM, Flags, TypeMapper, Materializer); if (VMD->getValue() == MappedV || (!MappedV && (Flags & RF_IgnoreMissingEntries))) - return mapToSelf(VM, MD); + return mapToSelf(VM, MD, Materializer, Flags); // FIXME: This assert crashes during bootstrap, but I think it should be // correct. For now, just match behaviour from before the metadata/value @@ -317,7 +335,8 @@ // assert((MappedV || (Flags & RF_NullMapMissingGlobalValues)) && // "Referenced metadata not in value map!"); if (MappedV) - return mapToMetadata(VM, MD, ValueAsMetadata::get(MappedV)); + return mapToMetadata(VM, MD, ValueAsMetadata::get(MappedV), Materializer, + Flags); return nullptr; } @@ -328,10 +347,25 @@ // If this is a module-level metadata and we know that nothing at the // module level is changing, then use an identity mapping. if (Flags & RF_NoModuleLevelChanges) - return mapToSelf(VM, MD); + return mapToSelf(VM, MD, Materializer, Flags); // Require resolved nodes whenever metadata might be remapped. - assert(Node->isResolved() && "Unexpected unresolved node"); + assert(((Flags & RF_HaveUnmaterializedMetadata) || Node->isResolved()) && + "Unexpected unresolved node"); + + if (Materializer && Node->isTemporary()) { + assert(Flags & RF_HaveUnmaterializedMetadata); + Metadata *TempMD = + Materializer->mapTemporaryMetadata(const_cast(MD)); + // If the above callback returned an existing temporary node, use it + // instead of the current temporary node. This happens when earlier + // function importing passes already created and saved a temporary + // metadata node for the same value id. + if (TempMD) { + mapToMetadata(VM, MD, TempMD, Materializer, Flags); + return TempMD; + } + } if (Node->isDistinct()) return mapDistinctNode(Node, DistinctWorklist, VM, Flags, TypeMapper, @@ -355,7 +389,7 @@ return NewMD; // Resolve cycles involving the entry metadata. - resolveCycles(NewMD); + resolveCycles(NewMD, !(Flags & RF_HaveUnmaterializedMetadata)); // Remap the operands of distinct MDNodes. while (!DistinctWorklist.empty()) Index: test/Linker/Inputs/thinlto_funcimport_debug.ll =================================================================== --- /dev/null +++ test/Linker/Inputs/thinlto_funcimport_debug.ll @@ -0,0 +1,38 @@ +; ModuleID = 'dbg_main.o' +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define i32 @main() #0 !dbg !4 { +entry: + %call = tail call i32 @func1(i32 10) #2, !dbg !11 + %call1 = tail call i32 @func2(i32 10) #2, !dbg !12 + ret i32 0, !dbg !13 +} + +declare i32 @func1(i32) #1 + +declare i32 @func2(i32) #1 + +attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #2 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!8, !9} +!llvm.ident = !{!10} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 251407) (llvm/trunk 251401)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, subprograms: !3) +!1 = !DIFile(filename: "dbg_main.c", directory: "/usr/local/google/home/tejohnson/llvm/tmp") +!2 = !{} +!3 = !{!4} +!4 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, variables: !2) +!5 = !DISubroutineType(types: !6) +!6 = !{!7} +!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) +!8 = !{i32 2, !"Dwarf Version", i32 4} +!9 = !{i32 2, !"Debug Info Version", i32 3} +!10 = !{!"clang version 3.8.0 (trunk 251407) (llvm/trunk 251401)"} +!11 = !DILocation(line: 4, column: 3, scope: !4) +!12 = !DILocation(line: 5, column: 3, scope: !4) +!13 = !DILocation(line: 6, column: 1, scope: !4) Index: test/Linker/thinlto_funcimport_debug.ll =================================================================== --- /dev/null +++ test/Linker/thinlto_funcimport_debug.ll @@ -0,0 +1,71 @@ +; Do setup work for all below tests: generate bitcode and combined index +; RUN: llvm-as -function-summary %s -o %t.bc +; RUN: llvm-as -function-summary %p/Inputs/thinlto_funcimport_debug.ll -o %t2.bc +; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc + +; If we import func1 and not func2 we should only link DISubprogram for func1 +; RUN: llvm-link %t2.bc -functionindex=%t3.thinlto.bc -import=func1:%t.bc -S | FileCheck %s + +; CHECK: declare i32 @func2 +; CHECK: define available_externally i32 @func1 +; CHECK: distinct !DISubprogram(name: "func1" +; CHECK: distinct !DISubprogram(name: "func2" + +; ModuleID = 'dbg.o' +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind readnone uwtable +define i32 @func1(i32 %n) #0 !dbg !4 { +entry: + tail call void @llvm.dbg.value(metadata i32 %n, i64 0, metadata !9, metadata !17), !dbg !18 + tail call void @llvm.dbg.value(metadata i32 5, i64 0, metadata !10, metadata !17), !dbg !19 + %cmp = icmp sgt i32 %n, 10, !dbg !20 + %. = select i1 %cmp, i32 10, i32 5, !dbg !22 + tail call void @llvm.dbg.value(metadata i32 %., i64 0, metadata !10, metadata !17), !dbg !19 + ret i32 %., !dbg !23 +} + +; Function Attrs: nounwind readnone uwtable +define i32 @func2(i32 %n) #0 !dbg !11 { +entry: + tail call void @llvm.dbg.value(metadata i32 %n, i64 0, metadata !13, metadata !17), !dbg !24 + ret i32 %n, !dbg !25 +} + +; Function Attrs: nounwind readnone +declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1 + +attributes #0 = { nounwind readnone uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind readnone } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!14, !15} +!llvm.ident = !{!16} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 251407) (llvm/trunk 251401)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, subprograms: !3) +!1 = !DIFile(filename: "dbg.c", directory: "/usr/local/google/home/tejohnson/llvm/tmp") +!2 = !{} +!3 = !{!4, !11} +!4 = distinct !DISubprogram(name: "func1", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, variables: !8) +!5 = !DISubroutineType(types: !6) +!6 = !{!7, !7} +!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) +!8 = !{!9, !10} +!9 = !DILocalVariable(name: "n", arg: 1, scope: !4, file: !1, line: 1, type: !7) +!10 = !DILocalVariable(name: "x", scope: !4, file: !1, line: 2, type: !7) +!11 = distinct !DISubprogram(name: "func2", scope: !1, file: !1, line: 8, type: !5, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, variables: !12) +!12 = !{!13} +!13 = !DILocalVariable(name: "n", arg: 1, scope: !11, file: !1, line: 8, type: !7) +!14 = !{i32 2, !"Dwarf Version", i32 4} +!15 = !{i32 2, !"Debug Info Version", i32 3} +!16 = !{!"clang version 3.8.0 (trunk 251407) (llvm/trunk 251401)"} +!17 = !DIExpression() +!18 = !DILocation(line: 1, column: 15, scope: !4) +!19 = !DILocation(line: 2, column: 7, scope: !4) +!20 = !DILocation(line: 3, column: 9, scope: !21) +!21 = distinct !DILexicalBlock(scope: !4, file: !1, line: 3, column: 7) +!22 = !DILocation(line: 3, column: 7, scope: !4) +!23 = !DILocation(line: 5, column: 3, scope: !4) +!24 = !DILocation(line: 8, column: 15, scope: !11) +!25 = !DILocation(line: 9, column: 3, scope: !11) Index: tools/llvm-link/llvm-link.cpp =================================================================== --- tools/llvm-link/llvm-link.cpp +++ tools/llvm-link/llvm-link.cpp @@ -102,16 +102,21 @@ // Read the specified bitcode file in and return it. This routine searches the // link path for the specified file to try to find it... // -static std::unique_ptr -loadFile(const char *argv0, const std::string &FN, LLVMContext &Context) { +static std::unique_ptr loadFile(const char *argv0, + const std::string &FN, + LLVMContext &Context, + bool MaterializeMetadata = true) { SMDiagnostic Err; if (Verbose) errs() << "Loading '" << FN << "'\n"; - std::unique_ptr Result = getLazyIRFileModule(FN, Err, Context); + std::unique_ptr Result = + getLazyIRFileModule(FN, Err, Context, !MaterializeMetadata); if (!Result) Err.print(argv0, errs()); - Result->materializeMetadata(); - UpgradeDebugInfo(*Result); + if (MaterializeMetadata) { + Result->materializeMetadata(); + UpgradeDebugInfo(*Result); + } return Result; } @@ -161,6 +166,7 @@ /// Import any functions requested via the -import option. static bool importFunctions(const char *argv0, LLVMContext &Context, Linker &L) { + StringMap *> ModuleToTempMDValsMap; for (const auto &Import : Imports) { // Identify the requested function and its bitcode source file. size_t Idx = Import.find(':'); @@ -172,7 +178,7 @@ std::string FileName = Import.substr(Idx + 1, std::string::npos); // Load the specified source module. - std::unique_ptr M = loadFile(argv0, FileName, Context); + std::unique_ptr M = loadFile(argv0, FileName, Context, false); if (!M.get()) { errs() << argv0 << ": error loading file '" << FileName << "'\n"; return false; @@ -214,8 +220,39 @@ Index = std::move(IndexOrErr.get()); } + // Save the mapping of value ids to temporary metadata created when + // importing this function. If we have already imported from this module, + // add new temporary metadata to the existing mapping. + DenseMap *TempMDVals = ModuleToTempMDValsMap[FileName]; + if (!TempMDVals) + TempMDVals = ModuleToTempMDValsMap[FileName] = + new DenseMap(); + // Link in the specified function. - if (L.linkInModule(M.get(), Linker::Flags::None, Index.get(), F)) + if (L.linkInModule(M.get(), Linker::Flags::None, Index.get(), F, + TempMDVals)) + return false; + } + + // Now link in metadata for all modules from which we imported functions. + for (StringMapEntry *> &SME : + ModuleToTempMDValsMap) { + // Load the specified source module. + std::unique_ptr M = loadFile(argv0, SME.getKey(), Context, true); + if (!M.get()) { + errs() << argv0 << ": error loading file '" << SME.getKey() << "'\n"; + return false; + } + + if (verifyModule(*M, &errs())) { + errs() << argv0 << ": " << SME.getKey() + << ": error: input module is broken!\n"; + return false; + } + + // Link in all necessary metadata from this module. + if (L.linkInModule(M.get(), Linker::Flags::MetadataLinkingPostpass, nullptr, + nullptr, SME.getValue())) return false; } return true;