Index: include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- include/llvm/IR/ModuleSummaryIndex.h +++ include/llvm/IR/ModuleSummaryIndex.h @@ -163,13 +163,13 @@ /// Struct that holds a reference to a particular GUID in a global value /// summary. struct ValueInfo { - PointerIntPair - RefAndFlag; + PointerIntPair + RefAndFlags; ValueInfo() = default; ValueInfo(bool HaveGVs, const GlobalValueSummaryMapTy::value_type *R) { - RefAndFlag.setPointer(R); - RefAndFlag.setInt(HaveGVs); + RefAndFlags.setPointer(R); + RefAndFlags.setInt(HaveGVs); } operator bool() const { return getRef(); } @@ -189,10 +189,12 @@ : getRef()->second.U.Name; } - bool haveGVs() const { return RefAndFlag.getInt(); } + bool haveGVs() const { return RefAndFlags.getInt() & 0x1; } + bool isReadOnly() const { return RefAndFlags.getInt() & 0x2; } + void setReadOnly() { RefAndFlags.setInt(RefAndFlags.getInt() | 0x2); } const GlobalValueSummaryMapTy::value_type *getRef() const { - return RefAndFlag.getPointer(); + return RefAndFlags.getPointer(); } bool isDSOLocal() const; @@ -543,6 +545,8 @@ std::move(TypeTestAssumeConstVCalls), std::move(TypeCheckedLoadConstVCalls)}); } + // Gets the number of immutable refs in RefEdgeList + unsigned immutableRefCount() const; /// Check if this is a function summary. static bool classof(const GlobalValueSummary *GVS) { @@ -652,19 +656,30 @@ /// Global variable summary information to aid decisions and /// implementation of importing. /// -/// Currently this doesn't add anything to the base \p GlobalValueSummary, -/// but is a placeholder as additional info may be added to the summary -/// for variables. +/// Global variable summary has extra flag, telling if it is +/// modified during the program run or not. This affects ThinLTO +/// internalization class GlobalVarSummary : public GlobalValueSummary { - public: - GlobalVarSummary(GVFlags Flags, std::vector Refs) - : GlobalValueSummary(GlobalVarKind, Flags, std::move(Refs)) {} + struct GVarFlags { + GVarFlags(bool ReadOnly = false) : ReadOnly(ReadOnly) {} + + unsigned ReadOnly : 1; + } VarFlags; + + GlobalVarSummary(GVFlags Flags, GVarFlags VarFlags, + std::vector Refs) + : GlobalValueSummary(GlobalVarKind, Flags, std::move(Refs)), + VarFlags(VarFlags) {} /// Check if this is a global variable summary. static bool classof(const GlobalValueSummary *GVS) { return GVS->getSummaryKind() == GlobalVarKind; } + + GVarFlags varflags() const { return VarFlags; } + void setReadOnly(bool RO) { VarFlags.ReadOnly = RO; } + bool isReadOnly() const { return VarFlags.ReadOnly; } }; struct TypeTestResolution { @@ -1135,6 +1150,9 @@ /// Print out strongly connected components for debugging. void dumpSCCs(raw_ostream &OS); + + /// Analyze index and detect unmodified globals + void propagateConstants(const DenseSet &PreservedSymbols); }; /// GraphTraits definition to build SCC for the index @@ -1184,6 +1202,14 @@ } }; +static inline bool canImportGlobalVar(GlobalValueSummary *S) { + assert(isa(S->getBaseObject())); + + // We don't import GV with references, because it can result + // in promotion of local variables in the source module. + return !GlobalValue::isInterposableLinkage(S->linkage()) && + !S->notEligibleToImport() && S->refs().empty(); +} } // end namespace llvm #endif // LLVM_IR_MODULESUMMARYINDEX_H Index: include/llvm/Transforms/IPO/FunctionImport.h =================================================================== --- include/llvm/Transforms/IPO/FunctionImport.h +++ include/llvm/Transforms/IPO/FunctionImport.h @@ -176,6 +176,14 @@ const DenseSet &GUIDPreservedSymbols, function_ref isPrevailing); +/// Compute dead symbols and run constant propagation in combined index +/// after that. +void computeDeadSymbolsWithConstProp( + ModuleSummaryIndex &Index, + const DenseSet &GUIDPreservedSymbols, + function_ref isPrevailing, + bool ImportEnabled); + /// Converts value \p GV to declaration, or replaces with a declaration if /// it is an alias. Returns true if converted, false if replaced. bool convertToDeclaration(GlobalValue &GV); Index: include/llvm/Transforms/Utils/FunctionImportUtils.h =================================================================== --- include/llvm/Transforms/Utils/FunctionImportUtils.h +++ include/llvm/Transforms/Utils/FunctionImportUtils.h @@ -113,7 +113,6 @@ bool renameModuleForThinLTO( Module &M, const ModuleSummaryIndex &Index, SetVector *GlobalsToImport = nullptr); - } // End llvm namespace #endif Index: lib/Analysis/ModuleSummaryAnalysis.cpp =================================================================== --- lib/Analysis/ModuleSummaryAnalysis.cpp +++ lib/Analysis/ModuleSummaryAnalysis.cpp @@ -220,10 +220,19 @@ } } -static void computeFunctionSummary( - ModuleSummaryIndex &Index, const Module &M, const Function &F, - BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT, - bool HasLocalsInUsedOrAsm, DenseSet &CantBePromoted) { +static bool isNonVolatileLoad(const Instruction *I) { + if (const auto *LI = dyn_cast(I)) + return !LI->isVolatile(); + + return false; +} + +static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, + const Function &F, BlockFrequencyInfo *BFI, + ProfileSummaryInfo *PSI, DominatorTree &DT, + bool HasLocalsInUsedOrAsm, + DenseSet &CantBePromoted, + bool IsThinLTO) { // Summary not currently supported for anonymous functions, they should // have been named. assert(F.hasName()); @@ -244,6 +253,7 @@ // Add personality function, prefix data and prologue data to function's ref // list. findRefEdges(Index, &F, RefEdges, Visited); + std::vector NonVolatileLoads; bool HasInlineAsmMaybeReferencingInternal = false; for (const BasicBlock &BB : F) @@ -251,6 +261,13 @@ if (isa(I)) continue; ++NumInsts; + if (isNonVolatileLoad(&I)) { + // Postpone processing of non-volatile load instructions + // See comments below + Visited.insert(&I); + NonVolatileLoads.push_back(&I); + continue; + } findRefEdges(Index, &I, RefEdges, Visited); auto CS = ImmutableCallSite(&I); if (!CS) @@ -340,6 +357,24 @@ } } + // By now we processed all instructions in a function, except + // non-volatile loads. All new refs we add in a loop below + // are obviously constant. All constant refs are grouped in the + // end of RefEdges vector, so we can use a single integer value + // to identify them. + unsigned RefCnt = RefEdges.size(); + for (const Instruction *I : NonVolatileLoads) { + Visited.erase(I); + findRefEdges(Index, I, RefEdges, Visited); + } + std::vector Refs = RefEdges.takeVector(); + // Regular LTO module doesn't participate in ThinLTO import, + // so no reference from it can be readonly, since this would + // require importing variable as local copy + if (IsThinLTO) + for (; RefCnt < Refs.size(); ++RefCnt) + Refs[RefCnt].setReadOnly(); + // Explicit add hot edges to enforce importing for designated GUIDs for // sample PGO, to enable the same inlines as the profiled optimized binary. for (auto &I : F.getImportGUIDs()) @@ -363,9 +398,9 @@ // Don't try to import functions with noinline attribute. F.getAttributes().hasFnAttribute(Attribute::NoInline)}; auto FuncSummary = llvm::make_unique( - Flags, NumInsts, FunFlags, RefEdges.takeVector(), - CallGraphEdges.takeVector(), TypeTests.takeVector(), - TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(), + Flags, NumInsts, FunFlags, std::move(Refs), CallGraphEdges.takeVector(), + TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(), + TypeCheckedLoadVCalls.takeVector(), TypeTestAssumeConstVCalls.takeVector(), TypeCheckedLoadConstVCalls.takeVector()); if (NonRenamableLocal) @@ -382,8 +417,13 @@ bool NonRenamableLocal = isNonRenamableLocal(V); GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal, /* Live = */ false, V.isDSOLocal()); - auto GVarSummary = - llvm::make_unique(Flags, RefEdges.takeVector()); + + // Don't mark variables we won't be able to internalize as read-only. + GlobalVarSummary::GVarFlags VarFlags( + !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() && + !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass()); + auto GVarSummary = llvm::make_unique(Flags, VarFlags, + RefEdges.takeVector()); if (NonRenamableLocal) CantBePromoted.insert(V.getGUID()); if (HasBlockAddress) @@ -487,13 +527,19 @@ Index.addGlobalValueSummary(*GV, std::move(Summary)); } else { std::unique_ptr Summary = - llvm::make_unique(GVFlags, - ArrayRef{}); + llvm::make_unique( + GVFlags, GlobalVarSummary::GVarFlags(), + ArrayRef{}); Index.addGlobalValueSummary(*GV, std::move(Summary)); } }); } + bool IsThinLTO = true; + if (auto *MD = + mdconst::extract_or_null(M.getModuleFlag("ThinLTO"))) + IsThinLTO = MD->getZExtValue(); + // Compute summaries for all functions defined in module, and save in the // index. for (auto &F : M) { @@ -514,7 +560,7 @@ computeFunctionSummary(Index, M, F, BFI, PSI, DT, !LocalsUsed.empty() || HasLocalInlineAsmSymbol, - CantBePromoted); + CantBePromoted, IsThinLTO); } // Compute summaries for all variables defined in module, and save in the @@ -545,11 +591,6 @@ setLiveRoot(Index, "llvm.global_dtors"); setLiveRoot(Index, "llvm.global.annotations"); - bool IsThinLTO = true; - if (auto *MD = - mdconst::extract_or_null(M.getModuleFlag("ThinLTO"))) - IsThinLTO = MD->getZExtValue(); - for (auto &GlobalList : Index) { // Ignore entries for references that are undefined in the current module. if (GlobalList.second.SummaryList.empty()) Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -7642,7 +7642,8 @@ if (ParseToken(lltok::rparen, "expected ')' here")) return true; - auto GS = llvm::make_unique(GVFlags, std::move(Refs)); + auto GS = llvm::make_unique( + GVFlags, GlobalVarSummary::GVarFlags(), std::move(Refs)); GS->setModulePath(ModulePath); Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -898,6 +898,11 @@ return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, Live, Local); } +// Decode the flags for GlobalVariable in the summary +static GlobalVarSummary::GVarFlags getDecodedGVarFlags(uint64_t RawFlags) { + return GlobalVarSummary::GVarFlags((RawFlags & 0x1) ? true : false); +} + static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { switch (Val) { default: // Map unknown visibilities to default. @@ -5170,6 +5175,12 @@ parseWholeProgramDevirtResolution(Record, Strtab, Slot, TypeId); } +static void setImmutableRefs(std::vector &Refs, unsigned Count) { + // Read-only refs are in the end of the refs list. + for (unsigned RefNo = Refs.size() - Count; RefNo < Refs.size(); ++RefNo) + Refs[RefNo].setReadOnly(); +} + // Eagerly parse the entire summary block. This populates the GlobalValueSummary // objects in the index. Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { @@ -5187,9 +5198,9 @@ } const uint64_t Version = Record[0]; const bool IsOldProfileFormat = Version == 1; - if (Version < 1 || Version > 4) + if (Version < 1 || Version > 5) return error("Invalid summary version " + Twine(Version) + - ", 1, 2, 3 or 4 expected"); + ", 1, 2, 3, 4 or 5 expected"); Record.clear(); // Keep around the last seen summary to be used when we see an optional @@ -5268,11 +5279,16 @@ unsigned InstCount = Record[2]; uint64_t RawFunFlags = 0; unsigned NumRefs = Record[3]; + unsigned NumImmutableRefs = 0; int RefListStartIndex = 4; if (Version >= 4) { RawFunFlags = Record[3]; NumRefs = Record[4]; RefListStartIndex = 5; + if (Version >= 5) { + NumImmutableRefs = Record[5]; + RefListStartIndex = 6; + } } auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); @@ -5291,6 +5307,7 @@ std::vector Calls = makeCallList( ArrayRef(Record).slice(CallGraphEdgeStartIndex), IsOldProfileFormat, HasProfile, HasRelBF); + setImmutableRefs(Refs, NumImmutableRefs); auto FS = llvm::make_unique( Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), std::move(Calls), std::move(PendingTypeTests), @@ -5339,14 +5356,21 @@ TheIndex.addGlobalValueSummary(GUID.first, std::move(AS)); break; } - // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, n x valueid] + // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags, n x valueid] case bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS: { unsigned ValueID = Record[0]; uint64_t RawFlags = Record[1]; + unsigned RefArrayStart = 2; + GlobalVarSummary::GVarFlags GVF; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + if (Version >= 5) { + GVF = getDecodedGVarFlags(Record[2]); + RefArrayStart = 3; + } std::vector Refs = - makeRefList(ArrayRef(Record).slice(2)); - auto FS = llvm::make_unique(Flags, std::move(Refs)); + makeRefList(ArrayRef(Record).slice(RefArrayStart)); + auto FS = + llvm::make_unique(Flags, GVF, std::move(Refs)); FS->setModulePath(getThisModule()->first()); auto GUID = getValueInfoFromValueId(ValueID); FS->setOriginalName(GUID.second); @@ -5365,12 +5389,17 @@ unsigned InstCount = Record[3]; uint64_t RawFunFlags = 0; unsigned NumRefs = Record[4]; + unsigned NumImmutableRefs = 0; int RefListStartIndex = 5; if (Version >= 4) { RawFunFlags = Record[4]; NumRefs = Record[5]; RefListStartIndex = 6; + if (Version >= 5) { + NumImmutableRefs = Record[6]; + RefListStartIndex = 7; + } } auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); @@ -5384,6 +5413,7 @@ ArrayRef(Record).slice(CallGraphEdgeStartIndex), IsOldProfileFormat, HasProfile, false); ValueInfo VI = getValueInfoFromValueId(ValueID).first; + setImmutableRefs(Refs, NumImmutableRefs); auto FS = llvm::make_unique( Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), std::move(Edges), std::move(PendingTypeTests), @@ -5432,10 +5462,17 @@ unsigned ValueID = Record[0]; uint64_t ModuleId = Record[1]; uint64_t RawFlags = Record[2]; + unsigned RefArrayStart = 3; + GlobalVarSummary::GVarFlags GVF; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + if (Version >= 5) { + GVF = getDecodedGVarFlags(Record[3]); + RefArrayStart = 4; + } std::vector Refs = - makeRefList(ArrayRef(Record).slice(3)); - auto FS = llvm::make_unique(Flags, std::move(Refs)); + makeRefList(ArrayRef(Record).slice(RefArrayStart)); + auto FS = + llvm::make_unique(Flags, GVF, std::move(Refs)); LastSeenSummary = FS.get(); FS->setModulePath(ModuleIdMap[ModuleId]); ValueInfo VI = getValueInfoFromValueId(ValueID).first; Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -991,6 +991,11 @@ return RawFlags; } +static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) { + uint64_t RawFlags = Flags.ReadOnly; + return RawFlags; +} + static unsigned getEncodedVisibility(const GlobalValue &GV) { switch (GV.getVisibility()) { case GlobalValue::DefaultVisibility: return 0; @@ -3489,6 +3494,7 @@ NameVals.push_back(FS->instCount()); NameVals.push_back(getEncodedFFlags(FS->fflags())); NameVals.push_back(FS->refs().size()); + NameVals.push_back(FS->immutableRefCount()); for (auto &RI : FS->refs()) NameVals.push_back(VE.getValueID(RI.getValue())); @@ -3530,6 +3536,7 @@ NameVals.push_back(VE.getValueID(&V)); GlobalVarSummary *VS = cast(Summary); NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); + NameVals.push_back(getEncodedGVarFlags(VS->varflags())); unsigned SizeBeforeRefs = NameVals.size(); for (auto &RI : VS->refs()) @@ -3546,7 +3553,7 @@ // Current version for the summary. // This is bumped whenever we introduce changes in the way some record are // interpreted, like flags for instance. -static const uint64_t INDEX_VERSION = 4; +static const uint64_t INDEX_VERSION = 5; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. @@ -3581,6 +3588,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -3597,6 +3605,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt // numrefs x valueid, n x (valueid [, rel_block_freq]) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -3691,6 +3700,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -3705,6 +3715,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -3777,6 +3788,7 @@ NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(VS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); + NameVals.push_back(getEncodedGVarFlags(VS->varflags())); for (auto &RI : VS->refs()) { auto RefValueId = getValueId(RI.getGUID()); if (!RefValueId) @@ -3802,17 +3814,21 @@ NameVals.push_back(FS->instCount()); NameVals.push_back(getEncodedFFlags(FS->fflags())); // Fill in below - NameVals.push_back(0); + NameVals.push_back(0); // numrefs + NameVals.push_back(0); // immutablerefcnt - unsigned Count = 0; + unsigned Count = 0, ImmutableRefCnt = 0; for (auto &RI : FS->refs()) { auto RefValueId = getValueId(RI.getGUID()); if (!RefValueId) continue; NameVals.push_back(*RefValueId); + if (RI.isReadOnly()) + ImmutableRefCnt++; Count++; } NameVals[5] = Count; + NameVals[6] = ImmutableRefCnt; bool HasProfileData = false; for (auto &EI : FS->calls()) { Index: lib/IR/ModuleSummaryIndex.cpp =================================================================== --- lib/IR/ModuleSummaryIndex.cpp +++ lib/IR/ModuleSummaryIndex.cpp @@ -30,6 +30,17 @@ }); } +// Gets the number of immutable refs in RefEdgeList +unsigned FunctionSummary::immutableRefCount() const { + // Here we take advantage of having all readonly references + // located in the end of the RefEdgeList. + auto Refs = refs(); + unsigned ImmutableRefCnt = 0; + for (int I = Refs.size() - 1; I >= 0 && Refs[I].isReadOnly(); --I) + ImmutableRefCnt++; + return ImmutableRefCnt; +} + // Collect for the given module the list of function it defines // (GUID -> Summary). void ModuleSummaryIndex::collectDefinedFunctionsForModule( @@ -84,6 +95,73 @@ return false; } +static void propagateConstantsToRefs(GlobalValueSummary *S) { + // If reference is not readonly then referenced summary is not + // readonly either. Note that: + // - All references from GlobalVarSummary are conservatively considered as + // not readonly. Tracking them properly requires more complex analysis + // then we have now. + // + // - AliasSummary objects have no refs at all so this function is a no-op + // for them. + for (auto &VI : S->refs()) { + if (VI.isReadOnly()) { + // We only mark refs as readonly when computing function summaries on + // analysis phase. + assert(isa(S)); + continue; + } + for (auto &Ref : VI.getSummaryList()) + // If references to alias is not readonly then aliasee is not readonly + if (auto *GVS = dyn_cast(Ref->getBaseObject())) + GVS->setReadOnly(false); + } +} + +// Do the constant propagation in combined index. +// The goal of constant propagation is internalization of readonly +// variables. To determine which variables are readonly and which +// are not we take following steps: +// - During analysis we speculatively assign readonly attribute to +// all variables which can be internalized. When computing function +// summary we also assign readonly attribute to a reference if +// function doesn't modify referenced variable. +// +// - After computing dead symbols in combined index we do the constant +// propagation. During this step we clear readonly attribute from +// all variables which: +// a. are dead, preserved or can't be imported +// b. referenced by any global variable initializer +// c. referenced by a function and reference is not readonly +// +// Internalization itself happens in the backend after import is finished +// See internalizeImmutableGVs. +void ModuleSummaryIndex::propagateConstants( + const DenseSet &GUIDPreservedSymbols) { + for (auto &P : *this) + for (auto &S : P.second.SummaryList) { + if (!isGlobalValueLive(S.get())) + // We don't examine references from dead objects + continue; + + // Global variable can't be marked read only if it is not eligible + // to import since we need to ensure that all external references + // get a local (imported) copy. It also can't be marked read only + // if it or any alias (since alias points to the same memory) are + // preserved or notEligibleToImport, since either of those means + // there could be writes that are not visible (because preserved + // means it could have external to DSO writes, and notEligibleToImport + // means it could have writes via inline assembly leading it to be + // in the @llvm.*used). + if (auto *GVS = dyn_cast(S->getBaseObject())) + // Here we intentionally pass S.get() not GVS, because S could be + // an alias. + if (!canImportGlobalVar(S.get()) || GUIDPreservedSymbols.count(P.first)) + GVS->setReadOnly(false); + propagateConstantsToRefs(S.get()); + } +} + // TODO: write a graphviz dumper for SCCs (see ModuleSummaryIndex::exportToDot) // then delete this function and update its tests LLVM_DUMP_METHOD @@ -108,6 +186,7 @@ struct Attributes { void add(const Twine &Name, const Twine &Value, const Twine &Comment = Twine()); + void addComment(const Twine &Comment); std::string getAsString() const; std::vector Attrs; @@ -129,6 +208,10 @@ A += Value.str(); A += "\""; Attrs.push_back(A); + addComment(Comment); +} + +void Attributes::addComment(const Twine &Comment) { if (!Comment.isTriviallyEmpty()) { if (Comments.empty()) Comments = " // "; @@ -237,6 +320,12 @@ OS << "\"]; // defined externally\n"; } +static bool hasReadOnlyFlag(const GlobalValueSummary *S) { + if (auto *GVS = dyn_cast(S)) + return GVS->isReadOnly(); + return false; +} + void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const { std::vector CrossModuleEdges; DenseMap> NodeMap; @@ -252,13 +341,17 @@ }; auto DrawEdge = [&](const char *Pfx, uint64_t SrcMod, GlobalValue::GUID SrcId, - uint64_t DstMod, GlobalValue::GUID DstId, int TypeOrHotness) { - // 0 corresponds to alias edge, 1 to ref edge, 2 to call with unknown - // hotness, ... - TypeOrHotness += 2; + uint64_t DstMod, GlobalValue::GUID DstId, + int TypeOrHotness) { + // 0 - alias + // 1 - reference + // 2 - constant reference + // Other value: (hotness - 3). + TypeOrHotness += 3; static const char *EdgeAttrs[] = { " [style=dotted]; // alias", " [style=dashed]; // ref", + " [style=dashed,color=forestgreen]; // const-ref", " // call (hotness : Unknown)", " [color=blue]; // call (hotness : Cold)", " // call (hotness : None)", @@ -301,6 +394,8 @@ A.add("shape", "box"); } else { A.add("shape", "Mrecord", "variable"); + if (Flags.Live && hasReadOnlyFlag(SummaryIt.second)) + A.addComment("immutable"); } auto VI = getValueInfo(SummaryIt.first); @@ -318,7 +413,7 @@ for (auto &SummaryIt : GVSMap) { auto *GVS = SummaryIt.second; for (auto &R : GVS->refs()) - Draw(SummaryIt.first, R.getGUID(), -1); + Draw(SummaryIt.first, R.getGUID(), R.isReadOnly() ? -1 : -2); if (auto *AS = dyn_cast_or_null(SummaryIt.second)) { GlobalValue::GUID AliaseeId; @@ -331,7 +426,7 @@ AliaseeId = AliaseeOrigId; } - Draw(SummaryIt.first, AliaseeId, -2); + Draw(SummaryIt.first, AliaseeId, -3); continue; } Index: lib/LTO/LTO.cpp =================================================================== --- lib/LTO/LTO.cpp +++ lib/LTO/LTO.cpp @@ -187,6 +187,8 @@ AddUnsigned(VI.isDSOLocal()); AddUsedCfiGlobal(VI.getGUID()); } + if (auto *GVS = dyn_cast(GS)) + AddUnsigned(GVS->isReadOnly()); if (auto *FS = dyn_cast(GS)) { for (auto &TT : FS->type_tests()) UsedTypeIds.insert(TT); @@ -803,7 +805,8 @@ return PrevailingType::Unknown; return It->second; }; - computeDeadSymbols(ThinLTO.CombinedIndex, GUIDPreservedSymbols, isPrevailing); + computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols, + isPrevailing, Conf.OptLevel > 0); // Setup output file to emit statistics. std::unique_ptr StatsFile = nullptr; Index: lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- lib/LTO/ThinLTOCodeGenerator.cpp +++ lib/LTO/ThinLTOCodeGenerator.cpp @@ -646,7 +646,8 @@ auto isPrevailing = [&](GlobalValue::GUID G) { return PrevailingType::Unknown; }; - computeDeadSymbols(Index, GUIDPreservedSymbols, isPrevailing); + computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing, + /* ImportEnabled = */ true); } /** Index: lib/Linker/IRMover.cpp =================================================================== --- lib/Linker/IRMover.cpp +++ lib/Linker/IRMover.cpp @@ -1062,11 +1062,6 @@ ValueMap.MD()[CU->getRawEnumTypes()].reset(nullptr); ValueMap.MD()[CU->getRawMacros()].reset(nullptr); ValueMap.MD()[CU->getRawRetainedTypes()].reset(nullptr); - // We import global variables only temporarily in order for instcombine - // and globalopt to perform constant folding and static constructor - // evaluation. After that elim-avail-extern will covert imported globals - // back to declarations, so we don't need debug info for them. - ValueMap.MD()[CU->getRawGlobalVariables()].reset(nullptr); // Imported entities only need to be mapped in if they have local // scope, as those might correspond to an imported entity inside a Index: lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- lib/Transforms/IPO/FunctionImport.cpp +++ lib/Transforms/IPO/FunctionImport.cpp @@ -294,10 +294,8 @@ LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n"); for (auto &RefSummary : VI.getSummaryList()) - if (RefSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind && - !RefSummary->notEligibleToImport() && - !GlobalValue::isInterposableLinkage(RefSummary->linkage()) && - RefSummary->refs().empty()) { + if (isa(RefSummary.get()) && + canImportGlobalVar(RefSummary.get())) { auto ILI = ImportList[RefSummary->modulePath()].insert(VI.getGUID()); // Only update stat if we haven't already imported this variable. if (ILI.second) @@ -824,6 +822,25 @@ NumLiveSymbols += LiveSymbols; } +// Compute dead symbols and propagate constants in combined index. +void llvm::computeDeadSymbolsWithConstProp( + ModuleSummaryIndex &Index, + const DenseSet &GUIDPreservedSymbols, + function_ref isPrevailing, + bool ImportEnabled) { + computeDeadSymbols(Index, GUIDPreservedSymbols, isPrevailing); + if (ImportEnabled) { + Index.propagateConstants(GUIDPreservedSymbols); + } else { + // If import is disabled we should drop read-only attribute + // from all summaries to prevent internalization. + for (auto &P : Index) + for (auto &S : P.second.SummaryList) + if (auto *GVS = dyn_cast(S.get())) + GVS->setReadOnly(false); + } +} + /// Compute the set of summaries needed for a ThinLTO backend compilation of /// \p ModulePath. void llvm::gatherImportedSummariesForModule( @@ -1018,6 +1035,22 @@ return NewFn; } +// Internalize values that we marked with specific attribute +// in processGlobalForThinLTO. +static void internalizeImmutableGVs(Module &M) { + for (auto &GV : M.globals()) { + // Skip GVs which have been converted to declarations + // by dropDeadSymbols. + if (GV.isDeclaration()) + continue; + if (auto *GVar = dyn_cast(&GV)) + if (GVar->hasAttribute("thinlto-internalize")) { + GVar->setLinkage(GlobalValue::InternalLinkage); + GVar->setVisibility(GlobalValue::DefaultVisibility); + } + } +} + // Automatically import functions in Module \p DestModule based on the summaries // index. Expected FunctionImporter::importFunctions( @@ -1141,6 +1174,8 @@ NumImportedModules++; } + internalizeImmutableGVs(DestModule); + NumImportedFunctions += (ImportedCount - ImportedGVCount); NumImportedGlobalVars += ImportedGVCount; Index: lib/Transforms/Utils/FunctionImportUtils.cpp =================================================================== --- lib/Transforms/Utils/FunctionImportUtils.cpp +++ lib/Transforms/Utils/FunctionImportUtils.cpp @@ -204,8 +204,9 @@ // Check the summaries to see if the symbol gets resolved to a known local // definition. + ValueInfo VI; if (GV.hasName()) { - ValueInfo VI = ImportIndex.getValueInfo(GV.getGUID()); + VI = ImportIndex.getValueInfo(GV.getGUID()); if (VI && VI.isDSOLocal()) { GV.setDSOLocal(true); if (GV.hasDLLImportStorageClass()) @@ -213,6 +214,18 @@ } } + // Mark read-only variables which can be imported with specific attribute. + // We can't internalize them now because IRMover will fail to link variable + // definitions to their external declarations during ThinLTO import. We'll + // internalize read-only variables later, after import is finished. + // See internalizeImmutableGVs. + if (!GV.isDeclaration() && VI) { + const auto &SL = VI.getSummaryList(); + auto *GVS = SL.empty() ? nullptr : dyn_cast(SL[0].get()); + if (GVS && GVS->isReadOnly()) + cast(&GV)->addAttribute("thinlto-internalize"); + } + bool DoPromote = false; if (GV.hasLocalLinkage() && ((DoPromote = shouldPromoteLocalToGlobal(&GV)) || isPerformingImport())) { @@ -230,7 +243,7 @@ // Remove functions imported as available externally defs from comdats, // as this is a declaration for the linker, and will be dropped eventually. // It is illegal for comdats to contain declarations. - auto *GO = dyn_cast_or_null(&GV); + auto *GO = dyn_cast(&GV); if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) { // The IRMover should not have placed any imported declarations in // a comdat, so the only declaration that should be in a comdat Index: test/Bitcode/summary_version.ll =================================================================== --- test/Bitcode/summary_version.ll +++ test/Bitcode/summary_version.ll @@ -2,7 +2,7 @@ ; RUN: opt -module-summary %s -o - | llvm-bcanalyzer -dump | FileCheck %s ; CHECK: +; CHECK: Index: test/Bitcode/thinlto-alias.ll =================================================================== --- test/Bitcode/thinlto-alias.ll +++ test/Bitcode/thinlto-alias.ll @@ -20,7 +20,7 @@ ; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: ; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK-NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph-cast.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-cast.ll +++ test/Bitcode/thinlto-function-summary-callgraph-cast.ll @@ -6,9 +6,9 @@ ; CHECK: +; CHECK-NEXT: ; "another_caller" has only references but no calls. -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK-NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph-pgo.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-pgo.ll +++ test/Bitcode/thinlto-function-summary-callgraph-pgo.ll @@ -17,7 +17,7 @@ ; CHECK: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll +++ test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll @@ -48,7 +48,7 @@ ; CHECK-NEXT: ; op4=hot1 op6=cold op8=hot2 op10=hot4 op12=none1 op14=hot3 op16=none2 op18=none3 op20=123 -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED_NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph-relbf.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-relbf.ll +++ test/Bitcode/thinlto-function-summary-callgraph-relbf.ll @@ -13,7 +13,7 @@ ; CHECK: ; CHECK: ; op4=none1 op6=hot1 op8=cold1 op10=none2 op12=hot2 op14=cold2 op16=none3 op18=hot3 op20=cold3 op22=123 -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED_NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph.ll +++ test/Bitcode/thinlto-function-summary-callgraph.ll @@ -17,7 +17,7 @@ ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/Bitcode/thinlto-function-summary-refgraph.ll =================================================================== --- test/Bitcode/thinlto-function-summary-refgraph.ll +++ test/Bitcode/thinlto-function-summary-refgraph.ll @@ -41,27 +41,27 @@ ; CHECK: +; CHECK-DAG: ; Function W contains a call to func3 as well as a reference to globalvar: ; op0=W op4=globalvar op5=func3 -; CHECK-DAG: +; CHECK-DAG: ; Function X contains call to foo, as well as address reference to foo ; which is in the same instruction as the call: ; op0=X op4=foo op5=foo -; CHECK-DAG: +; CHECK-DAG: ; Function Y contains call to func2, and ensures we don't incorrectly add ; a reference to it when reached while earlier analyzing the phi using its ; return value: ; op0=Y op4=func2 -; CHECK-DAG: +; CHECK-DAG: ; Function Z contains call to func2, and ensures we don't incorrectly add ; a reference to it when reached while analyzing subsequent use of its return ; value: ; op0=Z op4=func2 -; CHECK-DAG: +; CHECK-DAG: ; Variable bar initialization contains address reference to func: ; op0=bar op2=func -; CHECK-DAG: +; CHECK-DAG: ; CHECK: ; CHECK: