Index: llvm/include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -295,6 +295,9 @@ // Range information for accessed offsets for every argument. // [n x (paramno, range, numcalls, numcalls x (callee_guid, paramno, range))] FS_PARAM_ACCESS = 25, + // IFUNC: [valueid, flags, valueid] + FS_IFUNC = 26, + }; enum MetadataCodes { Index: llvm/include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- llvm/include/llvm/IR/ModuleSummaryIndex.h +++ llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -281,7 +281,12 @@ class GlobalValueSummary { public: /// Sububclass discriminator (for dyn_cast<> et al.) - enum SummaryKind : unsigned { AliasKind, FunctionKind, GlobalVarKind }; + enum SummaryKind : unsigned { + AliasKind, + IfuncKind, + FunctionKind, + GlobalVarKind + }; /// Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield. struct GVFlags { @@ -357,8 +362,9 @@ protected: GlobalValueSummary(SummaryKind K, GVFlags Flags, std::vector Refs) : Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) { - assert((K != AliasKind || Refs.empty()) && - "Expect no references for AliasSummary"); + + assert(((K != AliasKind && K != IfuncKind) || (Refs.empty())) && + "Expect no references for Alias and Ifunc summary"); } public: @@ -416,74 +422,97 @@ /// Return the list of values referenced by this global value definition. ArrayRef refs() const { return RefEdgeList; } - /// If this is an alias summary, returns the summary of the aliased object (a - /// global variable or function), otherwise returns itself. + /// If this is an global indirect value summary, returns the summary of the + /// indirect object (a global variable or function), otherwise returns itself. GlobalValueSummary *getBaseObject(); const GlobalValueSummary *getBaseObject() const; friend class ModuleSummaryIndex; }; -/// Alias summary information. -class AliasSummary : public GlobalValueSummary { - ValueInfo AliaseeValueInfo; - - /// This is the Aliasee in the same module as alias (could get from VI, trades - /// memory for time). Note that this pointer may be null (and the value info - /// empty) when we have a distributed index where the alias is being imported - /// (as a copy of the aliasee), but the aliasee is not. - GlobalValueSummary *AliaseeSummary; +class GlobalIndirectValueSummary : public GlobalValueSummary { + ValueInfo IndirectValueInfo; + /// This is the Aliasee in the same module as alias or Resolver in the same + /// module as ifunc (could get from VI, trades memory for time). Note that + /// this pointer may be null (and the value info empty) when we have a + /// distributed index where the alias is being imported (as a copy of the + /// aliasee), but the aliasee is not. + GlobalValueSummary *IndirectSummary; public: - AliasSummary(GVFlags Flags) - : GlobalValueSummary(AliasKind, Flags, ArrayRef{}), - AliaseeSummary(nullptr) {} + GlobalIndirectValueSummary(enum SummaryKind kind, GVFlags Flags) + : GlobalValueSummary(kind, Flags, ArrayRef{}), + IndirectSummary(nullptr) {} - /// Check if this is an alias summary. static bool classof(const GlobalValueSummary *GVS) { - return GVS->getSummaryKind() == AliasKind; + auto kind = GVS->getSummaryKind(); + return kind == AliasKind || kind == IfuncKind; } - void setAliasee(ValueInfo &AliaseeVI, GlobalValueSummary *Aliasee) { - AliaseeValueInfo = AliaseeVI; - AliaseeSummary = Aliasee; + void setSummary(ValueInfo &ValueInfo, GlobalValueSummary *Summary) { + IndirectValueInfo = ValueInfo; + IndirectSummary = Summary; } - bool hasAliasee() const { - assert(!!AliaseeSummary == (AliaseeValueInfo && - !AliaseeValueInfo.getSummaryList().empty()) && - "Expect to have both aliasee summary and summary list or neither"); - return !!AliaseeSummary; + bool hasSummary() const { + assert(!!IndirectSummary == (IndirectValueInfo && + !IndirectValueInfo.getSummaryList().empty()) && + "Expect to have both value summary and summary list or neither"); + return !!IndirectSummary; } - const GlobalValueSummary &getAliasee() const { - assert(AliaseeSummary && "Unexpected missing aliasee summary"); - return *AliaseeSummary; + const GlobalValueSummary &getSummary() const { + assert(IndirectSummary && "Unexpected missing resolver summary"); + return *IndirectSummary; } - GlobalValueSummary &getAliasee() { + GlobalValueSummary &getSummary() { return const_cast( - static_cast(this)->getAliasee()); + static_cast(this)->getSummary()); } - ValueInfo getAliaseeVI() const { - assert(AliaseeValueInfo && "Unexpected missing aliasee"); - return AliaseeValueInfo; + + ValueInfo getSummaryVI() const { + assert(IndirectValueInfo && "Unexpected missing indirect value info"); + return IndirectValueInfo; + } + + GlobalValue::GUID getSummaryGUID() const { + assert(IndirectValueInfo && "Unexpected missing indirect value info"); + return IndirectValueInfo.getGUID(); } - GlobalValue::GUID getAliaseeGUID() const { - assert(AliaseeValueInfo && "Unexpected missing aliasee"); - return AliaseeValueInfo.getGUID(); +}; + +/// Alias summary information. +class AliasSummary : public GlobalIndirectValueSummary { +public: + AliasSummary(GVFlags Flags) : GlobalIndirectValueSummary(AliasKind, Flags) {} + + /// Check if this is an alias summary. + static bool classof(const GlobalValueSummary *GVS) { + return GVS->getSummaryKind() == AliasKind; + } +}; + +/// Ifunc summary information. +class IfuncSummary : public GlobalIndirectValueSummary { +public: + IfuncSummary(GVFlags Flags) : GlobalIndirectValueSummary(IfuncKind, Flags) {} + + /// Check if this is an ifunc summary. + static bool classof(const GlobalValueSummary *GVS) { + return GVS->getSummaryKind() == IfuncKind; } }; const inline GlobalValueSummary *GlobalValueSummary::getBaseObject() const { - if (auto *AS = dyn_cast(this)) - return &AS->getAliasee(); + if (auto *GIV = dyn_cast(this)) + return &GIV->getSummary(); return this; } inline GlobalValueSummary *GlobalValueSummary::getBaseObject() { - if (auto *AS = dyn_cast(this)) - return &AS->getAliasee(); + if (auto *GIV = dyn_cast(this)) + return &GIV->getSummary(); return this; } Index: llvm/lib/Analysis/ModuleSummaryAnalysis.cpp =================================================================== --- llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -330,6 +330,13 @@ assert(!CalledFunction && "Expected null called function in callsite for alias"); CalledFunction = dyn_cast(GA->getBaseObject()); } + + if (auto *GIF = dyn_cast(CalledValue)) { + assert(!CalledFunction && + "Expected null called function in callsite for ifunc"); + CalledFunction = dyn_cast(GIF->getBaseObject()); + } + // Check if this is a direct call to a known function or a known // intrinsic, or an indirect call with profile data. if (CalledFunction) { @@ -347,10 +354,10 @@ if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None) Hotness = CalleeInfo::HotnessType::Cold; - // Use the original CalledValue, in case it was an alias. We want - // to record the call edge to the alias in that case. Eventually - // an alias summary will be created to associate the alias and - // aliasee. + // Use the original CalledValue, in case it was an alias or ifunc. + // We want to record the call edge to the aliasee or resolver in that + // case. Eventually an alias summary will be created to associate the + // alias and aliasee or ifunc and its resolver. auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo( cast(CalledValue))]; ValueInfo.updateHotness(Hotness); @@ -631,12 +638,31 @@ assert(AliaseeVI && "Alias expects aliasee summary to be available"); assert(AliaseeVI.getSummaryList().size() == 1 && "Expected a single entry per aliasee in per-module index"); - AS->setAliasee(AliaseeVI, AliaseeVI.getSummaryList()[0].get()); + AS->setSummary(AliaseeVI, AliaseeVI.getSummaryList()[0].get()); if (NonRenamableLocal) CantBePromoted.insert(A.getGUID()); Index.addGlobalValueSummary(A, std::move(AS)); } +static void computeIfuncSummary(ModuleSummaryIndex &Index, const GlobalIFunc &I, + DenseSet &CantBePromoted) { + bool NonRenamableLocal = isNonRenamableLocal(I); + GlobalValueSummary::GVFlags Flags(I.getLinkage(), NonRenamableLocal, + /* Live = */ false, I.isDSOLocal(), + I.hasLinkOnceODRLinkage() && + I.hasGlobalUnnamedAddr()); + auto IF = std::make_unique(Flags); + auto *Resolver = I.getBaseObject(); + auto ResolverVI = Index.getValueInfo(Resolver->getGUID()); + assert(ResolverVI && "Ifunc expects ifunc summary to be available"); + assert(ResolverVI.getSummaryList().size() == 1 && + "Expected a single entry per ifunc in per-module index"); + IF->setSummary(ResolverVI, ResolverVI.getSummaryList()[0].get()); + if (NonRenamableLocal) + CantBePromoted.insert(I.getGUID()); + Index.addGlobalValueSummary(I, std::move(IF)); +} + // Set LiveRoot flag on entries matching the given value name. static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) { if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name))) @@ -779,6 +805,9 @@ for (const GlobalAlias &A : M.aliases()) computeAliasSummary(Index, A, CantBePromoted); + for (const GlobalIFunc &I : M.ifuncs()) + computeIfuncSummary(Index, I, CantBePromoted); + for (auto *V : LocalsUsed) { auto *Summary = Index.getGlobalValueSummary(*V); assert(Summary && "Missing summary for global value"); Index: llvm/lib/Analysis/StackSafetyAnalysis.cpp =================================================================== --- llvm/lib/Analysis/StackSafetyAnalysis.cpp +++ llvm/lib/Analysis/StackSafetyAnalysis.cpp @@ -618,7 +618,7 @@ if (!GVS->isLive()) continue; if (const AliasSummary *AS = dyn_cast(GVS.get())) - if (!AS->hasAliasee()) + if (!AS->hasSummary()) continue; if (!isa(GVS->getBaseObject())) continue; @@ -654,7 +654,7 @@ if (FunctionSummary *FS = dyn_cast(S)) return FS; AliasSummary *AS = dyn_cast(S); - if (!AS || !AS->hasAliasee()) + if (!AS || !AS->hasSummary()) return nullptr; S = AS->getBaseObject(); if (S == AS) Index: llvm/lib/AsmParser/LLParser.cpp =================================================================== --- llvm/lib/AsmParser/LLParser.cpp +++ llvm/lib/AsmParser/LLParser.cpp @@ -8161,10 +8161,10 @@ auto FwdRefAliasees = ForwardRefAliasees.find(ID); if (FwdRefAliasees != ForwardRefAliasees.end()) { for (auto AliaseeRef : FwdRefAliasees->second) { - assert(!AliaseeRef.first->hasAliasee() && + assert(!AliaseeRef.first->hasSummary() && "Forward referencing alias already has aliasee"); assert(Summary && "Aliasee must be a definition"); - AliaseeRef.first->setAliasee(VI, Summary.get()); + AliaseeRef.first->setSummary(VI, Summary.get()); } ForwardRefAliasees.erase(FwdRefAliasees); } @@ -8468,7 +8468,7 @@ } else { auto Summary = Index->findSummaryInModule(AliaseeVI, ModulePath); assert(Summary && "Aliasee must be a definition"); - AS->setAliasee(AliaseeVI, Summary); + AS->setSummary(AliaseeVI, Summary); } AddGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, Index: llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp =================================================================== --- llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -139,6 +139,7 @@ STRINGIFY_CODE(MODULE_CODE, METADATA_VALUES_UNUSED) STRINGIFY_CODE(MODULE_CODE, SOURCE_FILENAME) STRINGIFY_CODE(MODULE_CODE, HASH) + STRINGIFY_CODE(MODULE_CODE, IFUNC) } case bitc::IDENTIFICATION_BLOCK_ID: switch (CodeID) { @@ -307,6 +308,7 @@ STRINGIFY_CODE(FS, TYPE_ID_METADATA) STRINGIFY_CODE(FS, BLOCK_COUNT) STRINGIFY_CODE(FS, PARAM_ACCESS) + STRINGIFY_CODE(FS, IFUNC) } case bitc::METADATA_ATTACHMENT_ID: switch (CodeID) { Index: llvm/lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -5801,6 +5801,7 @@ // v2: [strtab offset, strtab size, v1] case bitc::MODULE_CODE_GLOBALVAR: case bitc::MODULE_CODE_FUNCTION: + case bitc::MODULE_CODE_IFUNC: case bitc::MODULE_CODE_ALIAS: { StringRef Name; ArrayRef GVRecord; @@ -6145,13 +6146,41 @@ auto AliaseeInModule = TheIndex.findSummaryInModule(AliaseeVI, ModulePath); if (!AliaseeInModule) return error("Alias expects aliasee summary to be parsed"); - AS->setAliasee(AliaseeVI, AliaseeInModule); + AS->setSummary(AliaseeVI, AliaseeInModule); auto GUID = getValueInfoFromValueId(ValueID); AS->setOriginalName(GUID.second); TheIndex.addGlobalValueSummary(GUID.first, std::move(AS)); break; } + + case bitc::FS_IFUNC: { + unsigned ValueID = Record[0]; + uint64_t RawFlags = Record[1]; + unsigned ResolverID = Record[2]; + + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + auto IF = std::make_unique(Flags); + // The module path string ref set in the summary must be owned by the + // index's module string table. Since we don't have a module path + // string table section in the per-module index, we create a single + // module path string table entry with an empty (0) ID to take + // ownership. + IF->setModulePath(getThisModule()->first()); + + auto ResolverVI = getValueInfoFromValueId(ResolverID).first; + auto ResolverInModule = + TheIndex.findSummaryInModule(ResolverVI, ModulePath); + if (!ResolverInModule) + return error("Ifunc expects resolver summary to be parsed"); + IF->setSummary(ResolverVI, ResolverInModule); + + auto GUID = getValueInfoFromValueId(ValueID); + IF->setOriginalName(GUID.second); + TheIndex.addGlobalValueSummary(GUID.first, std::move(IF)); + break; + } + // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags, n x valueid] case bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS: { unsigned ValueID = Record[0]; @@ -6283,7 +6312,7 @@ auto AliaseeVI = getValueInfoFromValueId(AliaseeValueId).first; auto AliaseeInModule = TheIndex.findSummaryInModule(AliaseeVI, AS->modulePath()); - AS->setAliasee(AliaseeVI, AliaseeInModule); + AS->setSummary(AliaseeVI, AliaseeInModule); ValueInfo VI = getValueInfoFromValueId(ValueID).first; LastSeenGUID = VI.getGUID(); Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -451,11 +451,14 @@ for (auto &M : *ModuleToSummariesForIndex) for (auto &Summary : M.second) { Callback(Summary, false); - // Ensure aliasee is handled, e.g. for assigning a valueId, + // Ensure aliasee and ifunc resolver are handled, + // e.g. for assigning a valueId, // even if we are not importing the aliasee directly (the // imported alias will contain a copy of aliasee). if (auto *AS = dyn_cast(Summary.getSecond())) - Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true); + Callback({AS->getSummaryGUID(), &AS->getSummary()}, true); + if (auto *IF = dyn_cast(Summary.getSecond())) + Callback({IF->getSummaryGUID(), &IF->getSummary()}, true); } } else { for (auto &Summaries : Index) @@ -3916,6 +3919,14 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + // Abbrev for FS_IFUNC. + Abbv = std::make_shared(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_IFUNC)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + unsigned FSIfuncAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + // Abbrev for FS_TYPE_ID_METADATA Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA)); @@ -3969,6 +3980,21 @@ NameVals.clear(); } + for (const GlobalIFunc &I : M.ifuncs()) { + auto *Ifunc = I.getBaseObject(); + auto IfuncId = VE.getValueID(&I); + auto ResolverId = VE.getValueID(Ifunc); + + auto *Summary = Index->getGlobalValueSummary(I); + IfuncSummary *IF = cast(Summary); + + NameVals.push_back(IfuncId); + NameVals.push_back(getEncodedGVSummaryFlags(IF->flags())); + NameVals.push_back(ResolverId); + Stream.EmitRecord(bitc::FS_IFUNC, NameVals, FSIfuncAbbrev); + NameVals.clear(); + } + for (auto &S : Index->typeIdCompatibleVtableMap()) { writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, S.first, S.second, VE); @@ -4055,6 +4081,10 @@ // id of the aliasee. Save them in a vector for post-processing. SmallVector Aliases; + // The ifuncs are emitted as a post-pass, and will point to the value + // id of the resolver. Save them in a vector for post-processing. + SmallVector Ifuncs; + // Save the value id for each summary for alias emission. DenseMap SummaryToValueIdMap; @@ -4075,7 +4105,7 @@ }; std::set DefOrUseGUIDs; - forEachSummary([&](GVInfo I, bool IsAliasee) { + forEachSummary([&](GVInfo I, bool IsAliaseeOrIRes) { GlobalValueSummary *S = I.second; assert(S); DefOrUseGUIDs.insert(I.first); @@ -4086,10 +4116,11 @@ assert(ValueId); SummaryToValueIdMap[S] = *ValueId; - // If this is invoked for an aliasee, we want to record the above - // mapping, but then not emit a summary entry (if the aliasee is - // to be imported, we will invoke this separately with IsAliasee=false). - if (IsAliasee) + // If this is invoked for an aliasee or ifunc resolver, we want to record + // the above mapping, but then not emit a summary entry (if the aliasee is + // to be imported, we will invoke this separately with + // IsAliaseeOrIRes=false). + if (IsAliaseeOrIRes) return; if (auto *AS = dyn_cast(S)) { @@ -4099,6 +4130,13 @@ return; } + if (auto *IF = dyn_cast(S)) { + // Will process ifuncs as a post-pass because the reader wants all + // global to be loaded first. + Ifuncs.push_back(IF); + return; + } + if (auto *VS = dyn_cast(S)) { NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(VS->modulePath())); @@ -4216,7 +4254,7 @@ NameVals.push_back(AliasValueId); NameVals.push_back(Index.getModuleId(AS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(AS->flags())); - auto AliaseeValueId = SummaryToValueIdMap[&AS->getAliasee()]; + auto AliaseeValueId = SummaryToValueIdMap[&AS->getSummary()]; assert(AliaseeValueId); NameVals.push_back(AliaseeValueId); @@ -4225,7 +4263,7 @@ NameVals.clear(); MaybeEmitOriginalName(*AS); - if (auto *FS = dyn_cast(&AS->getAliasee())) + if (auto *FS = dyn_cast(&AS->getSummary())) getReferencedTypeIds(FS, ReferencedTypeIds); } Index: llvm/lib/IR/AsmWriter.cpp =================================================================== --- llvm/lib/IR/AsmWriter.cpp +++ llvm/lib/IR/AsmWriter.cpp @@ -2519,6 +2519,7 @@ void printSummaryInfo(unsigned Slot, const ValueInfo &VI); void printSummary(const GlobalValueSummary &Summary); void printAliasSummary(const AliasSummary *AS); + void printIfuncSummary(const IfuncSummary *IF); void printGlobalVarSummary(const GlobalVarSummary *GS); void printFunctionSummary(const FunctionSummary *FS); void printTypeIdSummary(const TypeIdSummary &TIS); @@ -2993,6 +2994,8 @@ switch (SK) { case GlobalValueSummary::AliasKind: return "alias"; + case GlobalValueSummary::IfuncKind: + return "ifunc"; case GlobalValueSummary::FunctionKind: return "function"; case GlobalValueSummary::GlobalVarKind: @@ -3006,8 +3009,19 @@ // The indexes emitted for distributed backends may not include the // aliasee summary (only if it is being imported directly). Handle // that case by just emitting "null" as the aliasee. - if (AS->hasAliasee()) - Out << "^" << Machine.getGUIDSlot(SummaryToGUIDMap[&AS->getAliasee()]); + if (AS->hasSummary()) + Out << "^" << Machine.getGUIDSlot(SummaryToGUIDMap[&AS->getSummary()]); + else + Out << "null"; +} + +void AssemblyWriter::printIfuncSummary(const IfuncSummary *IF) { + Out << ", resolver: "; + // The indexes emitted for distributed backends may not include the + // resolver summary (only if it is being imported directly). Handle + // that case by just emitting "null" as the resolver. + if (IF->hasSummary()) + Out << "^" << Machine.getGUIDSlot(SummaryToGUIDMap[&IF->getSummary()]); else Out << "null"; } @@ -3248,6 +3262,8 @@ if (Summary.getSummaryKind() == GlobalValueSummary::AliasKind) printAliasSummary(cast(&Summary)); + else if (Summary.getSummaryKind() == GlobalValueSummary::IfuncKind) + printIfuncSummary(cast(&Summary)); else if (Summary.getSummaryKind() == GlobalValueSummary::FunctionKind) printFunctionSummary(cast(&Summary)); else Index: llvm/lib/IR/Globals.cpp =================================================================== --- llvm/lib/IR/Globals.cpp +++ llvm/lib/IR/Globals.cpp @@ -18,6 +18,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalIFunc.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" @@ -165,6 +166,10 @@ if (const GlobalObject *GO = GA->getBaseObject()) return GO->getSection(); return ""; + } else if (auto *GA = dyn_cast(this)) { + if (const GlobalObject *GO = GA->getBaseObject()) + return GO->getSection(); + return ""; } return cast(this)->getSection(); } Index: llvm/lib/IR/ModuleSummaryIndex.cpp =================================================================== --- llvm/lib/IR/ModuleSummaryIndex.cpp +++ llvm/lib/IR/ModuleSummaryIndex.cpp @@ -590,7 +590,7 @@ R.isWriteOnly() ? -1 : (R.isReadOnly() ? -2 : -3)); if (auto *AS = dyn_cast_or_null(SummaryIt.second)) { - Draw(SummaryIt.first, AS->getAliaseeGUID(), -4); + Draw(SummaryIt.first, AS->getSummaryGUID(), -4); continue; } Index: llvm/lib/LTO/LTO.cpp =================================================================== --- llvm/lib/LTO/LTO.cpp +++ llvm/lib/LTO/LTO.cpp @@ -247,10 +247,10 @@ for (auto &ImpF : ImpM.second) { GlobalValueSummary *S = Index.findSummaryInModule(ImpF, ImpM.first()); AddUsedThings(S); - // If this is an alias, we also care about any types/etc. that the aliasee - // may reference. - if (auto *AS = dyn_cast_or_null(S)) - AddUsedThings(AS->getBaseObject()); + // If this is an alias/ifunc, we also care about any types/etc. + // that the aliasee/resolver may reference. + if (auto *GIV = dyn_cast_or_null(S)) + AddUsedThings(GIV->getBaseObject()); } auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) { @@ -315,7 +315,8 @@ } static void thinLTOResolvePrevailingGUID( - ValueInfo VI, DenseSet &GlobalInvolvedWithAlias, + ValueInfo VI, + DenseSet &GlobalInvolvedWithIndirectValue, function_ref isPrevailing, function_ref @@ -353,9 +354,10 @@ !GUIDPreservedSymbols.count(VI.getGUID())); } } - // Alias and aliasee can't be turned into available_externally. - else if (!isa(S.get()) && - !GlobalInvolvedWithAlias.count(S.get())) + // Alias and aliasee or ifunc and resolver can't be turned into + // available_externally. + else if (!isa(S.get()) && + !GlobalInvolvedWithIndirectValue.count(S.get())) S->setLinkage(GlobalValue::AvailableExternallyLinkage); if (S->linkage() != OriginalLinkage) recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage()); @@ -378,16 +380,16 @@ // We won't optimize the globals that are referenced by an alias for now // Ideally we should turn the alias into a global and duplicate the definition // when needed. - DenseSet GlobalInvolvedWithAlias; + DenseSet GlobalInvolvedWithIndirectValue; for (auto &I : Index) for (auto &S : I.second.SummaryList) - if (auto AS = dyn_cast(S.get())) - GlobalInvolvedWithAlias.insert(&AS->getAliasee()); + if (auto GIV = dyn_cast(S.get())) + GlobalInvolvedWithIndirectValue.insert(&GIV->getSummary()); for (auto &I : Index) - thinLTOResolvePrevailingGUID(Index.getValueInfo(I), GlobalInvolvedWithAlias, - isPrevailing, recordNewLinkage, - GUIDPreservedSymbols); + thinLTOResolvePrevailingGUID(Index.getValueInfo(I), + GlobalInvolvedWithIndirectValue, isPrevailing, + recordNewLinkage, GUIDPreservedSymbols); } static bool isWeakObjectWithRWAccess(GlobalValueSummary *GVS) { Index: llvm/lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- llvm/lib/Transforms/IPO/FunctionImport.cpp +++ llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -24,6 +24,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalIFunc.h" #include "llvm/IR/GlobalObject.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" @@ -533,11 +534,9 @@ // Populate the worklist with the import for the functions in the current // module for (auto &GVSummary : DefinedGVSummaries) { -#ifndef NDEBUG // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID // so this map look up (and possibly others) can be avoided. auto VI = Index.getValueInfo(GVSummary.first); -#endif if (!Index.isGlobalValueLive(GVSummary.second)) { LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n"); continue; @@ -547,6 +546,10 @@ if (!FuncSummary) // Skip import for global variables continue; + if ((!VI.getSummaryList().empty()) && + (isa(VI.getSummaryList()[0].get()))) + // Skip import for ifuncs + continue; LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n"); computeImportForFunction(*FuncSummary, Index, ImportInstrLimit, DefinedGVSummaries, Worklist, ImportList, @@ -838,7 +841,7 @@ } // Make value live and add it to the worklist if it was not live before. - auto visit = [&](ValueInfo VI, bool IsAliasee) { + auto visit = [&](ValueInfo VI, bool IsAliaseeOrResolver) { // FIXME: If we knew which edges were created for indirect call profiles, // we could skip them here. Any that are live should be reached via // other edges, e.g. reference edges. Otherwise, using a profile collected @@ -874,7 +877,7 @@ Interposable = true; } - if (!IsAliasee) { + if (!IsAliaseeOrResolver) { if (!KeepAliveLinkage) return; @@ -895,13 +898,14 @@ auto VI = Worklist.pop_back_val(); for (auto &Summary : VI.getSummaryList()) { Summary->setLive(true); - if (auto *AS = dyn_cast(Summary.get())) { - // If this is an alias, visit the aliasee VI to ensure that all copies - // are marked live and it is added to the worklist for further - // processing of its references. - visit(AS->getAliaseeVI(), true); + if (auto *GIV = dyn_cast(Summary.get())) { + // If this is an alias or ifunc, visit the aliasee / resolver VI to + // ensure that all copies are marked live and it is added to the + // worklist for further processing of its references. + visit(GIV->getSummaryVI(), true); continue; } + for (auto Ref : Summary->refs()) visit(Ref, false); if (auto *FS = dyn_cast(Summary.get())) Index: llvm/test/Bitcode/thinlto-ifunc.ll =================================================================== --- /dev/null +++ llvm/test/Bitcode/thinlto-ifunc.ll @@ -0,0 +1,39 @@ +; Test to check the callgraph in summary +; RUN: opt -module-summary %s -o %t.o +; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s +; RUN: opt -module-summary %s -o %t2.o +; RUN: llvm-dis -o - %t2.o | FileCheck %s --check-prefix=DIS + +@ifunc = dso_local ifunc i32 (), bitcast (i32 ()* ()* @resolver to i32 ()*) + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 ()* @resolver() #0 { + ret i32 ()* @called +} + +; Function Attrs: noinline nounwind optnone uwtable +define internal i32 @called() #0 { + ret i32 1 +} + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 @main() #0 { + %1 = call i32 @ifunc() + ret i32 0 +} + +; CHECK: +; CHECK-NEXT: +; CHECK-NEXT: +; CHECK-NEXT: +; CHECK-NEXT: + +; DIS: ^0 = module: (path: "{{.*}}", hash: (0, 0, 0, 0, 0)) +; DIS: ^1 = gv: (name: "ifunc", summaries: (ifunc: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), resolver: ^4))) ; guid = 1234216394087659437 +; DIS: ^2 = gv: (name: "called", summaries: (function: (module: ^0, flags: (linkage: internal, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 1))) ; guid = 14789455179670652464 +; DIS: ^3 = gv: (name: "main", summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, calls: ((callee: ^1))))) ; guid = 15822663052811949562 +; DIS: ^4 = gv: (name: "resolver", summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 1, refs: (^2)))) ; guid = 18291748799076262136 Index: llvm/test/ThinLTO/X86/Inputs/ifunc_import.ll =================================================================== --- /dev/null +++ llvm/test/ThinLTO/X86/Inputs/ifunc_import.ll @@ -0,0 +1,65 @@ +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" + +define dso_local void @called() { + ret void +} + +@globalIfunc = ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +@globalWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +@globalLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +@globalWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +@globalLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +define hidden void ()* @globalResolver() { +entry: + ret void ()* @called +} + +@internalIfunc = ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +@internalWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +@internalLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +@internalWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +@internalLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +define internal void ()* @internalResolver() { +entry: + ret void ()* @called +} + +@linkonceODRIfunc = ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +@linkonceODRWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +@linkonceODRLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +@linkonceODRWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +@linkonceODRLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +define linkonce_odr void ()* @linkonceODRResolver() { +entry: + ret void ()* @called +} + +@weakODRIfunc = ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +@weakODRWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +@weakODRLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +@weakODRWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +@weakODRLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +define weak_odr void ()* @weakODRResolver() { +entry: + ret void ()* @called +} + +@linkonceIfunc = ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +@linkonceWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +@linkonceLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +@linkonceWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +@linkonceLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +define linkonce void ()* @linkonceResolver() { +entry: + ret void ()* @called +} + +@weakIfunc = ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +@weakWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +@weakLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +@weakWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +@weakLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +define weak void ()* @weakResolver() { +entry: + ret void ()* @called +} Index: llvm/test/ThinLTO/X86/ifunc_import.ll =================================================================== --- /dev/null +++ llvm/test/ThinLTO/X86/ifunc_import.ll @@ -0,0 +1,154 @@ +; RUN: opt -module-summary %s -o %t1.bc +; RUN: opt -module-summary %p/Inputs/ifunc_import.ll -o %t2.bc +; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc +; RUN: llvm-lto -thinlto-action=promote -thinlto-index %t.index.bc %t2.bc -o - | llvm-dis -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=PROMOTE +; RUN: llvm-lto -thinlto-action=import -thinlto-index %t.index.bc %t1.bc -o - | llvm-dis -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=IMPORT + +; PROMOTE-DAG: @globalIfunc = ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +; PROMOTE-DAG: @globalWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +; PROMOTE-DAG: @globalLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +; PROMOTE-DAG: @globalWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +; PROMOTE-DAG: @globalLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @globalResolver to void ()*) +; PROMOTE-DAG: @internalIfunc = ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +; PROMOTE-DAG: @internalWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +; PROMOTE-DAG: @internalLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +; PROMOTE-DAG: @internalWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +; PROMOTE-DAG: @internalLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @internalResolver to void ()*) +; PROMOTE-DAG: @linkonceODRIfunc = ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +; PROMOTE-DAG: @linkonceODRWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +; PROMOTE-DAG: @linkonceODRLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +; PROMOTE-DAG: @linkonceODRWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +; PROMOTE-DAG: @linkonceODRLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @linkonceODRResolver to void ()*) +; PROMOTE-DAG: @weakODRIfunc = ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +; PROMOTE-DAG: @weakODRWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +; PROMOTE-DAG: @weakODRLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +; PROMOTE-DAG: @weakODRWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +; PROMOTE-DAG: @weakODRLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @weakODRResolver to void ()*) +; PROMOTE-DAG: @linkonceIfunc = ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +; PROMOTE-DAG: @linkonceWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +; PROMOTE-DAG: @linkonceLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +; PROMOTE-DAG: @linkonceWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +; PROMOTE-DAG: @linkonceLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @linkonceResolver to void ()*) +; PROMOTE-DAG: @weakIfunc = ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +; PROMOTE-DAG: @weakWeakIfunc = weak ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +; PROMOTE-DAG: @weakLinkonceIfunc = linkonce ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +; PROMOTE-DAG: @weakWeakODRIfunc = weak_odr ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) +; PROMOTE-DAG: @weakLinkonceODRIfunc = linkonce_odr ifunc void (), bitcast (void ()* ()* @weakResolver to void ()*) + +; PROMOTE-DAG: define hidden void ()* @globalResolver() +; PROMOTE-DAG: define internal void ()* @internalResolver() +; PROMOTE-DAG: define weak_odr void ()* @linkonceODRResolver() +; PROMOTE-DAG: define weak_odr void ()* @weakODRResolver() +; PROMOTE-DAG: define weak void ()* @linkonceResolver() +; PROMOTE-DAG: define weak void ()* @weakResolver() + +; IMPORT-DAG: declare void @globalIfunc() +; IMPORT-DAG: declare void @globalWeakIfunc() +; IMPORT-DAG: declare void @globalLinkonceIfunc() +; IMPORT-DAG: declare void @globalWeakODRIfunc() +; IMPORT-DAG: declare void @globalLinkonceODRIfunc() +; IMPORT-DAG: declare void @internalIfunc() +; IMPORT-DAG: declare void @internalWeakIfunc() +; IMPORT-DAG: declare void @internalLinkonceIfunc() +; IMPORT-DAG: declare void @internalWeakODRIfunc() +; IMPORT-DAG: declare void @internalLinkonceODRIfunc() +; IMPORT-DAG: declare void @linkonceODRIfunc() +; IMPORT-DAG: declare void @linkonceODRWeakIfunc() +; IMPORT-DAG: declare void @linkonceODRLinkonceIfunc() +; IMPORT-DAG: declare void @linkonceODRWeakODRIfunc() +; IMPORT-DAG: declare void @linkonceODRLinkonceODRIfunc() +; IMPORT-DAG: declare void @weakODRIfunc() +; IMPORT-DAG: declare void @weakODRWeakIfunc() +; IMPORT-DAG: declare void @weakODRLinkonceIfunc() +; IMPORT-DAG: declare void @weakODRWeakODRIfunc() +; IMPORT-DAG: declare void @weakODRLinkonceODRIfunc() +; IMPORT-DAG: declare void @linkonceIfunc() +; IMPORT-DAG: declare void @linkonceWeakIfunc() +; IMPORT-DAG: declare void @linkonceLinkonceIfunc() +; IMPORT-DAG: declare void @linkonceWeakODRIfunc() +; IMPORT-DAG: declare void @linkonceLinkonceODRIfunc() +; IMPORT-DAG: declare void @weakIfunc() +; IMPORT-DAG: declare void @weakWeakIfunc() +; IMPORT-DAG: declare void @weakLinkonceIfunc() +; IMPORT-DAG: declare void @weakWeakODRIfunc() +; IMPORT-DAG: declare void @weakLinkonceODRIfunc() + +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" + +define i32 @main() { +entry: + call void @globalIfunc() + call void @globalWeakIfunc() + call void @globalLinkonceIfunc() + call void @globalWeakODRIfunc() + call void @globalLinkonceODRIfunc() + + call void @internalIfunc() + call void @internalWeakIfunc() + call void @internalLinkonceIfunc() + call void @internalWeakODRIfunc() + call void @internalLinkonceODRIfunc() + + call void @linkonceODRIfunc() + call void @linkonceODRWeakIfunc() + call void @linkonceODRLinkonceIfunc() + call void @linkonceODRWeakODRIfunc() + call void @linkonceODRLinkonceODRIfunc() + + call void @weakODRIfunc() + call void @weakODRWeakIfunc() + call void @weakODRLinkonceIfunc() + call void @weakODRWeakODRIfunc() + call void @weakODRLinkonceODRIfunc() + + call void @linkonceIfunc() + call void @linkonceWeakIfunc() + call void @linkonceLinkonceIfunc() + call void @linkonceWeakODRIfunc() + call void @linkonceLinkonceODRIfunc() + + call void @weakIfunc() + call void @weakWeakIfunc() + call void @weakLinkonceIfunc() + call void @weakWeakODRIfunc() + call void @weakLinkonceODRIfunc() + + ret i32 0 +} + + +declare void @globalIfunc() +declare void @globalWeakIfunc() +declare void @globalLinkonceIfunc() +declare void @globalWeakODRIfunc() +declare void @globalLinkonceODRIfunc() + +declare void @internalIfunc() +declare void @internalWeakIfunc() +declare void @internalLinkonceIfunc() +declare void @internalWeakODRIfunc() +declare void @internalLinkonceODRIfunc() + +declare void @linkonceODRIfunc() +declare void @linkonceODRWeakIfunc() +declare void @linkonceODRLinkonceIfunc() +declare void @linkonceODRWeakODRIfunc() +declare void @linkonceODRLinkonceODRIfunc() + +declare void @weakODRIfunc() +declare void @weakODRWeakIfunc() +declare void @weakODRLinkonceIfunc() +declare void @weakODRWeakODRIfunc() +declare void @weakODRLinkonceODRIfunc() + +declare void @linkonceIfunc() +declare void @linkonceWeakIfunc() +declare void @linkonceLinkonceIfunc() +declare void @linkonceWeakODRIfunc() +declare void @linkonceLinkonceODRIfunc() + +declare void @weakIfunc() +declare void @weakWeakIfunc() +declare void @weakLinkonceIfunc() +declare void @weakWeakODRIfunc() +declare void @weakLinkonceODRIfunc()