diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -295,6 +295,9 @@ // Range information for accessed offsets for every argument. // [n x (paramno, range, numcalls, numcalls x (callee_guid, paramno, range))] FS_PARAM_ACCESS = 25, + // IFUNC: [valueid, flags, valueid] + FS_IFUNC = 26, + }; enum MetadataCodes { diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -281,7 +281,12 @@ class GlobalValueSummary { public: /// Sububclass discriminator (for dyn_cast<> et al.) - enum SummaryKind : unsigned { AliasKind, FunctionKind, GlobalVarKind }; + enum SummaryKind : unsigned { + AliasKind, + IfuncKind, + FunctionKind, + GlobalVarKind + }; /// Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield. struct GVFlags { @@ -475,6 +480,52 @@ } }; +/// Ifunc summary information. +class IfuncSummary : public GlobalValueSummary { + ValueInfo ResolverValueInfo; + GlobalValueSummary *ResolverSummary; + +public: + IfuncSummary(GVFlags Flags) + : GlobalValueSummary(IfuncKind, Flags, ArrayRef{}), + ResolverSummary(nullptr) {} + + /// Check if this is an ifunc summary. + static bool classof(const GlobalValueSummary *GVS) { + return GVS->getSummaryKind() == IfuncKind; + } + + void setResolver(ValueInfo &ResolverVI, GlobalValueSummary *Resolver) { + ResolverValueInfo = ResolverVI; + ResolverSummary = Resolver; + } + + bool hasResolver() const { + assert(!!ResolverSummary == (ResolverValueInfo && + !ResolverValueInfo.getSummaryList().empty()) && + "Expect to have both resolver summary and summary list or neither"); + return !!ResolverSummary; + } + + const GlobalValueSummary &getResolver() const { + assert(ResolverSummary && "Unexpected missing resolver summary"); + return *ResolverSummary; + } + + GlobalValueSummary &getResolver() { + return const_cast( + static_cast(this)->getResolver()); + } + ValueInfo getResolverVI() const { + assert(ResolverValueInfo && "Unexpected missing resolver"); + return ResolverValueInfo; + } + GlobalValue::GUID getResolverGUID() const { + assert(ResolverValueInfo && "Unexpected missing resolver"); + return ResolverValueInfo.getGUID(); + } +}; + const inline GlobalValueSummary *GlobalValueSummary::getBaseObject() const { if (auto *AS = dyn_cast(this)) return &AS->getAliasee(); diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -330,6 +330,13 @@ assert(!CalledFunction && "Expected null called function in callsite for alias"); CalledFunction = dyn_cast(GA->getBaseObject()); } + + if (auto *GIF = dyn_cast(CalledValue)) { + assert(!CalledFunction && + "Expected null called function in callsite for ifunc"); + CalledFunction = dyn_cast(GIF->getBaseObject()); + } + // Check if this is a direct call to a known function or a known // intrinsic, or an indirect call with profile data. if (CalledFunction) { @@ -347,10 +354,10 @@ if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None) Hotness = CalleeInfo::HotnessType::Cold; - // Use the original CalledValue, in case it was an alias. We want - // to record the call edge to the alias in that case. Eventually - // an alias summary will be created to associate the alias and - // aliasee. + // Use the original CalledValue, in case it was an alias or ifunc. + // We want to record the call edge to the alias in that case. + // Eventuallyan alias summary will be created to associate the alias + // and aliasee or ifunc and its resolver. auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo( cast(CalledValue))]; ValueInfo.updateHotness(Hotness); @@ -637,6 +644,25 @@ Index.addGlobalValueSummary(A, std::move(AS)); } +static void computeIfuncSummary(ModuleSummaryIndex &Index, const GlobalIFunc &I, + DenseSet &CantBePromoted) { + bool NonRenamableLocal = isNonRenamableLocal(I); + GlobalValueSummary::GVFlags Flags(I.getLinkage(), NonRenamableLocal, + /* Live = */ false, I.isDSOLocal(), + I.hasLinkOnceODRLinkage() && + I.hasGlobalUnnamedAddr()); + auto IF = std::make_unique(Flags); + auto *Resolver = I.getBaseObject(); + auto ResolverVI = Index.getValueInfo(Resolver->getGUID()); + assert(ResolverVI && "Ifunc expects ifunc summary to be available"); + assert(ResolverVI.getSummaryList().size() == 1 && + "Expected a single entry per ifunc in per-module index"); + IF->setResolver(ResolverVI, ResolverVI.getSummaryList()[0].get()); + if (NonRenamableLocal) + CantBePromoted.insert(I.getGUID()); + Index.addGlobalValueSummary(I, std::move(IF)); +} + // Set LiveRoot flag on entries matching the given value name. static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) { if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name))) @@ -779,6 +805,9 @@ for (const GlobalAlias &A : M.aliases()) computeAliasSummary(Index, A, CantBePromoted); + for (const GlobalIFunc &I : M.ifuncs()) + computeIfuncSummary(Index, I, CantBePromoted); + for (auto *V : LocalsUsed) { auto *Summary = Index.getGlobalValueSummary(*V); assert(Summary && "Missing summary for global value"); diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -139,6 +139,7 @@ STRINGIFY_CODE(MODULE_CODE, METADATA_VALUES_UNUSED) STRINGIFY_CODE(MODULE_CODE, SOURCE_FILENAME) STRINGIFY_CODE(MODULE_CODE, HASH) + STRINGIFY_CODE(MODULE_CODE, IFUNC) } case bitc::IDENTIFICATION_BLOCK_ID: switch (CodeID) { @@ -307,6 +308,7 @@ STRINGIFY_CODE(FS, TYPE_ID_METADATA) STRINGIFY_CODE(FS, BLOCK_COUNT) STRINGIFY_CODE(FS, PARAM_ACCESS) + STRINGIFY_CODE(FS, IFUNC) } case bitc::METADATA_ATTACHMENT_ID: switch (CodeID) { diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -5706,6 +5706,7 @@ // v2: [strtab offset, strtab size, v1] case bitc::MODULE_CODE_GLOBALVAR: case bitc::MODULE_CODE_FUNCTION: + case bitc::MODULE_CODE_IFUNC: case bitc::MODULE_CODE_ALIAS: { StringRef Name; ArrayRef GVRecord; @@ -6057,6 +6058,34 @@ TheIndex.addGlobalValueSummary(GUID.first, std::move(AS)); break; } + + case bitc::FS_IFUNC: { + unsigned ValueID = Record[0]; + uint64_t RawFlags = Record[1]; + unsigned ResolverID = Record[2]; + + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + auto IF = std::make_unique(Flags); + // The module path string ref set in the summary must be owned by the + // index's module string table. Since we don't have a module path + // string table section in the per-module index, we create a single + // module path string table entry with an empty (0) ID to take + // ownership. + IF->setModulePath(getThisModule()->first()); + + auto ResolverVI = getValueInfoFromValueId(ResolverID).first; + auto ResolverInModule = + TheIndex.findSummaryInModule(ResolverVI, ModulePath); + if (!ResolverInModule) + return error("Ifunc expects resolver summary to be parsed"); + IF->setResolver(ResolverVI, ResolverInModule); + + auto GUID = getValueInfoFromValueId(ValueID); + IF->setOriginalName(GUID.second); + TheIndex.addGlobalValueSummary(GUID.first, std::move(IF)); + break; + } + // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags, n x valueid] case bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS: { unsigned ValueID = Record[0]; diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -443,11 +443,14 @@ for (auto &M : *ModuleToSummariesForIndex) for (auto &Summary : M.second) { Callback(Summary, false); - // Ensure aliasee is handled, e.g. for assigning a valueId, + // Ensure aliasee and ifunc resolver are handled, + // e.g. for assigning a valueId, // even if we are not importing the aliasee directly (the // imported alias will contain a copy of aliasee). if (auto *AS = dyn_cast(Summary.getSecond())) Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true); + if (auto *IF = dyn_cast(Summary.getSecond())) + Callback({IF->getResolverGUID(), &IF->getResolver()}, true); } } else { for (auto &Summaries : Index) @@ -3868,6 +3871,14 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + // Abbrev for FS_IFUNC. + Abbv = std::make_shared(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_IFUNC)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + unsigned FSIfuncAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + // Abbrev for FS_TYPE_ID_METADATA Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA)); @@ -3921,6 +3932,21 @@ NameVals.clear(); } + for (const GlobalIFunc &I : M.ifuncs()) { + auto *Ifunc = I.getBaseObject(); + auto IfuncId = VE.getValueID(&I); + auto ResolverId = VE.getValueID(Ifunc); + + auto *Summary = Index->getGlobalValueSummary(I); + IfuncSummary *IF = cast(Summary); + + NameVals.push_back(IfuncId); + NameVals.push_back(getEncodedGVSummaryFlags(IF->flags())); + NameVals.push_back(ResolverId); + Stream.EmitRecord(bitc::FS_IFUNC, NameVals, FSIfuncAbbrev); + NameVals.clear(); + } + for (auto &S : Index->typeIdCompatibleVtableMap()) { writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, S.first, S.second, VE); @@ -4007,6 +4033,10 @@ // id of the aliasee. Save them in a vector for post-processing. SmallVector Aliases; + // The ifuncs are emitted as a post-pass, and will point to the value + // id of the resolver. Save them in a vector for post-processing. + SmallVector Ifuncs; + // Save the value id for each summary for alias emission. DenseMap SummaryToValueIdMap; @@ -4027,7 +4057,7 @@ }; std::set DefOrUseGUIDs; - forEachSummary([&](GVInfo I, bool IsAliasee) { + forEachSummary([&](GVInfo I, bool IsAliaseeOrIRes) { GlobalValueSummary *S = I.second; assert(S); DefOrUseGUIDs.insert(I.first); @@ -4038,10 +4068,11 @@ assert(ValueId); SummaryToValueIdMap[S] = *ValueId; - // If this is invoked for an aliasee, we want to record the above - // mapping, but then not emit a summary entry (if the aliasee is - // to be imported, we will invoke this separately with IsAliasee=false). - if (IsAliasee) + // If this is invoked for an aliasee or ifunc resolver, we want to record + // the above mapping, but then not emit a summary entry (if the aliasee is + // to be imported, we will invoke this separately with + // IsAliaseeOrIRes=false). + if (IsAliaseeOrIRes) return; if (auto *AS = dyn_cast(S)) { @@ -4051,6 +4082,13 @@ return; } + if (auto *IF = dyn_cast(S)) { + // Will process ifuncs as a post-pass because the reader wants all + // global to be loaded first. + Ifuncs.push_back(IF); + return; + } + if (auto *VS = dyn_cast(S)) { NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(VS->modulePath())); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -2500,6 +2500,7 @@ void printSummaryInfo(unsigned Slot, const ValueInfo &VI); void printSummary(const GlobalValueSummary &Summary); void printAliasSummary(const AliasSummary *AS); + void printIfuncSummary(const IfuncSummary *IF); void printGlobalVarSummary(const GlobalVarSummary *GS); void printFunctionSummary(const FunctionSummary *FS); void printTypeIdSummary(const TypeIdSummary &TIS); @@ -2972,6 +2973,8 @@ switch (SK) { case GlobalValueSummary::AliasKind: return "alias"; + case GlobalValueSummary::IfuncKind: + return "ifunc"; case GlobalValueSummary::FunctionKind: return "function"; case GlobalValueSummary::GlobalVarKind: @@ -2991,6 +2994,17 @@ Out << "null"; } +void AssemblyWriter::printIfuncSummary(const IfuncSummary *IF) { + Out << ", resolver: "; + // The indexes emitted for distributed backends may not include the + // resolver summary (only if it is being imported directly). Handle + // that case by just emitting "null" as the resolver. + if (IF->hasResolver()) + Out << "^" << Machine.getGUIDSlot(SummaryToGUIDMap[&IF->getResolver()]); + else + Out << "null"; +} + void AssemblyWriter::printGlobalVarSummary(const GlobalVarSummary *GS) { auto VTableFuncs = GS->vTableFuncs(); Out << ", varFlags: (readonly: " << GS->VarFlags.MaybeReadOnly << ", " @@ -3227,6 +3241,8 @@ if (Summary.getSummaryKind() == GlobalValueSummary::AliasKind) printAliasSummary(cast(&Summary)); + else if (Summary.getSummaryKind() == GlobalValueSummary::IfuncKind) + printIfuncSummary(cast(&Summary)); else if (Summary.getSummaryKind() == GlobalValueSummary::FunctionKind) printFunctionSummary(cast(&Summary)); else diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -77,6 +77,7 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalIFunc.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/InlineAsm.h" @@ -379,6 +380,9 @@ for (const GlobalAlias &GA : M.aliases()) visitGlobalAlias(GA); + for (const GlobalIFunc &I : M.ifuncs()) + visitGlobalIfunc(I); + for (const NamedMDNode &NMD : M.named_metadata()) visitNamedMDNode(NMD); @@ -404,6 +408,7 @@ void visitGlobalValue(const GlobalValue &GV); void visitGlobalVariable(const GlobalVariable &GV); void visitGlobalAlias(const GlobalAlias &GA); + void visitGlobalIfunc(const GlobalIFunc &GIF); void visitAliaseeSubExpr(const GlobalAlias &A, const Constant &C); void visitAliaseeSubExpr(SmallPtrSetImpl &Visited, const GlobalAlias &A, const Constant &C); @@ -679,7 +684,7 @@ for (Value *Op : InitArray->operands()) { Value *V = Op->stripPointerCasts(); Assert(isa(V) || isa(V) || - isa(V), + isa(V) || isa(V), "invalid llvm.used member", V); Assert(V->hasName(), "members of llvm.used must be named", V); } @@ -770,6 +775,12 @@ visitGlobalValue(GA); } +void Verifier::visitGlobalIfunc(const GlobalIFunc &GIF) { + auto *Resolver = GIF.getResolver(); + Assert(Resolver, "Ifunc resolver cannot be NULL!", &GIF); + visitGlobalValue(GIF); +} + void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { // There used to be various other llvm.dbg.* nodes, but we don't support // upgrading them and we want to reserve the namespace for future uses. diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp --- a/llvm/lib/Transforms/IPO/FunctionImport.cpp +++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -828,7 +828,7 @@ } // Make value live and add it to the worklist if it was not live before. - auto visit = [&](ValueInfo VI, bool IsAliasee) { + auto visit = [&](ValueInfo VI, bool IsAliaseeOrResolver) { // FIXME: If we knew which edges were created for indirect call profiles, // we could skip them here. Any that are live should be reached via // other edges, e.g. reference edges. Otherwise, using a profile collected @@ -864,7 +864,7 @@ Interposable = true; } - if (!IsAliasee) { + if (!IsAliaseeOrResolver) { if (!KeepAliveLinkage) return; @@ -892,6 +892,14 @@ continue; } + if (auto *FI = dyn_cast(Summary.get())) { + // If this is an ifunc, visit the resolver VI to ensure that all copies + // are marked live and it is added to the worklist for further + // processing of its references. + visit(FI->getResolverVI(), true); + continue; + } + Summary->setLive(true); for (auto Ref : Summary->refs()) visit(Ref, false); diff --git a/llvm/test/Bitcode/thinlto-ifunc.ll b/llvm/test/Bitcode/thinlto-ifunc.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Bitcode/thinlto-ifunc.ll @@ -0,0 +1,39 @@ +; Test to check the callgraph in summary +; RUN: opt -module-summary %s -o %t.o +; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s +; RUN: opt -module-summary %s -o %t2.o +; RUN: llvm-dis -o - %t2.o | FileCheck %s --check-prefix=DIS + +@ifunc = dso_local ifunc i32 (), bitcast (i32 ()* ()* @resolver to i32 ()*) + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 ()* @resolver() #0 { + ret i32 ()* @called +} + +; Function Attrs: noinline nounwind optnone uwtable +define internal i32 @called() #0 { + ret i32 1 +} + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 @main() #0 { + %1 = call i32 @ifunc() + ret i32 0 +} + +; CHECK: +; CHECK-NEXT: +; CHECK-NEXT: +; CHECK-NEXT: +; CHECK-NEXT: + +; DIS: ^0 = module: (path: "{{.*}}", hash: (0, 0, 0, 0, 0)) +; DIS: ^1 = gv: (name: "ifunc", summaries: (ifunc: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), resolver: ^4))) ; guid = 1234216394087659437 +; DIS: ^2 = gv: (name: "called", summaries: (function: (module: ^0, flags: (linkage: internal, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 1))) ; guid = 14789455179670652464 +; DIS: ^3 = gv: (name: "main", summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, calls: ((callee: ^1))))) ; guid = 15822663052811949562 +; DIS: ^4 = gv: (name: "resolver", summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 1, refs: (^2)))) ; guid = 18291748799076262136