Index: include/llvm/IR/InstrTypes.h =================================================================== --- include/llvm/IR/InstrTypes.h +++ include/llvm/IR/InstrTypes.h @@ -866,6 +866,8 @@ /// @returns true iff the proposed cast is valid. /// @brief Determine if a cast is valid without creating one. static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy); + /// Same as the above method, but test if type SrcTy can be casted to DestTy. + static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy); /// @brief Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -120,6 +120,7 @@ void initializeGCOVProfilerPass(PassRegistry&); void initializePGOInstrumentationGenPass(PassRegistry&); void initializePGOInstrumentationUsePass(PassRegistry&); +void initializePGOIndirectCallPromotionPass(PassRegistry&); void initializeInstrProfilingPass(PassRegistry&); void initializeAddressSanitizerPass(PassRegistry&); void initializeAddressSanitizerModulePass(PassRegistry&); Index: include/llvm/LinkAllPasses.h =================================================================== --- include/llvm/LinkAllPasses.h +++ include/llvm/LinkAllPasses.h @@ -90,6 +90,7 @@ (void) llvm::createGCOVProfilerPass(); (void) llvm::createPGOInstrumentationGenPass(); (void) llvm::createPGOInstrumentationUsePass(); + (void) llvm::createPGOIndirectCallPromotionPass(); (void) llvm::createInstrProfilingPass(); (void) llvm::createFunctionImportPass(); (void) llvm::createFunctionInliningPass(); Index: include/llvm/ProfileData/InstrProf.h =================================================================== --- include/llvm/ProfileData/InstrProf.h +++ include/llvm/ProfileData/InstrProf.h @@ -156,6 +156,10 @@ std::string getPGOFuncName(const Function &F, uint64_t Version = INSTR_PROF_INDEX_VERSION); +/// Return the modified name for function \c F suitable to be +/// used the key for profile lookup in LTO compilation. +std::string getPGOFuncNameLTO(const Function &F); + /// Return the modified name for a function suitable to be /// used the key for profile lookup. The function's original /// name is \c RawFuncName and has linkage of type \c Linkage. @@ -318,7 +322,7 @@ /// A wrapper interface to populate the PGO symtab with functions /// decls from module \c M. This interface is used by transformation /// passes such as indirect function call promotion. - void create(const Module &M); + void create(const Module &M, bool InLTO = false); /// Create InstrProfSymtab from a set of names iteratable from /// \p IterRange. This interface is used by IndexedProfReader. template void create(const NameIterRange &IterRange); Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -83,6 +83,7 @@ ModulePass *createPGOInstrumentationGenPass(); ModulePass * createPGOInstrumentationUsePass(StringRef Filename = StringRef("")); +ModulePass *createPGOIndirectCallPromotionPass(bool InLTO = false); /// Options for the frontend instrumentation based profiling pass. struct InstrProfOptions { Index: lib/IR/Instructions.cpp =================================================================== --- lib/IR/Instructions.cpp +++ lib/IR/Instructions.cpp @@ -3009,6 +3009,11 @@ // Check for type sanity on the arguments Type *SrcTy = S->getType(); + return castIsValid(op, SrcTy, DstTy); +} + +bool +CastInst::castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy) { if (!SrcTy->isFirstClassType() || !DstTy->isFirstClassType() || SrcTy->isAggregateType() || DstTy->isAggregateType()) Index: lib/ProfileData/InstrProf.cpp =================================================================== --- lib/ProfileData/InstrProf.cpp +++ lib/ProfileData/InstrProf.cpp @@ -88,6 +88,21 @@ Version); } +std::string getPGOFuncNameLTO(const Function &F) { + // First check if these is a metadata. + MDNode *MD = F.getMetadata("PGOFuncName"); + if (MD != nullptr) { + StringRef S = cast(MD->getOperand(0))->getString(); + return S.str(); + } + // Strip out prefix from the the internalized symbols. + std::string PGOName = getPGOFuncName(F); + size_t S = PGOName.find_first_of(':'); + if (S == std::string::npos) + return PGOName; + return PGOName.substr(S + 1); +} + StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { if (FileName.empty()) return PGOFuncName; @@ -148,9 +163,15 @@ return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName); } -void InstrProfSymtab::create(const Module &M) { - for (const Function &F : M) - addFuncName(getPGOFuncName(F)); +void InstrProfSymtab::create(const Module &M, bool InLTO) { + for (const Function &F : M) { + if (!F.hasName()) + continue; + if (InLTO) + addFuncName(getPGOFuncNameLTO(F)); + else + addFuncName(getPGOFuncName(F)); + } finalizeSymtab(); } Index: lib/Transforms/IPO/PassManagerBuilder.cpp =================================================================== --- lib/Transforms/IPO/PassManagerBuilder.cpp +++ lib/Transforms/IPO/PassManagerBuilder.cpp @@ -217,6 +217,9 @@ } if (!PGOInstrUse.empty()) MPM.add(createPGOInstrumentationUsePass(PGOInstrUse)); + + // Indirect call promotion. + MPM.add(createPGOIndirectCallPromotionPass()); } void PassManagerBuilder::addFunctionSimplificationPasses( legacy::PassManagerBase &MPM) { @@ -572,6 +575,9 @@ // Infer attributes about declarations if possible. PM.add(createInferFunctionAttrsLegacyPass()); + // Indirect call promotion. (xur) + PM.add(createPGOIndirectCallPromotionPass(true)); + // Propagate constants at call sites into the functions they call. This // opens opportunities for globalopt (and inlining) by substituting function // pointers passed as arguments to direct uses of functions. Index: lib/Transforms/Instrumentation/Instrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/Instrumentation.cpp +++ lib/Transforms/Instrumentation/Instrumentation.cpp @@ -62,6 +62,7 @@ initializeGCOVProfilerPass(Registry); initializePGOInstrumentationGenPass(Registry); initializePGOInstrumentationUsePass(Registry); + initializePGOIndirectCallPromotionPass(Registry); initializeInstrProfilingPass(Registry); initializeMemorySanitizerPass(Registry); initializeThreadSanitizerPass(Registry); Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -88,6 +88,7 @@ STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile."); STATISTIC(NumOfPGOMissing, "Number of functions without profile."); STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations."); +STATISTIC(NumOfPGOICallPromotion, "Number of indirect call promotions."); // Command line option to specify the file to read profile from. This is // mainly used for testing. @@ -102,8 +103,27 @@ static cl::opt DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling")); - +static cl::opt + ICPCountThreshold("icp-count-threshold", cl::Hidden, cl::ZeroOrMore, + cl::init(1000), + cl::desc("The minimum count to the direct call target " + "for the promotion")); + +static cl::opt + ICPPercentThreshold("icp-percent-threshold", cl::init(33), cl::Hidden, + cl::ZeroOrMore, + cl::desc("The percentage threshold for the promotion")); + +static cl::opt + MaxNumPromotions("icp-max-prom", cl::init(2), cl::Hidden, cl::ZeroOrMore, + cl::desc("Max number of promotions for a single indirect " + "call callsite")); + +static cl::opt ICPLTOMode("icp-lto", cl::init(false), cl::Hidden, + cl::desc("Run indirect-call promotion in LTO " + "mode")); namespace { + class PGOInstrumentationGen : public ModulePass { public: static char ID; @@ -149,6 +169,29 @@ AU.addRequired(); } }; + +class PGOIndirectCallPromotion : public ModulePass { +public: + static char ID; + + PGOIndirectCallPromotion(bool InLTO = false) : ModulePass(ID), InLTO(InLTO) { + // Commandline option has the priority for InLTO. + InLTO |= ICPLTOMode; + + initializePGOIndirectCallPromotionPass(*PassRegistry::getPassRegistry()); + } + + const char *getPassName() const override { + return "PGOIndirectCallPromotion"; + } + +private: + bool runOnModule(Module &M) override; + + // If this pass is called in LTO. We need to special handling the PGOFuncName + // for the static variables due to LTO's internalization. + bool InLTO; +}; } // end anonymous namespace char PGOInstrumentationGen::ID = 0; @@ -175,6 +218,16 @@ return new PGOInstrumentationUse(Filename.str()); } +char PGOIndirectCallPromotion::ID = 0; +INITIALIZE_PASS(PGOIndirectCallPromotion, "pgo-icall-prom", + "Use PGO instrumentation profile to promote indirect calls to " + "direct calls.", + false, false) + +ModulePass *llvm::createPGOIndirectCallPromotionPass(bool InLTO) { + return new PGOIndirectCallPromotion(InLTO); +} + namespace { /// \brief An MST based instrumentation for PGO /// @@ -246,6 +299,11 @@ BlockFrequencyInfo *BFI = nullptr) : F(Func), FunctionHash(0), MST(F, BPI, BFI) { FuncName = getPGOFuncName(F); + if (FuncName != Func.getName()) { + LLVMContext &C = F.getContext(); + MDNode *N = MDNode::get(C, MDString::get(C, FuncName.c_str())); + F.setMetadata("PGOFuncName", N); + } computeCFGHash(); DEBUG(dumpInfo("after CFGMST")); @@ -748,7 +806,7 @@ << IndirectCallSiteIndex << " out of " << NumValueSites << "\n"); annotateValueSite(*M, *I, ProfileRecord, IPVK_IndirectCallTarget, - IndirectCallSiteIndex); + IndirectCallSiteIndex, MaxNumPromotions); IndirectCallSiteIndex++; } } @@ -831,3 +889,359 @@ } return true; } + +// The class for main data structure to promote indirect calls to conditional +// direct calls. +class ICallPromotionFunc { +private: + Function &F; + Module *M; + + // Symtab that maps value to instrumented function name. + InstrProfSymtab *Symtab; + + // PGO function name to Function map. + const StringMap *ProfNameMap; + + // Allocate space to read the profile annotation. + std::unique_ptr ValueDataArray; + + bool isPromotionProfitable(uint64_t Count, uint64_t TotalCount); + + enum TargetStatus { + OK, // Should be able to promote. + NotAvailableInModule, // Cannot find the target in current module. + NotFoundinNameMap, // Cannot find the function defines. + ReturnTypeMismatch, // Return type mismatch b/w target and indirectcall. + NumArgsMismatch, // Number of arguments does not match. + ArgTypeMismatch // Type mismatch in the arguments (cannot bitcast). + }; + + TargetStatus isPromotionLegal(CallInst *Inst, uint64_t Target, Function *&F); + + typedef std::vector> ICPWorkList; + + // Check if the indirectcall call site should be promoted. Return the number + // of promotions. + uint32_t analyzeCandidates(CallInst *Inst, InstrProfValueData ValueData[], + uint32_t NumVals, uint64_t TotalCount, + ICPWorkList &W); + + bool doVersion(CallInst *Inst, Function *F, uint64_t Count, + uint64_t TotalCount); + + uint32_t doPromotion(CallInst *Inst, const ICPWorkList &W, + uint64_t &TotalCount); + + const char *StatusToString(const TargetStatus S) { + switch (S) { + case OK: + return "OK to promote"; + case NotAvailableInModule: + return "Cannot find the target"; + case NotFoundinNameMap: + return "Cannot find the target define"; + case ReturnTypeMismatch: + return "Return type mismatch"; + case NumArgsMismatch: + return "The number of arguments mismatch"; + case ArgTypeMismatch: + return "Argument Type mismatch"; + } + llvm_unreachable("Should not reach here"); + } + + // Noncopyable + ICallPromotionFunc(const ICallPromotionFunc &other) = delete; + ICallPromotionFunc &operator=(const ICallPromotionFunc &other) = delete; + +public: + ICallPromotionFunc(Function &Func, Module *Modu, InstrProfSymtab *Symtab, + const StringMap *ProfNameMap) + : F(Func), M(Modu), Symtab(Symtab), ProfNameMap(ProfNameMap) { + ValueDataArray = llvm::make_unique(MaxNumPromotions); + } + bool performPromotion(); +}; + +bool ICallPromotionFunc::isPromotionProfitable(uint64_t Count, + uint64_t TotalCount) { + if (TotalCount < ICPCountThreshold || Count < ICPCountThreshold) + return false; + if (TotalCount < Count) + Count = TotalCount; + if ((double)Count / TotalCount * 100 < ICPPercentThreshold) + return false; + return true; +} + +ICallPromotionFunc::TargetStatus +ICallPromotionFunc::isPromotionLegal(CallInst *Inst, uint64_t Target, + Function *&TargetFunction) { + StringRef TargetFuncName = Symtab->getFuncName(Target); + if (TargetFuncName.empty()) + return NotAvailableInModule; + auto ProfNameI = ProfNameMap->find(TargetFuncName); + if (ProfNameI == ProfNameMap->end()) + return NotFoundinNameMap; + + Function *DirectCallee = ProfNameI->getValue(); + // Check the return type. + Type *CallRetType = Inst->getType(); + if (!CallRetType->isVoidTy()) { + Type *FuncRetType = DirectCallee->getReturnType(); + if (FuncRetType != CallRetType && + !CastInst::castIsValid(Instruction::BitCast, FuncRetType, CallRetType)) + return ReturnTypeMismatch; + } + + // Check if the arguments are compatible with the parameters + FunctionType *DirectCalleeType = DirectCallee->getFunctionType(); + unsigned ParaNum = DirectCalleeType->getFunctionNumParams(); + CallSite CS(Inst); + unsigned ArgNum = CS.arg_size(); + + if (ParaNum != ArgNum && !DirectCalleeType->isVarArg()) + return NumArgsMismatch; + + for (unsigned I = 0; I < ParaNum; ++I) { + Type *PTy = DirectCalleeType->getFunctionParamType(I); + Type *ATy = CS.getArgument(I)->getType(); + if (PTy == ATy) + continue; + if (!CastInst::castIsValid(Instruction::BitCast, CS.getArgument(I), PTy)) + return ArgTypeMismatch; + } + + DEBUG(dbgs() << " Promote the icall to " << TargetFuncName << "\n"); + TargetFunction = DirectCallee; + return OK; +} + +// Indirect-call promotion heursitic. The direct targets are sorted based on +// the count. Stop at the first target that is not promoted. +uint32_t ICallPromotionFunc::analyzeCandidates(CallInst *Inst, + InstrProfValueData ValueData[], + uint32_t NumVals, + uint64_t TotalCount, + ICPWorkList &WorkList) { + uint32_t NumPromotions = 0; + + DEBUG(dbgs() << " \nWork on " << *Inst << " Num_targets: " << NumVals + << "\n"); + for (uint32_t I = 0; I < MaxNumPromotions && I < NumVals; I++) { + uint64_t Count = ValueData[I].Count; + uint64_t Target = ValueData[I].Value; + DEBUG(dbgs() << " Candidate " << I << " Count=" << Count + << " Target_func: " << Target << "\n"); + + if (!isPromotionProfitable(Count, TotalCount)) { + DEBUG(dbgs() << " Not promote: Cold target.\n"); + break; + } + Function *TargetFunction = nullptr; + TargetStatus Status = isPromotionLegal(Inst, Target, TargetFunction); + if (Status != OK) { + StringRef TargetFuncName = Symtab->getFuncName(Target); + const char *Reason = StatusToString(Status); + DEBUG(dbgs() << " Not promote: " << Reason << "\n"); + Twine Msg = + Twine("Cannot promote indirect call to ") + + (TargetFuncName.empty() ? Twine(Target) : Twine(TargetFuncName)) + + Twine(" with count of ") + Twine(Count) + ": " + Reason; + emitOptimizationRemarkMissed(F.getContext(), "PGOIndirectCallPromotion", + F, Inst->getDebugLoc(), Msg); + break; + } + WorkList.push_back(std::make_pair(TargetFunction, Count)); + assert(Count <= TotalCount); + TotalCount -= Count; + ++NumPromotions; + } + return NumPromotions; +} + +// This furnction does the actual indireciton call promotion transformation: +// For an icall like: +// Ret = (*Foo)(Args); +// It transforms to: +// if (Foo == DirectCallee) +// Ret1 = DirectCallee(Args); +// else +// Ret2 = (*Foo)(Args); +// Ret = phi(Ret1, Ret2); +// It adds type casts for the args do not match the parameters and the return +// value. Branch weights meta data also updated. +// +bool ICallPromotionFunc::doVersion(CallInst *Inst, Function *DirectCallee, + uint64_t Count, uint64_t TotalCount) { + BasicBlock *BB = Inst->getParent(); + DEBUG(dbgs() << "\n\n== Basic Block Before ==\n"); + DEBUG(dbgs() << *BB << "\n"); + + assert(DirectCallee != nullptr); + CallSite CS(Inst); + Value *OrigCallee = CS.getCalledValue(); + + IRBuilder<> BBBuilder(Inst); + LLVMContext &Ctx = Inst->getContext(); + Value *BCI1 = + BBBuilder.CreateBitCast(OrigCallee, Type::getInt8PtrTy(Ctx), ""); + Value *BCI2 = + BBBuilder.CreateBitCast(DirectCallee, Type::getInt8PtrTy(Ctx), ""); + Value *PtrCmp = BBBuilder.CreateICmpEQ(BCI1, BCI2, ""); + + TerminatorInst *ThenTerm, *ElseTerm; + uint64_t ElseCount = TotalCount - Count; + uint64_t MaxCount = (Count >= ElseCount ? Count : ElseCount); + uint64_t Scale = calculateCountScale(MaxCount); + MDBuilder MDB(M->getContext()); + MDNode *BranchWeights = MDB.createBranchWeights( + scaleBranchCount(Count, Scale), scaleBranchCount(ElseCount, Scale)); + SplitBlockAndInsertIfThenElse(PtrCmp, Inst, &ThenTerm, &ElseTerm, + BranchWeights); + + BasicBlock *DirectCallBB = ThenTerm->getParent(); + DirectCallBB->setName("if.true"); + Instruction *NewInst = Inst->clone(); + ((CallInst *)NewInst)->setCalledFunction(DirectCallee); + ((CallInst *)NewInst)->mutateFunctionType(DirectCallee->getFunctionType()); + NewInst->setMetadata(LLVMContext::MD_prof, 0); + CallSite NewCS(NewInst); + DirectCallBB->getInstList().insert(DirectCallBB->getFirstInsertionPt(), + NewInst); + FunctionType *DirectCalleeType = DirectCallee->getFunctionType(); + unsigned ParaNum = DirectCalleeType->getFunctionNumParams(); + for (unsigned I = 0; I < ParaNum; ++I) { + Type *ATy = NewCS.getArgument(I)->getType(); + Type *PTy = DirectCalleeType->getParamType(I); + if (ATy != PTy) { + BitCastInst *BI = new BitCastInst(NewCS.getArgument(I), PTy, "", NewInst); + NewCS.setArgument(I, BI); + } + } + + BasicBlock *MergeBB = Inst->getParent(); + MergeBB->setName("if.end"); + BasicBlock *IndirectCallBB = ElseTerm->getParent(); + IndirectCallBB->setName("if.false"); + + // Create a phi to unify the return values of calls + if (!(Inst->getType()->isVoidTy())) { + PHINode *CallRetPhi = PHINode::Create(Inst->getType(), 0); + MergeBB->getInstList().push_front(CallRetPhi); + Inst->replaceAllUsesWith(CallRetPhi); + + // Handle the case where the return types differ. + Type *CallRetType = Inst->getType(); + Type *FuncRetType = DirectCallee->getReturnType(); + if (FuncRetType != CallRetType) { + BasicBlock::iterator It(NewInst); + Instruction *NextInst = &*(++It); + NewInst = new BitCastInst(NewInst, CallRetType, "", NextInst); + } + CallRetPhi->addIncoming(NewInst, DirectCallBB); + CallRetPhi->addIncoming(Inst, IndirectCallBB); + } + + Inst->removeFromParent(); + IndirectCallBB->getInstList().insert(IndirectCallBB->getFirstInsertionPt(), + Inst); + DEBUG(dbgs() << "\n== Basic Blocks After ==\n"); + DEBUG(dbgs() << *BB << *DirectCallBB << *IndirectCallBB << *MergeBB << "\n"); + + Twine Msg = + Twine("Promote indirect call to " + DirectCallee->getName() + + " with count " + Twine(Count) + " out of " + Twine(TotalCount)); + emitOptimizationRemark(F.getContext(), "PGOIndirectCallPromotion", F, + Inst->getDebugLoc(), Msg); + return true; +} + +// Promote indirectcall to conditional direct-call for one callsite. +uint32_t ICallPromotionFunc::doPromotion(CallInst *Inst, + const ICPWorkList &WorkList, + uint64_t &TotalCount) { + uint32_t Num = WorkList.size(); + if (Num == 0) + return 0; + uint32_t NumPromoted = 0; + for (uint32_t I = 0; I < Num; I++) { + uint64_t Count = WorkList[I].second; + if (!doVersion(Inst, WorkList[I].first, Count, TotalCount)) + break; + assert(TotalCount >= Count); + TotalCount -= Count; + NumPromoted++; + } + + NumOfPGOICallPromotion += NumPromoted; + return NumPromoted; +} + +// Traverse all the indirect-call callsite and get the value profile +// annotation to perform indirect-call promotion. +bool ICallPromotionFunc::performPromotion() { + PGOIndirectCallSiteVisitor ICV; + ICV.visit(F); + bool Changed = false; + for (auto &I : ICV.IndirectCallInsts) { + uint32_t NumVals; + uint64_t TotalCount; + bool Res = + getValueProfDataFromInst(*I, IPVK_IndirectCallTarget, MaxNumPromotions, + ValueDataArray.get(), NumVals, TotalCount); + if (!Res) + continue; + ICPWorkList WorkList; + uint32_t NumPromoted = 0; + if (analyzeCandidates(I, ValueDataArray.get(), NumVals, TotalCount, + WorkList) > 0) + NumPromoted = doPromotion(I, WorkList, TotalCount); + + if (NumPromoted == 0) + continue; + + Changed = true; + // Adjust the MD.prof metadata. First delete the old one. + I->setMetadata(LLVMContext::MD_prof, 0); + // If all promoted, we don't need the MD.prof metadat. + if (TotalCount == 0 || NumPromoted == NumVals) + continue; + // Otherwise we need update with the un-promoted records back. + annotateValueSite(*M, *I, &(ValueDataArray.get()[NumPromoted]), + NumVals - NumPromoted, TotalCount, + IPVK_IndirectCallTarget, MaxNumPromotions); + } + return Changed; +} + +bool PGOIndirectCallPromotion::runOnModule(Module &M) { + if (DisableValueProfiling) + return false; + + StringMap ProfNameMap; + // Build a profile-function-name to function map. + for (auto &F : M) { + if (F.isDeclaration()) + continue; + if (!F.hasName()) + continue; + if (InLTO) + ProfNameMap[getPGOFuncNameLTO(F)] = &F; + else + ProfNameMap[getPGOFuncName(F)] = &F; + } + + InstrProfSymtab Symtab; + Symtab.create(M, InLTO); + bool Changed = false; + for (auto &F : M) { + if (F.isDeclaration()) + continue; + ICallPromotionFunc ICallPromotion(F, &M, &Symtab, &ProfNameMap); + Changed |= ICallPromotion.performPromotion(); + } + + return Changed; +} Index: test/Transforms/PGOProfile/indirect_call_annotation.ll =================================================================== --- test/Transforms/PGOProfile/indirect_call_annotation.ll +++ test/Transforms/PGOProfile/indirect_call_annotation.ll @@ -1,5 +1,5 @@ ; RUN: llvm-profdata merge %S/Inputs/indirect_call.proftext -o %t.profdata -; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION +; RUN: opt < %s -pgo-instr-use -icp-max-prom=3 -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: test/Transforms/PGOProfile/indirect_call_promotion.ll =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/indirect_call_promotion.ll @@ -0,0 +1,48 @@ +; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@foo = common global i32 ()* null, align 8 + +define i32 @func1() { +entry: + ret i32 0 +} + +define i32 @func2() { +entry: + ret i32 1 +} + +define i32 @func3() { +entry: + ret i32 2 +} + +define i32 @func4() { +entry: + ret i32 3 +} + +define i32 @bar() { +entry: + %tmp = load i32 ()*, i32 ()** @foo, align 8 +; ICALL-PROM: [[BITCAST:%[0-9]+]] = bitcast i32 ()* %tmp to i8* +; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq i8* [[BITCAST]], bitcast (i32 ()* @func4 to i8*) +; ICALL-PROM: br i1 [[CMP]], label %if.true, label %if.false, !prof [[BRANCH_WEIGHT:![0-9]+]] +; ICALL-PROM: if.true: +; ICALL-PROM: [[DIRCALL_RET:%[0-9]+]] = call i32 @func4() +; ICALL-PROM: br label %if.end + %call = call i32 %tmp(), !prof !1 +; ICALL-PROM: if.false: +; ICALL-PROM: %call = call i32 %tmp(), !prof [[NEW_VP_METADATA:![0-9]+]] + ret i32 %call +; ICALL-PROM: if.end: +; ICALL-PROM: [[PHI_RET:%[0-9]+]] = phi i32 [ [[DIRCALL_RET]], %if.true ], [ %call, %if.false ] +; ICALL-PROM: ret i32 [[PHI_RET]] +} + +!1 = !{!"VP", i32 0, i64 1600, i64 7651369219802541373, i64 1030, i64 -4377547752858689819, i64 410, i64 -6929281286627296573, i64 150, i64 -2545542355363006406, i64 10} + +; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1030, i32 570} +; ICALL-PROM: [[NEW_VP_METADATA]] = !{!"VP", i32 0, i64 570, i64 -4377547752858689819, i64 410}