Index: include/llvm/Transforms/Utils/CallPromotionUtils.h =================================================================== --- include/llvm/Transforms/Utils/CallPromotionUtils.h +++ include/llvm/Transforms/Utils/CallPromotionUtils.h @@ -31,6 +31,23 @@ bool isLegalToPromote(CallSite CS, Function *Callee, const char **FailureReason = nullptr); +/// Promote the given indirect call site to unconditionally call \p Callee. +/// +/// If the function type of the call site doesn't match that of the callee, +/// bitcast instructions are inserted where appropriate. If \p Casts is +/// non-null, these bitcasts are collected in the provided container. +void promoteCall(CallSite CS, Function *Callee, + SmallVectorImpl *Casts = nullptr); + +/// Demote the given direct call site to call \p CalledValue. +/// +/// This function is meant to be used in conjunction with \p promoteCall. As +/// such, any bitcast instructions contained in \p Casts, which were created by +/// \p promoteCall, will be erased. Together, these functions can be used to +/// temporarily promote a call to facilitate analyzing the program as if the +/// call were direct. +void demoteCall(CallSite CS, Value *CalledValue, ArrayRef Casts); + /// Promote the given indirect call site to conditionally call \p Callee. /// /// This function creates an if-then-else structure at the location of the call Index: lib/Transforms/IPO/SampleProfile.cpp =================================================================== --- lib/Transforms/IPO/SampleProfile.cpp +++ lib/Transforms/IPO/SampleProfile.cpp @@ -826,8 +826,8 @@ R->getValue()->getSubprogram() && isLegalToPromote(CallSite(I), R->getValue(), &Reason)) { uint64_t C = FS->getEntrySamples(); - Instruction *DI = - pgo::promoteIndirectCall(I, R->getValue(), C, Sum, false, ORE); + Instruction *DI = I; + I = pgo::promoteIndirectCall(I, R->getValue(), C, Sum, false, ORE); Sum -= C; PromotedInsns.insert(I); // If profile mismatches, we should not attempt to inline DI. Index: lib/Transforms/Instrumentation/IndirectCallPromotion.cpp =================================================================== --- lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -199,7 +199,7 @@ // Promote a list of targets for one indirect-call callsite. Return // the number of promotions. - uint32_t tryToPromote(Instruction *Inst, + uint32_t tryToPromote(Instruction *&Inst, const std::vector &Candidates, uint64_t &TotalCount); @@ -312,8 +312,8 @@ if (AttachProfToDirectCall) { SmallVector Weights; Weights.push_back(Count); - MDBuilder MDB(NewInst->getContext()); - NewInst->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); + MDBuilder MDB(Inst->getContext()); + Inst->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); } using namespace ore; @@ -330,14 +330,14 @@ // Promote indirect-call to conditional direct-call for one callsite. uint32_t ICallPromotionFunc::tryToPromote( - Instruction *Inst, const std::vector &Candidates, + Instruction *&Inst, const std::vector &Candidates, uint64_t &TotalCount) { uint32_t NumPromoted = 0; for (auto &C : Candidates) { uint64_t Count = C.Count; - pgo::promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount, - SamplePGO, &ORE); + Inst = pgo::promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount, + SamplePGO, &ORE); assert(TotalCount >= Count); TotalCount -= Count; NumOfPGOICallPromotion++; Index: lib/Transforms/Utils/CallPromotionUtils.cpp =================================================================== --- lib/Transforms/Utils/CallPromotionUtils.cpp +++ lib/Transforms/Utils/CallPromotionUtils.cpp @@ -25,10 +25,9 @@ /// Fix-up phi nodes in an invoke instruction's normal destination. /// /// After versioning an invoke instruction, values coming from the original -/// block will now either be coming from the original block or the "else" block. +/// block will now be coming from the "merge" block. static void fixupPHINodeForNormalDest(InvokeInst *Invoke, BasicBlock *OrigBlock, - BasicBlock *ElseBlock, - Instruction *NewInst) { + BasicBlock *MergeBlock) { for (auto &I : *Invoke->getNormalDest()) { auto *Phi = dyn_cast(&I); if (!Phi) @@ -36,8 +35,7 @@ int Idx = Phi->getBasicBlockIndex(OrigBlock); if (Idx == -1) continue; - Phi->setIncomingBlock(Idx, ElseBlock); - Phi->addIncoming(NewInst, OrigBlock); + Phi->setIncomingBlock(Idx, MergeBlock); } } @@ -61,52 +59,27 @@ } } -/// Get the phi node having the returned value of a call or invoke instruction -/// as it's operand. -static bool getRetPhiNode(Instruction *Inst, BasicBlock *Block) { - BasicBlock *FromBlock = Inst->getParent(); - for (auto &I : *Block) { - PHINode *PHI = dyn_cast(&I); - if (!PHI) - break; - int Idx = PHI->getBasicBlockIndex(FromBlock); - if (Idx == -1) - continue; - auto *V = PHI->getIncomingValue(Idx); - if (V == Inst) - return true; - } - return false; -} - /// Create a phi node for the returned value of a call or invoke instruction. /// /// After versioning a call or invoke instruction that returns a value, we have /// to merge the value of the original and new instructions. We do this by /// creating a phi node and replacing uses of the original instruction with this /// phi node. -static void createRetPHINode(Instruction *OrigInst, Instruction *NewInst) { +static void createRetPHINode(Instruction *OrigInst, Instruction *NewInst, + BasicBlock *MergeBlock, IRBuilder<> &Builder) { if (OrigInst->getType()->isVoidTy() || OrigInst->use_empty()) return; - BasicBlock *RetValBB = NewInst->getParent(); - if (auto *Invoke = dyn_cast(NewInst)) - RetValBB = Invoke->getNormalDest(); - BasicBlock *PhiBB = RetValBB->getSingleSuccessor(); - - if (getRetPhiNode(OrigInst, PhiBB)) - return; - - IRBuilder<> Builder(&PhiBB->front()); + Builder.SetInsertPoint(&MergeBlock->front()); PHINode *Phi = Builder.CreatePHI(OrigInst->getType(), 0); SmallVector UsersToUpdate; for (User *U : OrigInst->users()) UsersToUpdate.push_back(U); for (User *U : UsersToUpdate) U->replaceUsesOfWith(OrigInst, Phi); + Phi->addIncoming(NewInst, NewInst->getParent()); Phi->addIncoming(OrigInst, OrigInst->getParent()); - Phi->addIncoming(NewInst, RetValBB); } /// Cast a call or invoke instruction to the given type. @@ -115,7 +88,8 @@ /// that of the callee. If this is the case, we have to cast the returned value /// to the correct type. The location of the cast depends of if we have a call /// or invoke instruction. -Instruction *createRetBitCast(CallSite CS, Type *RetTy) { +static void createRetBitCast(CallSite CS, Type *RetTy, + SmallVectorImpl *Casts) { // Save the users of the calling instruction. These uses will be changed to // use the bitcast after we create it. @@ -127,19 +101,20 @@ // value. The location depends on if we have a call or invoke instruction. Instruction *InsertBefore = nullptr; if (auto *Invoke = dyn_cast(CS.getInstruction())) - InsertBefore = &*Invoke->getNormalDest()->getFirstInsertionPt(); + InsertBefore = + &SplitEdge(Invoke->getParent(), Invoke->getNormalDest())->front(); else InsertBefore = &*std::next(CS.getInstruction()->getIterator()); // Bitcast the return value to the correct type. auto *Cast = CastInst::Create(Instruction::BitCast, CS.getInstruction(), RetTy, "", InsertBefore); + if (Casts) + Casts->push_back(Cast); // Replace all the original uses of the calling instruction with the bitcast. for (User *U : UsersToUpdate) U->replaceUsesOfWith(CS.getInstruction(), Cast); - - return Cast; } /// Predicate and clone the given call site. @@ -155,17 +130,17 @@ IRBuilder<> Builder(CS.getInstruction()); Instruction *OrigInst = CS.getInstruction(); + BasicBlock *OrigBlock = OrigInst->getParent(); // Create the compare. The called value and callee must have the same type to // be compared. - auto *LHS = - Builder.CreateBitCast(CS.getCalledValue(), Builder.getInt8PtrTy()); - auto *RHS = Builder.CreateBitCast(Callee, Builder.getInt8PtrTy()); - auto *Cond = Builder.CreateICmpEQ(LHS, RHS); + if (CS.getCalledValue()->getType() != Callee->getType()) + Callee = Builder.CreateBitCast(Callee, CS.getCalledValue()->getType()); + auto *Cond = Builder.CreateICmpEQ(CS.getCalledValue(), Callee); // Create an if-then-else structure. The original instruction is moved into - // the "else" block, and a clone of the original instruction is placed in the - // "then" block. + // the "then" block, and a clone of the original instruction is placed in the + // "else" block. TerminatorInst *ThenTerm = nullptr; TerminatorInst *ElseTerm = nullptr; SplitBlockAndInsertIfThenElse(Cond, CS.getInstruction(), &ThenTerm, &ElseTerm, @@ -179,8 +154,8 @@ MergeBlock->setName("if.end.icp"); Instruction *NewInst = OrigInst->clone(); - OrigInst->moveBefore(ElseTerm); - NewInst->insertBefore(ThenTerm); + OrigInst->moveBefore(ThenTerm); + NewInst->insertBefore(ElseTerm); // If the original call site is an invoke instruction, we have extra work to // do since invoke instructions are terminating. We have to fix-up phi nodes @@ -198,14 +173,18 @@ Builder.CreateBr(OrigInvoke->getNormalDest()); // Fix-up phi nodes in the original invoke's normal and unwind destinations. - fixupPHINodeForNormalDest(OrigInvoke, MergeBlock, ElseBlock, NewInst); - fixupPHINodeForUnwindDest(OrigInvoke, MergeBlock, ThenBlock, ElseBlock); + fixupPHINodeForNormalDest(OrigInvoke, OrigBlock, MergeBlock); + fixupPHINodeForUnwindDest(OrigInvoke, OrigBlock, ThenBlock, ElseBlock); - // Now set the normal destination of new the invoke instruction to be the + // Now set the normal destinations of the invoke instructions to be the // "merge" block. + OrigInvoke->setNormalDest(MergeBlock); NewInvoke->setNormalDest(MergeBlock); } + // Create a phi node for the returned value of the call site. + createRetPHINode(OrigInst, NewInst, MergeBlock, Builder); + return NewInst; } @@ -253,7 +232,8 @@ return true; } -static void promoteCall(CallSite CS, Function *Callee, Instruction *&Cast) { +void llvm::promoteCall(CallSite CS, Function *Callee, + SmallVectorImpl *Casts) { assert(!CS.getCalledFunction() && "Only indirect call sites can be promoted"); // Set the called function of the call site to be the given callee. @@ -288,31 +268,48 @@ auto *Cast = CastInst::Create(Instruction::BitCast, U.get(), FormalTy, "", CS.getInstruction()); CS.setArgument(ArgNo, Cast); + if (Casts) + Casts->push_back(Cast); } } // If the return type of the call site doesn't match that of the callee, cast // the returned value to the appropriate type. if (!CallSiteRetTy->isVoidTy() && CallSiteRetTy != CalleeRetTy) - Cast = createRetBitCast(CS, CallSiteRetTy); + createRetBitCast(CS, CallSiteRetTy, Casts); +} + +void llvm::demoteCall(CallSite CS, Value *CalledValue, + ArrayRef Casts) { + assert(CS.getCalledFunction() && "Only direct call sites can be demoted"); + + // For each cast instruction, replace all of its uses with its source + // operand. + for (CastInst *Cast : Casts) { + while (!Cast->user_empty()) + Cast->user_back()->replaceUsesOfWith(Cast, Cast->getOperand(0)); + Cast->eraseFromParent(); + } + + // Set the called value of the call site, and mutate its function type. + CS.setCalledFunction(CalledValue); + CS.mutateFunctionType(cast( + cast(CalledValue->getType())->getElementType())); } Instruction *llvm::promoteCallWithIfThenElse(CallSite CS, Function *Callee, MDNode *BranchWeights) { // Version the indirect call site. If the called value is equal to the given - // callee, 'NewInst' will be executed, otherwise the original call site will + // callee, the original call site will be executed, otherwise 'NewInst' will // be executed. Instruction *NewInst = versionCallSite(CS, Callee, BranchWeights); - // Promote 'NewInst' so that it directly calls the desired function. - Instruction *Cast = nullptr; - promoteCall(CallSite(NewInst), Callee, Cast); - - // Create a phi node for the returned value of the call site. - createRetPHINode(CS.getInstruction(), Cast ? Cast : NewInst); + // Promote the original call site so that it directly calls the desired + // function. + promoteCall(CS, Callee); - // Return the new direct call. + // Return the new indirect call. return NewInst; } Index: test/Transforms/PGOProfile/icp_covariant_call_return.ll =================================================================== --- test/Transforms/PGOProfile/icp_covariant_call_return.ll +++ test/Transforms/PGOProfile/icp_covariant_call_return.ll @@ -22,19 +22,18 @@ %vtable = load %struct.Base* (%struct.B*)**, %struct.Base* (%struct.B*)*** %tmp2, align 8 %vfn = getelementptr inbounds %struct.Base* (%struct.B*)*, %struct.Base* (%struct.B*)** %vtable, i64 0 %tmp3 = load %struct.Base* (%struct.B*)*, %struct.Base* (%struct.B*)** %vfn, align 8 -; ICALL-PROM: [[BITCAST:%[0-9]+]] = bitcast %struct.Base* (%struct.B*)* %tmp3 to i8* -; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq i8* [[BITCAST]], bitcast (%struct.Derived* (%struct.D*)* @_ZN1D4funcEv to i8*) +; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq %struct.Base* (%struct.B*)* %tmp3, bitcast (%struct.Derived* (%struct.D*)* @_ZN1D4funcEv to %struct.Base* (%struct.B*)*) ; ICALL-PROM: br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]] ; ICALL-PROM:if.true.direct_targ: ; ICALL-PROM: [[ARG_BITCAST:%[0-9]+]] = bitcast %struct.B* %tmp1 to %struct.D* -; ICALL-PROM: [[DIRCALL_RET:%[0-9]+]] = call %struct.Derived* @_ZN1D4funcEv(%struct.D* [[ARG_BITCAST]]) +; ICALL-PROM: [[DIRCALL_RET:%.+]] = call %struct.Derived* @_ZN1D4funcEv(%struct.D* [[ARG_BITCAST]]) ; ICALL-PROM: [[DIRCALL_RET_CAST:%[0-9]+]] = bitcast %struct.Derived* [[DIRCALL_RET]] to %struct.Base* ; ICALL-PROM: br label %if.end.icp ; ICALL-PROM:if.false.orig_indirect: -; ICALL-PROM: %call1 = call %struct.Base* %tmp3(%struct.B* %tmp1) +; ICALL-PROM: [[CALL1:%.+]] = call %struct.Base* %tmp3(%struct.B* %tmp1) ; ICALL-PROM: br label %if.end.icp ; ICALL-PROM:if.end.icp: -; ICALL-PROM: [[PHI_RET:%[0-9]+]] = phi %struct.Base* [ %call1, %if.false.orig_indirect ], [ [[DIRCALL_RET_CAST]], %if.true.direct_targ ] +; ICALL-PROM: [[PHI_RET:%[0-9]+]] = phi %struct.Base* [ [[CALL1]], %if.false.orig_indirect ], [ [[DIRCALL_RET_CAST]], %if.true.direct_targ ] %call1 = call %struct.Base* %tmp3(%struct.B* %tmp1), !prof !1 ret %struct.Base* %call1 } Index: test/Transforms/PGOProfile/icp_covariant_invoke_return.ll =================================================================== --- test/Transforms/PGOProfile/icp_covariant_invoke_return.ll +++ test/Transforms/PGOProfile/icp_covariant_invoke_return.ll @@ -32,18 +32,19 @@ %vtable = load %struct.Base* (%struct.B*)**, %struct.Base* (%struct.B*)*** %tmp2, align 8 %vfn = getelementptr inbounds %struct.Base* (%struct.B*)*, %struct.Base* (%struct.B*)** %vtable, i64 0 %tmp3 = load %struct.Base* (%struct.B*)*, %struct.Base* (%struct.B*)** %vfn, align 8 -; ICALL-PROM: [[BITCAST:%[0-9]+]] = bitcast %struct.Base* (%struct.B*)* %tmp3 to i8* -; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq i8* [[BITCAST]], bitcast (%struct.Derived* (%struct.D*)* @_ZN1D4funcEv to i8*) +; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq %struct.Base* (%struct.B*)* %tmp3, bitcast (%struct.Derived* (%struct.D*)* @_ZN1D4funcEv to %struct.Base* (%struct.B*)*) ; ICALL-PROM: br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]] ; ICALL-PROM:if.true.direct_targ: ; ICALL-PROM: [[ARG_BITCAST:%[0-9]+]] = bitcast %struct.B* %tmp1 to %struct.D* -; ICALL-PROM: [[DIRCALL_RET:%[0-9]+]] = invoke %struct.Derived* @_ZN1D4funcEv(%struct.D* [[ARG_BITCAST]]) -; ICALL-PROM: to label %if.end.icp unwind label %lpad +; ICALL-PROM: [[DIRCALL_RET:%.+]] = invoke %struct.Derived* @_ZN1D4funcEv(%struct.D* [[ARG_BITCAST]]) +; ICALL-PROM: to label %if.true.direct_targ.if.end.icp_crit_edge unwind label %lpad +; ICALL-PROM:if.true.direct_targ.if.end.icp_crit_edge: +; ICALL-PROM: [[DIRCALL_RET_CAST:%[0-9]+]] = bitcast %struct.Derived* %call2 to %struct.Base* +; ICALL-PROM: br label %if.end.icp ; ICALL-PROM:if.false.orig_indirect: ; ICAll-PROM: %call2 = invoke %struct.Base* %tmp3(%struct.B* %tmp1) ; ICAll-PROM: to label %invoke.cont1 unwind label %lpad ; ICALL-PROM:if.end.icp: -; ICALL-PROM: [[DIRCALL_RET_CAST:%[0-9]+]] = bitcast %struct.Derived* [[DIRCALL_RET]] to %struct.Base* ; ICALL-PROM: br label %invoke.cont1 %call2 = invoke %struct.Base* %tmp3(%struct.B* %tmp1) to label %invoke.cont1 unwind label %lpad, !prof !1 Index: test/Transforms/PGOProfile/icp_invoke.ll =================================================================== --- test/Transforms/PGOProfile/icp_invoke.ll +++ test/Transforms/PGOProfile/icp_invoke.ll @@ -20,8 +20,7 @@ define i32 @_Z3goov() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { entry: %tmp = load void ()*, void ()** @foo1, align 8 -; ICP: [[BITCAST_IC1:%[0-9]+]] = bitcast void ()* %tmp to i8* -; ICP: [[CMP_IC1:%[0-9]+]] = icmp eq i8* [[BITCAST_IC1]], bitcast (void ()* @_ZL4bar1v to i8*) +; ICP: [[CMP_IC1:%[0-9]+]] = icmp eq void ()* %tmp, @_ZL4bar1v ; ICP: br i1 [[CMP_IC1]], label %[[TRUE_LABEL_IC1:.*]], label %[[FALSE_LABEL_IC1:.*]], !prof [[BRANCH_WEIGHT:![0-9]+]] ; ICP:[[TRUE_LABEL_IC1]]: ; ICP: invoke void @_ZL4bar1v() @@ -49,17 +48,19 @@ try.cont: %tmp6 = load i32 ()*, i32 ()** @foo2, align 8 -; ICP: [[BITCAST_IC2:%[0-9]+]] = bitcast i32 ()* %tmp6 to i8* -; ICP: [[CMP_IC2:%[0-9]+]] = icmp eq i8* [[BITCAST_IC2]], bitcast (i32 ()* @_ZL4bar2v to i8*) +; ICP: [[CMP_IC2:%[0-9]+]] = icmp eq i32 ()* %tmp6, @_ZL4bar2v ; ICP: br i1 [[CMP_IC2]], label %[[TRUE_LABEL_IC2:.*]], label %[[FALSE_LABEL_IC2:.*]], !prof [[BRANCH_WEIGHT:![0-9]+]] ; ICP:[[TRUE_LABEL_IC2]]: -; ICP: [[RESULT_IC2:%[0-9]+]] = invoke i32 @_ZL4bar2v() +; ICP: [[RESULT_IC2_0:%.+]] = invoke i32 @_ZL4bar2v() ; ICP: to label %[[DCALL_NORMAL_DEST_IC2:.*]] unwind label %lpad1 ; ICP:[[FALSE_LABEL_IC2]]: +; ICP: [[RESULT_IC2_1:%.+]] = invoke i32 %tmp6() +; ICP: to label %[[DCALL_NORMAL_DEST_IC2:.*]] unwind label %lpad1 %call = invoke i32 %tmp6() to label %try.cont8 unwind label %lpad1, !prof !3 ; ICP:[[DCALL_NORMAL_DEST_IC2]]: +; ICP: [[PHI_0:%.+]] = phi i32 [ [[RESULT_IC2_1]], %[[FALSE_LABEL_IC2]] ], [ [[RESULT_IC2_0]], %[[TRUE_LABEL_IC2]] ] ; ICP: br label %try.cont8 lpad1: %tmp7 = landingpad { i8*, i32 } @@ -77,7 +78,7 @@ try.cont8: %i.0 = phi i32 [ undef, %catch6 ], [ %call, %try.cont ] -; ICP: %i.0 = phi i32 [ undef, %catch6 ], [ %call, %[[FALSE_LABEL_IC2]] ], [ [[RESULT_IC2]], %[[DCALL_NORMAL_DEST_IC2]] ] +; ICP: %i.0 = phi i32 [ undef, %catch6 ], [ [[PHI_0]], %[[DCALL_NORMAL_DEST_IC2]] ] ret i32 %i.0 eh.resume: Index: test/Transforms/PGOProfile/icp_invoke_nouse.ll =================================================================== --- test/Transforms/PGOProfile/icp_invoke_nouse.ll +++ test/Transforms/PGOProfile/icp_invoke_nouse.ll @@ -18,8 +18,7 @@ if.end: ; preds = %entry %fptr = load i32 ()*, i32 ()** @pfptr, align 8 -; ICP: [[BITCAST_IC1:%[0-9]+]] = bitcast i32 ()* %fptr to i8* -; ICP: [[CMP_IC1:%[0-9]+]] = icmp eq i8* [[BITCAST_IC1]], bitcast (i32 ()* @_ZL4bar1v to i8*) +; ICP: [[CMP_IC1:%[0-9]+]] = icmp eq i32 ()* %fptr, @_ZL4bar1v ; ICP: br i1 [[CMP_IC1]], label %[[TRUE_LABEL_IC1:.*]], label %[[FALSE_LABEL_IC1:.*]], !prof [[BRANCH_WEIGHT:![0-9]+]] ; ICP:[[TRUE_LABEL_IC1]]: ; ICP: invoke i32 @_ZL4bar1v() Index: test/Transforms/PGOProfile/icp_vararg.ll =================================================================== --- test/Transforms/PGOProfile/icp_vararg.ll +++ test/Transforms/PGOProfile/icp_vararg.ll @@ -13,19 +13,18 @@ define i32 @bar() #1 { entry: %tmp = load i32 (i32, ...)*, i32 (i32, ...)** @foo, align 8 -; ICALL-PROM: [[BITCAST:%[0-9]+]] = bitcast i32 (i32, ...)* %tmp to i8* -; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq i8* [[BITCAST]], bitcast (i32 (i32, ...)* @va_func to i8*) +; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq i32 (i32, ...)* %tmp, @va_func ; ICALL-PROM: br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]] ; ICALL-PROM:if.true.direct_targ: -; ICALL-PROM: [[DIRCALL_RET:%[0-9]+]] = call i32 (i32, ...) @va_func(i32 3, i32 12, i32 22, i32 4) +; ICALL-PROM: [[DIRCALL_RET:%.+]] = call i32 (i32, ...) @va_func(i32 3, i32 12, i32 22, i32 4) ; ICALL-PROM: br label %if.end.icp %call = call i32 (i32, ...) %tmp(i32 3, i32 12, i32 22, i32 4), !prof !1 ; ICALL-PROM:if.false.orig_indirect: -; ICALL-PROM: %call = call i32 (i32, ...) %tmp(i32 3, i32 12, i32 22, i32 4) +; ICALL-PROM: [[CALL:%.+]] = call i32 (i32, ...) %tmp(i32 3, i32 12, i32 22, i32 4) ; ICALL-PROM: br label %if.end.icp ret i32 %call ; ICALL-PROM:if.end.icp: -; ICALL-PROM: [[PHI_RET:%[0-9]+]] = phi i32 [ %call, %if.false.orig_indirect ], [ [[DIRCALL_RET]], %if.true.direct_targ ] +; ICALL-PROM: [[PHI_RET:%[0-9]+]] = phi i32 [ [[CALL]], %if.false.orig_indirect ], [ [[DIRCALL_RET]], %if.true.direct_targ ] ; ICALL-PROM: ret i32 [[PHI_RET]] } Index: test/Transforms/PGOProfile/indirect_call_promotion.ll =================================================================== --- test/Transforms/PGOProfile/indirect_call_promotion.ll +++ test/Transforms/PGOProfile/indirect_call_promotion.ll @@ -43,19 +43,18 @@ define i32 @bar() { entry: %tmp = load i32 ()*, i32 ()** @foo, align 8 -; ICALL-PROM: [[BITCAST:%[0-9]+]] = bitcast i32 ()* %tmp to i8* -; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq i8* [[BITCAST]], bitcast (i32 ()* @func4 to i8*) +; ICALL-PROM: [[CMP:%[0-9]+]] = icmp eq i32 ()* %tmp, @func4 ; ICALL-PROM: br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]] ; ICALL-PROM: if.true.direct_targ: -; ICALL-PROM: [[DIRCALL_RET:%[0-9]+]] = call i32 @func4() +; ICALL-PROM: [[DIRCALL_RET:%.+]] = call i32 @func4() ; ICALL-PROM-SAMPLEPGO: call i32 @func4(), !prof [[CALL_METADATA:![0-9]+]] ; ICALL-PROM: br label %if.end.icp %call = call i32 %tmp(), !prof !1 ; ICALL-PROM: if.false.orig_indirect: -; ICALL-PROM: %call = call i32 %tmp(), !prof [[NEW_VP_METADATA:![0-9]+]] +; ICALL-PROM: [[CALL:%.+]] = call i32 %tmp(), !prof [[NEW_VP_METADATA:![0-9]+]] ret i32 %call ; ICALL-PROM: if.end.icp: -; ICALL-PROM: [[PHI_RET:%[0-9]+]] = phi i32 [ %call, %if.false.orig_indirect ], [ [[DIRCALL_RET]], %if.true.direct_targ ] +; ICALL-PROM: [[PHI_RET:%[0-9]+]] = phi i32 [ [[CALL]], %if.false.orig_indirect ], [ [[DIRCALL_RET]], %if.true.direct_targ ] ; ICALL-PROM: ret i32 [[PHI_RET]] }