Index: llvm/docs/BranchWeightMetadata.rst =================================================================== --- llvm/docs/BranchWeightMetadata.rst +++ llvm/docs/BranchWeightMetadata.rst @@ -78,6 +78,27 @@ i32 } +``InvokeInst`` +^^^^^^^^^^^^^^^^^^ + +Invoke instruction may have branch weight metadata with one or two weights. +The second weight is optional and corresponds to the unwind branch. +If only one weight is set then it contains the execution count of the call +and used in SamplePGO mode only as described for the call instruction. If both +weights are specified then the second weight contains count of unwind branch +taken and the first weights contains the execution count of the call minus +the count of unwind branch taken. Both weights specified are used to calculate +BranchProbability as for BranchInst and for SamplePGO the sum of both weights +is used. + +.. code-block:: none + + !0 = metadata !{ + metadata !"branch_weights", + i32 + [ , i32 ] + } + Other ^^^^^ Index: llvm/include/llvm/IR/Instruction.h =================================================================== --- llvm/include/llvm/IR/Instruction.h +++ llvm/include/llvm/IR/Instruction.h @@ -332,8 +332,8 @@ /// Returns false if no metadata was found. bool extractProfTotalWeight(uint64_t &TotalVal) const; - /// Sets the branch_weights metadata to \p W for CallInst. - void setProfWeight(uint64_t W); + /// Copies the branch_weights metadata if any from source CallBase to this. + bool copyProfWeights(const Instruction &SrcCB); /// Set the debug location information for this instruction. void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); } Index: llvm/lib/Analysis/BranchProbabilityInfo.cpp =================================================================== --- llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -282,7 +282,8 @@ bool BranchProbabilityInfo::calcMetadataWeights(const BasicBlock *BB) { const Instruction *TI = BB->getTerminator(); assert(TI->getNumSuccessors() > 1 && "expected more than one successor!"); - if (!(isa(TI) || isa(TI) || isa(TI))) + if (!(isa(TI) || isa(TI) || isa(TI) || + isa(TI))) return false; MDNode *WeightsNode = TI->getMetadata(LLVMContext::MD_prof); Index: llvm/lib/IR/Instruction.cpp =================================================================== --- llvm/lib/IR/Instruction.cpp +++ llvm/lib/IR/Instruction.cpp @@ -757,11 +757,23 @@ return New; } -void Instruction::setProfWeight(uint64_t W) { +bool Instruction::copyProfWeights(const Instruction &SrcCB) { assert(isa(this) && "Can only set weights for call like instructions"); - SmallVector Weights; - Weights.push_back(W); - MDBuilder MDB(getContext()); - setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); + assert(isa(&SrcCB) && + "Can only set weights for call like instructions"); + auto *ProfileData = SrcCB.getMetadata(LLVMContext::MD_prof); + if (!ProfileData) + return false; + + auto *ProfDataName = dyn_cast(ProfileData->getOperand(0)); + if (!ProfDataName) + return false; + + if (!ProfDataName->getString().equals("branch_weights") && + !(ProfDataName->getString().equals("VP") && + ProfileData->getNumOperands() > 3)) + return false; + + setMetadata(LLVMContext::MD_prof, ProfileData); } Index: llvm/lib/IR/Verifier.cpp =================================================================== --- llvm/lib/IR/Verifier.cpp +++ llvm/lib/IR/Verifier.cpp @@ -4141,8 +4141,10 @@ ExpectedNumOperands = BI->getNumSuccessors(); else if (SwitchInst *SI = dyn_cast(&I)) ExpectedNumOperands = SI->getNumSuccessors(); - else if (isa(&I) || isa(&I)) + else if (isa(&I)) ExpectedNumOperands = 1; + else if (isa(&I)) + ; // ExpectedNumOperands will not be needed. else if (IndirectBrInst *IBI = dyn_cast(&I)) ExpectedNumOperands = IBI->getNumDestinations(); else if (isa(&I)) @@ -4151,8 +4153,12 @@ CheckFailed("!prof branch_weights are not allowed for this instruction", MD); - Assert(MD->getNumOperands() == 1 + ExpectedNumOperands, - "Wrong number of operands", MD); + if (isa(&I)) + Assert(MD->getNumOperands() == 2 || MD->getNumOperands() == 3, + "Wrong number of InvokeInst operands", MD); + else + Assert(MD->getNumOperands() == 1 + ExpectedNumOperands, + "Wrong number of operands", MD); for (unsigned i = 1; i < MD->getNumOperands(); ++i) { auto &MDO = MD->getOperand(i); Assert(MDO, "second operand should not be null", MD); Index: llvm/lib/Transforms/IPO/ArgumentPromotion.cpp =================================================================== --- llvm/lib/Transforms/IPO/ArgumentPromotion.cpp +++ llvm/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -337,9 +337,7 @@ AttributeList::get(F->getContext(), CallPAL.getFnAttributes(), CallPAL.getRetAttributes(), ArgAttrVec)); NewCS->setDebugLoc(CB.getDebugLoc()); - uint64_t W; - if (CB.extractProfTotalWeight(W)) - NewCS->setProfWeight(W); + NewCS->copyProfWeights(CB); Args.clear(); ArgAttrVec.clear(); Index: llvm/lib/Transforms/IPO/Attributor.cpp =================================================================== --- llvm/lib/Transforms/IPO/Attributor.cpp +++ llvm/lib/Transforms/IPO/Attributor.cpp @@ -1568,9 +1568,7 @@ } // Copy over various properties and the new attributes. - uint64_t W; - if (OldCB->extractProfTotalWeight(W)) - NewCB->setProfWeight(W); + NewCB->copyProfWeights(*OldCB); NewCB->setCallingConv(OldCB->getCallingConv()); NewCB->setDebugLoc(OldCB->getDebugLoc()); NewCB->takeName(OldCB); Index: llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp =================================================================== --- llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp +++ llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp @@ -206,9 +206,7 @@ NewCB->setCallingConv(CB->getCallingConv()); NewCB->setAttributes(PAL); NewCB->setDebugLoc(CB->getDebugLoc()); - uint64_t W; - if (CB->extractProfTotalWeight(W)) - NewCB->setProfWeight(W); + NewCB->copyProfWeights(*CB); Args.clear(); @@ -937,9 +935,7 @@ NewCB->setCallingConv(CB.getCallingConv()); NewCB->setAttributes(NewCallPAL); NewCB->setDebugLoc(CB.getDebugLoc()); - uint64_t W; - if (CB.extractProfTotalWeight(W)) - NewCB->setProfWeight(W); + NewCB->copyProfWeights(CB); Args.clear(); ArgAttrVec.clear(); Index: llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp =================================================================== --- llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -4896,9 +4896,7 @@ // Preserve the weight metadata for the new call instruction. The metadata // is used by SamplePGO to check callsite's hotness. - uint64_t W; - if (Caller->extractProfTotalWeight(W)) - NewCall->setProfWeight(W); + NewCall->copyProfWeights(*Caller); // Insert a cast of the return type as necessary. Instruction *NC = NewCall; Index: llvm/test/Analysis/BranchProbabilityInfo/basic.ll =================================================================== --- llvm/test/Analysis/BranchProbabilityInfo/basic.ll +++ llvm/test/Analysis/BranchProbabilityInfo/basic.ll @@ -317,6 +317,35 @@ ret i32 0 } +; CHECK-LABEL: test_invoke_code_profiled +define void @test_invoke_code_profiled(i1 %c) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: +; CHECK: edge entry -> invoke.to0 probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge entry -> lpad probability is 0x00000800 / 0x80000000 = 0.00% + invoke i32 @InvokeCall() to label %invoke.to0 unwind label %lpad + +invoke.to0: +; CHECK: edge invoke.to0 -> invoke.to1 probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge invoke.to0 -> lpad probability is 0x00000800 / 0x80000000 = 0.00% + invoke i32 @InvokeCall() to label %invoke.to1 unwind label %lpad, + !prof !{!"branch_weights", i32 444} + +invoke.to1: +; CHECK: invoke.to1 -> invoke.to2 probability is 0x55555555 / 0x80000000 = 66.67% +; CHECK: invoke.to1 -> lpad probability is 0x2aaaaaab / 0x80000000 = 33.33% + invoke i32 @InvokeCall() to label %invoke.to2 unwind label %lpad, + !prof !{!"branch_weights", i32 222, i32 111} + ret void + +invoke.to2: + ret void + +lpad: + %ll = landingpad { i8*, i32 } + cleanup + ret void +} + declare i32 @__gxx_personality_v0(...) declare void @ColdFunc() declare i32 @InvokeCall() Index: llvm/test/Transforms/Inline/inline-hot-callsite.ll =================================================================== --- llvm/test/Transforms/Inline/inline-hot-callsite.ll +++ llvm/test/Transforms/Inline/inline-hot-callsite.ll @@ -39,6 +39,66 @@ ret i32 %y3 } +declare i32 @__gxx_personality_v0(...) + +define i32 @invoker2(i32 %y1) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +; CHECK-LABEL: @invoker2( +; CHECK: invoke i32 @callee2 +; CHECK-NOT: invoke i32 @callee1 +; CHECK: ret i32 + %y2 = invoke i32 @callee2(i32 %y1) to label %next unwind label %lpad, !prof !22 + +next: + %y3 = invoke i32 @callee1(i32 %y2) to label %exit unwind label %lpad, !prof !21 + +exit: + ret i32 1 + +lpad: + %ll = landingpad { i8*, i32 } cleanup + ret i32 1 +} + +define i32 @invoker3(i32 %y1) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +; CHECK-LABEL: @invoker3( +; CHECK: invoke i32 @callee2 +; CHECK-NOT: invoke i32 @callee1 +; CHECK: ret i32 + %y2 = invoke i32 @callee2(i32 %y1) to label %next unwind label %lpad, + !prof !{!"branch_weights", i64 1, i64 0} + +next: + %y3 = invoke i32 @callee1(i32 %y2) to label %exit unwind label %lpad, + !prof !{!"branch_weights", i64 300, i64 1} + +exit: + ret i32 1 + +lpad: + %ll = landingpad { i8*, i32 } cleanup + ret i32 1 +} + +define i32 @invoker4(i32 %y1) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +; CHECK-LABEL: @invoker4( +; CHECK: invoke i32 @callee2 +; CHECK-NOT: invoke i32 @callee1 +; CHECK: ret i32 + %y2 = invoke i32 @callee2(i32 %y1) to label %next unwind label %lpad, + !prof !{!"branch_weights", i64 1, i64 0} + +next: + %y3 = invoke i32 @callee1(i32 %y2) to label %exit unwind label %lpad, + !prof !{!"branch_weights", i64 0, i64 300} + +exit: + ret i32 1 + +lpad: + %ll = landingpad { i8*, i32 } cleanup + ret i32 1 +} + declare void @extern() !llvm.module.flags = !{!1} Index: llvm/test/Transforms/InstCombine/cast-call-combine-prof.ll =================================================================== --- llvm/test/Transforms/InstCombine/cast-call-combine-prof.ll +++ llvm/test/Transforms/InstCombine/cast-call-combine-prof.ll @@ -32,7 +32,7 @@ unreachable } -; CHECK: ![[$PROF]] = !{!"branch_weights", i32 2000} +; CHECK: ![[$PROF]] = !{!"VP", i32 0, i64 2000, i64 -3913987384944532146, i64 2000} !0 = !{!"VP", i32 0, i64 2000, i64 -3913987384944532146, i64 2000} !llvm.module.flags = !{!1}