diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -2736,7 +2736,33 @@ PN->blocks(), std::back_inserter(IfBlocks), [](BasicBlock *IfBlock) { return cast(IfBlock->getTerminator())->isUnconditional(); }); - assert(!IfBlocks.empty() && "Will have at least one block to speculate."); + assert((IfBlocks.size() == 1 || IfBlocks.size() == 2) && + "Will have either one or two blocks to speculate."); + + // If the branch is non-unpredictable, see if we either predictably jump to + // the merge bb (if we have only a single 'then' block), or if we predictably + // jump to one specific 'then' block (if we have two of them). + // It isn't beneficial to speculatively execute the code + // from the block that we know is predictably not entered. + if (!DomBI->getMetadata(LLVMContext::MD_unpredictable)) { + uint64_t TWeight, FWeight; + if (DomBI->extractProfMetadata(TWeight, FWeight) && + (TWeight + FWeight) != 0) { + BranchProbability BITrueProb = + BranchProbability::getBranchProbability(TWeight, TWeight + FWeight); + BranchProbability Likely = TTI.getPredictableBranchThreshold(); + BranchProbability BIFalseProb = BITrueProb.getCompl(); + if (IfBlocks.size() == 1) { + BranchProbability BIBBProb = + DomBI->getSuccessor(0) == BB ? BITrueProb : BIFalseProb; + if (BIBBProb >= Likely) + return false; + } else { + if (BITrueProb >= Likely || BIFalseProb >= Likely) + return false; + } + } + } // Don't try to fold an unreachable block. For example, the phi node itself // can't be the candidate if-condition for a select that we want to form. diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll --- a/llvm/test/Transforms/PGOProfile/chr.ll +++ b/llvm/test/Transforms/PGOProfile/chr.ll @@ -2013,7 +2013,8 @@ ; CHECK-NEXT: [[C1:%.*]] = icmp slt i64 [[V2]], 100 ; CHECK-NEXT: br i1 [[C1]], label [[BB0_SPLIT:%.*]], label [[BB0_SPLIT_NONCHR:%.*]], !prof [[PROF15]] ; CHECK: common.ret: -; CHECK-NEXT: ret i64 99 +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i64 [ 99, [[BB0_SPLIT]] ], [ 99, [[BB0_SPLIT_NONCHR]] ] +; CHECK-NEXT: ret i64 [[COMMON_RET_OP]] ; CHECK: bb0.split: ; CHECK-NEXT: [[V299:%.*]] = mul i64 [[V2]], 7860086430977039991 ; CHECK-NEXT: store i64 [[V299]], i64* [[J:%.*]], align 4 diff --git a/llvm/test/Transforms/SimplifyCFG/fold-two-entry-phi-node-with-one-block-profmd.ll b/llvm/test/Transforms/SimplifyCFG/fold-two-entry-phi-node-with-one-block-profmd.ll --- a/llvm/test/Transforms/SimplifyCFG/fold-two-entry-phi-node-with-one-block-profmd.ll +++ b/llvm/test/Transforms/SimplifyCFG/fold-two-entry-phi-node-with-one-block-profmd.ll @@ -59,8 +59,12 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: call void @sideeffect0() ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: br i1 [[CMP]], label [[END:%.*]], label [[COND_TRUE:%.*]], !prof [[PROF0]] +; CHECK: cond.true: ; CHECK-NEXT: [[V0:%.*]] = add i32 [[C:%.*]], [[D:%.*]] -; CHECK-NEXT: [[RES:%.*]] = select i1 [[CMP]], i32 0, i32 [[V0]], !prof [[PROF0]] +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[V0]], [[COND_TRUE]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: call void @sideeffect1() ; CHECK-NEXT: ret i32 [[RES]] ; diff --git a/llvm/test/Transforms/SimplifyCFG/fold-two-entry-phi-node-with-two-blocks-profmd.ll b/llvm/test/Transforms/SimplifyCFG/fold-two-entry-phi-node-with-two-blocks-profmd.ll --- a/llvm/test/Transforms/SimplifyCFG/fold-two-entry-phi-node-with-two-blocks-profmd.ll +++ b/llvm/test/Transforms/SimplifyCFG/fold-two-entry-phi-node-with-two-blocks-profmd.ll @@ -39,9 +39,15 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: call void @sideeffect0() ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]], !prof [[PROF0:![0-9]+]] +; CHECK: cond.true: ; CHECK-NEXT: [[V0:%.*]] = add i32 [[C:%.*]], [[D:%.*]] +; CHECK-NEXT: br label [[END:%.*]] +; CHECK: cond.false: ; CHECK-NEXT: [[V1:%.*]] = sub i32 [[C]], [[D]] -; CHECK-NEXT: [[RES:%.*]] = select i1 [[CMP]], i32 [[V0]], i32 [[V1]], !prof [[PROF0:![0-9]+]] +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[V0]], [[COND_TRUE]] ], [ [[V1]], [[COND_FALSE]] ] ; CHECK-NEXT: call void @sideeffect1() ; CHECK-NEXT: ret i32 [[RES]] ; @@ -99,9 +105,15 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: call void @sideeffect0() ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: br i1 [[CMP]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]], !prof [[PROF0]] +; CHECK: cond.true: ; CHECK-NEXT: [[V0:%.*]] = add i32 [[C:%.*]], [[D:%.*]] +; CHECK-NEXT: br label [[END:%.*]] +; CHECK: cond.false: ; CHECK-NEXT: [[V1:%.*]] = sub i32 [[C]], [[D]] -; CHECK-NEXT: [[RES:%.*]] = select i1 [[CMP]], i32 [[V1]], i32 [[V0]], !prof [[PROF0]] +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[V0]], [[COND_TRUE]] ], [ [[V1]], [[COND_FALSE]] ] ; CHECK-NEXT: call void @sideeffect1() ; CHECK-NEXT: ret i32 [[RES]] ;