diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp --- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp +++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp @@ -20,6 +20,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -77,118 +78,131 @@ STATISTIC(NumSimpl, "Number of blocks simplified"); -/// If we have more than one empty (other than phi node) return blocks, -/// merge them together to promote recursive block merging. -static bool mergeEmptyReturnBlocks(Function &F, DomTreeUpdater *DTU) { - bool Changed = false; +static bool tailMergeBlocksWithSimilarFunctionTerminators(Function &F, + DomTreeUpdater *DTU) { + SmallMapVector, 4> + Structure; - std::vector Updates; - SmallVector DeadBlocks; - - BasicBlock *RetBlock = nullptr; - - // Scan all the blocks in the function, looking for empty return blocks. - for (BasicBlock &BB : make_early_inc_range(F)) { + // Scan all the blocks in the function, record the interesting-ones. + for (BasicBlock &BB : F) { if (DTU && DTU->isBBPendingDeletion(&BB)) continue; - // Only look at return blocks. - ReturnInst *Ret = dyn_cast(BB.getTerminator()); - if (!Ret) continue; + // We are only interested in function-terminating blocks. + if (!succ_empty(&BB)) + continue; + + auto *Term = BB.getTerminator(); + + // Fow now only support `ret` function terminators. + // FIXME: lift this restriction. + if (Term->getOpcode() != Instruction::Ret) + continue; + + // We can't tail-merge block that contains a musttail call. + if (BB.getTerminatingMustTailCall()) + continue; + + // Calls to experimental_deoptimize must be followed by a return + // of the value computed by experimental_deoptimize. + // I.e., we can not change `ret` to `br` for this block. + if (auto *CI = + dyn_cast_or_null(Term->getPrevNonDebugInstruction())) { + if (Function *F = CI->getCalledFunction()) + if (Intrinsic::ID ID = F->getIntrinsicID()) + if (ID == Intrinsic::experimental_deoptimize) + continue; + } + + // PHI nodes cannot have token type, so if the terminator has an operand + // with token type, we can not tail-merge this kind of function terminators. + if (any_of(Term->operands(), + [](Value *Op) { return Op->getType()->isTokenTy(); })) + continue; // Only look at the block if it is empty or the only other thing in it is a // single PHI node that is the operand to the return. - if (Ret != &BB.front()) { + // FIXME: lift this restriction. + if (Term != &BB.front()) { // Check for something else in the block. - BasicBlock::iterator I(Ret); + BasicBlock::iterator I(Term); --I; // Skip over debug info. while (isa(I) && I != BB.begin()) --I; if (!isa(I) && - (!isa(I) || I != BB.begin() || Ret->getNumOperands() == 0 || - Ret->getOperand(0) != &*I)) + (!isa(I) || I != BB.begin() || Term->getNumOperands() == 0 || + Term->getOperand(0) != &*I)) continue; } - // If this is the first returning block, remember it and keep going. - if (!RetBlock) { - RetBlock = &BB; - continue; - } + // Canonical blocks are uniqued based on the terminator type (opcode). + Structure[Term->getOpcode()].emplace_back(&BB); + } - // Skip merging if this would result in a CallBr instruction with a - // duplicate destination. FIXME: See note in CodeGenPrepare.cpp. - bool SkipCallBr = false; - for (pred_iterator PI = pred_begin(&BB), E = pred_end(&BB); - PI != E && !SkipCallBr; ++PI) { - if (auto *CBI = dyn_cast((*PI)->getTerminator())) - for (unsigned i = 0, e = CBI->getNumSuccessors(); i != e; ++i) - if (RetBlock == CBI->getSuccessor(i)) { - SkipCallBr = true; - break; - } - } - if (SkipCallBr) + bool Changed = false; + + std::vector Updates; + + for (ArrayRef BBs : make_second_range(Structure)) { + BasicBlock *CanonicalBB = nullptr; + SmallVector Ops; + + // We don't want to change IR just because we can. + // Only do that if there are at least two blocks we'll tail-merge. + if (BBs.size() < 2) continue; - // Otherwise, we found a duplicate return block. Merge the two. Changed = true; - // Case when there is no input to the return or when the returned values - // agree is trivial. Note that they can't agree if there are phis in the - // blocks. - if (Ret->getNumOperands() == 0 || - Ret->getOperand(0) == - cast(RetBlock->getTerminator())->getOperand(0)) { - // All predecessors of BB should now branch to RetBlock instead. - if (DTU) { - SmallPtrSet PredsOfBB(pred_begin(&BB), pred_end(&BB)); - SmallPtrSet PredsOfRetBlock(pred_begin(RetBlock), - pred_end(RetBlock)); - Updates.reserve(Updates.size() + 2 * PredsOfBB.size()); - for (auto *Predecessor : PredsOfBB) - // But, iff Predecessor already branches to RetBlock, - // don't (re-)add DomTree edge, because it already exists. - if (!PredsOfRetBlock.contains(Predecessor)) - Updates.push_back({DominatorTree::Insert, Predecessor, RetBlock}); - for (auto *Predecessor : PredsOfBB) - Updates.push_back({DominatorTree::Delete, Predecessor, &BB}); - } - BB.replaceAllUsesWith(RetBlock); - DeadBlocks.emplace_back(&BB); - continue; - } - - // If the canonical return block has no PHI node, create one now. - PHINode *RetBlockPHI = dyn_cast(RetBlock->begin()); - if (!RetBlockPHI) { - Value *InVal = cast(RetBlock->getTerminator())->getOperand(0); - pred_iterator PB = pred_begin(RetBlock), PE = pred_end(RetBlock); - RetBlockPHI = PHINode::Create(Ret->getOperand(0)->getType(), - std::distance(PB, PE), "merge", - &RetBlock->front()); - - for (pred_iterator PI = PB; PI != PE; ++PI) - RetBlockPHI->addIncoming(InVal, *PI); - RetBlock->getTerminator()->setOperand(0, RetBlockPHI); - } - - // Turn BB into a block that just unconditionally branches to the return - // block. This handles the case when the two return blocks have a common - // predecessor but that return different things. - RetBlockPHI->addIncoming(Ret->getOperand(0), &BB); - BB.getTerminator()->eraseFromParent(); - BranchInst::Create(RetBlock, &BB); if (DTU) - Updates.push_back({DominatorTree::Insert, &BB, RetBlock}); + Updates.reserve(Updates.size() + BBs.size()); + + for (BasicBlock *BB : BBs) { + auto *Term = BB->getTerminator(); + + // Do we already have a canonical block for this function terminator type? + if (CanonicalBB == nullptr) { + // We do not. Create one now, placing it *before* the first block that + // will branch to it. + CanonicalBB = BasicBlock::Create( + F.getContext(), Twine("common.") + Term->getOpcodeName(), &F, + BBs[0]); + // We'll also need a PHI node per each operand of the terminator. + Ops.resize(Term->getNumOperands()); + for (auto I : zip(Term->operands(), Ops)) { + std::get<1>(I) = PHINode::Create(std::get<0>(I)->getType(), + /*NumReservedValues=*/BBs.size(), + CanonicalBB->getName() + ".op"); + CanonicalBB->getInstList().push_back(std::get<1>(I)); + } + // Make it so that this canonical block actually has the right + // terminator. + auto *CanonicalTerm = Term->clone(); + CanonicalBB->getInstList().push_back(CanonicalTerm); + // If the canonical terminator has operands, rewrite it to take PHI's. + for (auto I : zip(Ops, CanonicalTerm->operands())) + std::get<1>(I) = std::get<0>(I); + } + + // Aha, found a new non-canonical function terminator. + // If it has operands, forward them to the PHI nodes in the canonical + // block. + for (auto I : zip(Term->operands(), Ops)) + std::get<1>(I)->addIncoming(std::get<0>(I), BB); + + // And turn BB into a block that just unconditionally branches + // to the canonical block. + Term->eraseFromParent(); + BranchInst::Create(CanonicalBB, BB); + if (DTU) + Updates.push_back({DominatorTree::Insert, BB, CanonicalBB}); + } } if (DTU) DTU->applyUpdates(Updates); - DeleteDeadBlocks(DeadBlocks, DTU); - return Changed; } @@ -240,7 +254,8 @@ DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager); bool EverChanged = removeUnreachableBlocks(F, DT ? &DTU : nullptr); - EverChanged |= mergeEmptyReturnBlocks(F, DT ? &DTU : nullptr); + EverChanged |= + tailMergeBlocksWithSimilarFunctionTerminators(F, DT ? &DTU : nullptr); EverChanged |= iterativelySimplifyCFG(F, TTI, DT ? &DTU : nullptr, Options); // If neither pass changed anything, we're done. diff --git a/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll b/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll --- a/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll +++ b/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll @@ -75,8 +75,8 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: [[L86_OFF:%.*]] = add i32 [[L86:%.*]], -1 ; CHECK-NEXT: [[SWITCH:%.*]] = icmp ult i32 [[L86_OFF]], 24 -; CHECK-NEXT: br i1 [[SWITCH]], label [[FOR_END_I_IF_END8_I_CRIT_EDGE_I:%.*]], label [[FOR_INC_I_3_I_5:%.*]] -; CHECK: for.end.i.if.end8.i_crit_edge.i: +; CHECK-NEXT: br i1 [[SWITCH]], label [[COMMON_RET:%.*]], label [[FOR_INC_I_3_I_5:%.*]] +; CHECK: common.ret: ; CHECK-NEXT: ret i32 0 ; CHECK: for.inc.i.3.i.5: ; CHECK-NEXT: [[DOTNOT30:%.*]] = icmp ne i32 [[L86]], 25 diff --git a/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch.ll b/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch.ll --- a/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch.ll +++ b/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch.ll @@ -70,10 +70,10 @@ ; Test folding switch -> branch define i32 @test4(i32 %C) { ; CHECK-LABEL: @test4( -; CHECK-NEXT: L1: +; CHECK-NEXT: common.ret: ; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[C:%.*]], 0 -; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[COND]], i32 1, i32 0 -; CHECK-NEXT: ret i32 [[SPEC_SELECT]] +; CHECK-NEXT: [[DOT:%.*]] = select i1 [[COND]], i32 1, i32 0 +; CHECK-NEXT: ret i32 [[DOT]] ; switch i32 %C, label %L1 [ i32 0, label %L2 @@ -87,10 +87,10 @@ ; Can fold into a cond branch! define i32 @test5(i32 %C) { ; CHECK-LABEL: @test5( -; CHECK-NEXT: L1: +; CHECK-NEXT: common.ret: ; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[C:%.*]], 0 -; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[COND]], i32 1, i32 0 -; CHECK-NEXT: ret i32 [[SPEC_SELECT]] +; CHECK-NEXT: [[DOT:%.*]] = select i1 [[COND]], i32 1, i32 0 +; CHECK-NEXT: ret i32 [[DOT]] ; switch i32 %C, label %L1 [ i32 0, label %L2 diff --git a/llvm/test/Transforms/SimplifyCFG/ConditionalTrappingConstantExpr.ll b/llvm/test/Transforms/SimplifyCFG/ConditionalTrappingConstantExpr.ll --- a/llvm/test/Transforms/SimplifyCFG/ConditionalTrappingConstantExpr.ll +++ b/llvm/test/Transforms/SimplifyCFG/ConditionalTrappingConstantExpr.ll @@ -9,14 +9,14 @@ define i32 @admiral(i32 %a, i32 %b) { ; CHECK-LABEL: @admiral( ; CHECK-NEXT: [[C:%.*]] = icmp sle i32 [[A:%.*]], [[B:%.*]] -; CHECK-NEXT: br i1 [[C]], label [[BB2:%.*]], label [[BB1:%.*]] +; CHECK-NEXT: br i1 [[C]], label [[COMMON_RET:%.*]], label [[BB1:%.*]] ; CHECK: bb1: ; CHECK-NEXT: [[D:%.*]] = icmp sgt i32 sdiv (i32 -32768, i32 ptrtoint (i32* @G to i32)), 0 ; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[D]], i32 927, i32 42 -; CHECK-NEXT: br label [[BB2]] -; CHECK: bb2: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 42, [[TMP0:%.*]] ], [ [[SPEC_SELECT]], [[BB1]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK-NEXT: br label [[COMMON_RET]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 42, [[TMP0:%.*]] ], [ [[SPEC_SELECT]], [[BB1]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; %c = icmp sle i32 %a, %b br i1 %c, label %bb2, label %bb1 @@ -31,13 +31,13 @@ define i32 @ackbar(i1 %c) { ; CHECK-LABEL: @ackbar( -; CHECK-NEXT: br i1 [[C:%.*]], label [[BB5:%.*]], label [[BB6:%.*]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[BB5:%.*]], label [[COMMON_RET:%.*]] ; CHECK: bb5: ; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 icmp sgt (i32 sdiv (i32 32767, i32 ptrtoint (i32* @G to i32)), i32 0), i32 42, i32 927 -; CHECK-NEXT: br label [[BB6]] -; CHECK: bb6: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 42, [[TMP0:%.*]] ], [ [[SPEC_SELECT]], [[BB5]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK-NEXT: br label [[COMMON_RET]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 42, [[TMP0:%.*]] ], [ [[SPEC_SELECT]], [[BB5]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; br i1 %c, label %bb5, label %bb6 bb5: @@ -52,10 +52,11 @@ define i32 @tarp(i1 %c) { ; CHECK-LABEL: @tarp( -; CHECK-NEXT: bb9: -; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 fcmp oeq (float fdiv (float 3.000000e+00, float sitofp (i32 ptrtoint (i32* @G to i32) to float)), float 1.000000e+00), i32 42, i32 927 -; CHECK-NEXT: [[MERGE:%.*]] = select i1 [[C:%.*]], i32 [[SPEC_SELECT]], i32 42 -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK-NEXT: common.ret: +; CHECK-NEXT: [[C_NOT:%.*]] = xor i1 [[C:%.*]], true +; CHECK-NEXT: [[BRMERGE:%.*]] = or i1 [[C_NOT]], fcmp oeq (float fdiv (float 3.000000e+00, float sitofp (i32 ptrtoint (i32* @G to i32) to float)), float 1.000000e+00) +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = select i1 [[BRMERGE]], i32 42, i32 927 +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; br i1 %c, label %bb8, label %bb9 bb8: diff --git a/llvm/test/Transforms/SimplifyCFG/FoldValueComparisonIntoPredecessors-domtree-preservation-edgecase.ll b/llvm/test/Transforms/SimplifyCFG/FoldValueComparisonIntoPredecessors-domtree-preservation-edgecase.ll --- a/llvm/test/Transforms/SimplifyCFG/FoldValueComparisonIntoPredecessors-domtree-preservation-edgecase.ll +++ b/llvm/test/Transforms/SimplifyCFG/FoldValueComparisonIntoPredecessors-domtree-preservation-edgecase.ll @@ -4,19 +4,19 @@ define dso_local i32 @readCBPandCoeffsFromNAL(i1 %c, i32 %x, i32 %y) local_unnamed_addr { ; CHECK-LABEL: @readCBPandCoeffsFromNAL( ; CHECK-NEXT: if.end: -; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_END80:%.*]], label [[IF_THEN64:%.*]] -; CHECK: if.then64: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[Y:%.*]], [[IF_END:%.*]] ], [ 1, [[IF_END172237:%.*]] ], [ 0, [[IF_END80]] ], [ 0, [[IF_END80]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_END80:%.*]], label [[COMMON_RET:%.*]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ [[Y:%.*]], [[IF_END:%.*]] ], [ 1, [[IF_END172237:%.*]] ], [ 0, [[IF_END80]] ], [ 0, [[IF_END80]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: if.end80: ; CHECK-NEXT: switch i32 [[X:%.*]], label [[INFLOOP:%.*]] [ ; CHECK-NEXT: i32 10, label [[IF_END172237]] ; CHECK-NEXT: i32 14, label [[IF_END172237]] -; CHECK-NEXT: i32 9, label [[IF_THEN64]] -; CHECK-NEXT: i32 12, label [[IF_THEN64]] +; CHECK-NEXT: i32 9, label [[COMMON_RET]] +; CHECK-NEXT: i32 12, label [[COMMON_RET]] ; CHECK-NEXT: ] ; CHECK: if.end172237: -; CHECK-NEXT: br label [[IF_THEN64]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: infloop: ; CHECK-NEXT: br label [[INFLOOP]] ; diff --git a/llvm/test/Transforms/SimplifyCFG/FoldValueComparisonIntoPredecessors-no-new-successors.ll b/llvm/test/Transforms/SimplifyCFG/FoldValueComparisonIntoPredecessors-no-new-successors.ll --- a/llvm/test/Transforms/SimplifyCFG/FoldValueComparisonIntoPredecessors-no-new-successors.ll +++ b/llvm/test/Transforms/SimplifyCFG/FoldValueComparisonIntoPredecessors-no-new-successors.ll @@ -5,8 +5,8 @@ ; CHECK-LABEL: @widget( ; CHECK-NEXT: bb: ; CHECK-NEXT: [[SWITCH:%.*]] = icmp ult i32 [[ARG:%.*]], 2 -; CHECK-NEXT: br i1 [[SWITCH]], label [[BB2:%.*]], label [[INFLOOP:%.*]] -; CHECK: bb2: +; CHECK-NEXT: br i1 [[SWITCH]], label [[COMMON_RET:%.*]], label [[INFLOOP:%.*]] +; CHECK: common.ret: ; CHECK-NEXT: ret void ; CHECK: infloop: ; CHECK-NEXT: br label [[INFLOOP]] diff --git a/llvm/test/Transforms/SimplifyCFG/SimplifyEqualityComparisonWithOnlyPredecessor-domtree-preservation-edgecase.ll b/llvm/test/Transforms/SimplifyCFG/SimplifyEqualityComparisonWithOnlyPredecessor-domtree-preservation-edgecase.ll --- a/llvm/test/Transforms/SimplifyCFG/SimplifyEqualityComparisonWithOnlyPredecessor-domtree-preservation-edgecase.ll +++ b/llvm/test/Transforms/SimplifyCFG/SimplifyEqualityComparisonWithOnlyPredecessor-domtree-preservation-edgecase.ll @@ -10,13 +10,13 @@ ; CHECK-NEXT: [[C1_NOT:%.*]] = xor i1 [[C1:%.*]], true ; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[C0_NOT]], i1 true, i1 [[C1_NOT]] ; CHECK-NEXT: [[R0_MUX:%.*]] = select i1 [[C0_NOT]], i32 [[R0:%.*]], i32 [[R1:%.*]] -; CHECK-NEXT: br i1 [[BRMERGE]], label [[IF_THEN:%.*]], label [[DO_BODY:%.*]] -; CHECK: if.then: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[R0_MUX]], [[ENTRY:%.*]] ], [ [[R1]], [[DO_BODY]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK-NEXT: br i1 [[BRMERGE]], label [[COMMON_RET:%.*]], label [[DO_BODY:%.*]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ [[R0_MUX]], [[ENTRY:%.*]] ], [ [[R1]], [[DO_BODY]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: do.body: ; CHECK-NEXT: call void @zzz() -; CHECK-NEXT: switch i32 [[V:%.*]], label [[IF_THEN]] [ +; CHECK-NEXT: switch i32 [[V:%.*]], label [[COMMON_RET]] [ ; CHECK-NEXT: i32 10, label [[DO_BODY]] ; CHECK-NEXT: i32 32, label [[DO_BODY]] ; CHECK-NEXT: i32 9, label [[DO_BODY]] diff --git a/llvm/test/Transforms/SimplifyCFG/basictest.ll b/llvm/test/Transforms/SimplifyCFG/basictest.ll --- a/llvm/test/Transforms/SimplifyCFG/basictest.ll +++ b/llvm/test/Transforms/SimplifyCFG/basictest.ll @@ -84,6 +84,7 @@ ; PR5795 define void @test5(i32 %A) { ; CHECK-LABEL: @test5( +; CHECK-NEXT: common.ret: ; CHECK-NEXT: ret void ; switch i32 %A, label %return [ diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll --- a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll +++ b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll @@ -76,8 +76,8 @@ ; CHECK-NEXT: [[C_0:%.*]] = icmp sgt i32 [[X:%.*]], 0 ; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[C_0]], i1 true, i1 [[C_2:%.*]] ; CHECK-NEXT: [[DOTMUX:%.*]] = select i1 [[C_0]], i16 0, i16 20 -; CHECK-NEXT: br i1 [[BRMERGE]], label [[EXIT_1:%.*]], label [[FOR_COND]] -; CHECK: exit.1: +; CHECK-NEXT: br i1 [[BRMERGE]], label [[COMMON_RET:%.*]], label [[FOR_COND]] +; CHECK: common.ret: ; CHECK-NEXT: ret i16 [[DOTMUX]] ; entry: diff --git a/llvm/test/Transforms/SimplifyCFG/pr48778-sdiv-speculation.ll b/llvm/test/Transforms/SimplifyCFG/pr48778-sdiv-speculation.ll --- a/llvm/test/Transforms/SimplifyCFG/pr48778-sdiv-speculation.ll +++ b/llvm/test/Transforms/SimplifyCFG/pr48778-sdiv-speculation.ll @@ -4,15 +4,15 @@ ; sdiv INT_MIN / -1 should not be speculated. define i32 @test(i1 %cmp) { ; CHECK-LABEL: @test( -; CHECK-NEXT: br i1 [[CMP:%.*]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK-NEXT: br i1 [[CMP:%.*]], label [[IF:%.*]], label [[COMMON_RET:%.*]] ; CHECK: if: ; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 -2147483648, -1 ; CHECK-NEXT: [[CMP2:%.*]] = icmp ne i32 [[DIV]], 0 ; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP2]], i32 1, i32 0 -; CHECK-NEXT: br label [[ELSE]] -; CHECK: else: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[SPEC_SELECT]], [[IF]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK-NEXT: br label [[COMMON_RET]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[SPEC_SELECT]], [[IF]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; br i1 %cmp, label %if, label %else diff --git a/llvm/test/Transforms/SimplifyCFG/rangereduce.ll b/llvm/test/Transforms/SimplifyCFG/rangereduce.ll --- a/llvm/test/Transforms/SimplifyCFG/rangereduce.ll +++ b/llvm/test/Transforms/SimplifyCFG/rangereduce.ll @@ -10,21 +10,21 @@ ; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP1]], 2 ; CHECK-NEXT: [[TMP3:%.*]] = shl i32 [[TMP1]], 30 ; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]] -; CHECK-NEXT: switch i32 [[TMP4]], label [[DEF:%.*]] [ +; CHECK-NEXT: switch i32 [[TMP4]], label [[COMMON_RET:%.*]] [ ; CHECK-NEXT: i32 0, label [[ONE:%.*]] ; CHECK-NEXT: i32 1, label [[TWO:%.*]] ; CHECK-NEXT: i32 2, label [[THREE:%.*]] ; CHECK-NEXT: i32 3, label [[THREE]] ; CHECK-NEXT: ] -; CHECK: def: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ], [ 8867, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: one: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: two: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: three: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; switch i32 %a, label %def [ i32 97, label %one @@ -47,21 +47,21 @@ ; Optimization shouldn't trigger; bitwidth > 64 define i128 @test2(i128 %a) { ; CHECK-LABEL: @test2( -; CHECK-NEXT: switch i128 [[A:%.*]], label [[DEF:%.*]] [ +; CHECK-NEXT: switch i128 [[A:%.*]], label [[COMMON_RET:%.*]] [ ; CHECK-NEXT: i128 97, label [[ONE:%.*]] ; CHECK-NEXT: i128 101, label [[TWO:%.*]] ; CHECK-NEXT: i128 105, label [[THREE:%.*]] ; CHECK-NEXT: i128 109, label [[THREE]] ; CHECK-NEXT: ] -; CHECK: def: -; CHECK-NEXT: [[MERGE:%.*]] = phi i128 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ] -; CHECK-NEXT: ret i128 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i128 [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ], [ 8867, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i128 [[COMMON_RET_OP]] ; CHECK: one: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: two: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: three: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; switch i128 %a, label %def [ i128 97, label %one @@ -84,20 +84,20 @@ ; Optimization shouldn't trigger; no holes present define i32 @test3(i32 %a) { ; CHECK-LABEL: @test3( -; CHECK-NEXT: switch i32 [[A:%.*]], label [[DEF:%.*]] [ +; CHECK-NEXT: switch i32 [[A:%.*]], label [[COMMON_RET:%.*]] [ ; CHECK-NEXT: i32 97, label [[ONE:%.*]] ; CHECK-NEXT: i32 98, label [[TWO:%.*]] ; CHECK-NEXT: i32 99, label [[THREE:%.*]] ; CHECK-NEXT: ] -; CHECK: def: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ], [ 8867, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: one: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: two: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: three: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; switch i32 %a, label %def [ i32 97, label %one @@ -119,21 +119,21 @@ ; Optimization shouldn't trigger; not an arithmetic progression define i32 @test4(i32 %a) { ; CHECK-LABEL: @test4( -; CHECK-NEXT: switch i32 [[A:%.*]], label [[DEF:%.*]] [ +; CHECK-NEXT: switch i32 [[A:%.*]], label [[COMMON_RET:%.*]] [ ; CHECK-NEXT: i32 97, label [[ONE:%.*]] ; CHECK-NEXT: i32 102, label [[TWO:%.*]] ; CHECK-NEXT: i32 105, label [[THREE:%.*]] ; CHECK-NEXT: i32 109, label [[THREE]] ; CHECK-NEXT: ] -; CHECK: def: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ], [ 8867, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: one: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: two: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: three: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; switch i32 %a, label %def [ i32 97, label %one @@ -156,21 +156,21 @@ ; Optimization shouldn't trigger; not a power of two define i32 @test5(i32 %a) { ; CHECK-LABEL: @test5( -; CHECK-NEXT: switch i32 [[A:%.*]], label [[DEF:%.*]] [ +; CHECK-NEXT: switch i32 [[A:%.*]], label [[COMMON_RET:%.*]] [ ; CHECK-NEXT: i32 97, label [[ONE:%.*]] ; CHECK-NEXT: i32 102, label [[TWO:%.*]] ; CHECK-NEXT: i32 107, label [[THREE:%.*]] ; CHECK-NEXT: i32 112, label [[THREE]] ; CHECK-NEXT: ] -; CHECK: def: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ], [ 8867, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: one: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: two: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: three: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; switch i32 %a, label %def [ i32 97, label %one @@ -196,21 +196,21 @@ ; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP1]], 2 ; CHECK-NEXT: [[TMP3:%.*]] = shl i32 [[TMP1]], 30 ; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]] -; CHECK-NEXT: switch i32 [[TMP4]], label [[DEF:%.*]] [ +; CHECK-NEXT: switch i32 [[TMP4]], label [[COMMON_RET:%.*]] [ ; CHECK-NEXT: i32 3, label [[ONE:%.*]] ; CHECK-NEXT: i32 2, label [[TWO:%.*]] ; CHECK-NEXT: i32 1, label [[THREE:%.*]] ; CHECK-NEXT: i32 0, label [[THREE]] ; CHECK-NEXT: ] -; CHECK: def: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ], [ 8867, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: one: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: two: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: three: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; switch i32 %a, label %def [ i32 -97, label %one @@ -237,14 +237,14 @@ ; CHECK-NEXT: [[TMP3:%.*]] = shl i8 [[TMP1]], 6 ; CHECK-NEXT: [[TMP4:%.*]] = or i8 [[TMP2]], [[TMP3]] ; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i8 [[TMP4]], 4 -; CHECK-NEXT: br i1 [[TMP5]], label [[SWITCH_LOOKUP:%.*]], label [[DEF:%.*]] +; CHECK-NEXT: br i1 [[TMP5]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]] ; CHECK: switch.lookup: ; CHECK-NEXT: [[SWITCH_CAST:%.*]] = zext i8 [[TMP4]] to i32 ; CHECK-NEXT: [[SWITCH_SHIFTAMT:%.*]] = mul i32 [[SWITCH_CAST]], 8 ; CHECK-NEXT: [[SWITCH_DOWNSHIFT:%.*]] = lshr i32 -943228976, [[SWITCH_SHIFTAMT]] ; CHECK-NEXT: [[SWITCH_MASKED:%.*]] = trunc i32 [[SWITCH_DOWNSHIFT]] to i8 ; CHECK-NEXT: ret i8 [[SWITCH_MASKED]] -; CHECK: def: +; CHECK: common.ret: ; CHECK-NEXT: ret i8 -93 ; switch i8 %a, label %def [ @@ -271,21 +271,21 @@ ; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP1]], 2 ; CHECK-NEXT: [[TMP3:%.*]] = shl i32 [[TMP1]], 30 ; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]] -; CHECK-NEXT: switch i32 [[TMP4]], label [[DEF:%.*]] [ +; CHECK-NEXT: switch i32 [[TMP4]], label [[COMMON_RET:%.*]] [ ; CHECK-NEXT: i32 0, label [[ONE:%.*]] ; CHECK-NEXT: i32 1, label [[TWO:%.*]] ; CHECK-NEXT: i32 2, label [[THREE:%.*]] ; CHECK-NEXT: i32 4, label [[THREE]] ; CHECK-NEXT: ] -; CHECK: def: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ], [ 8867, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: one: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: two: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: three: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; switch i32 %a, label %def [ i32 97, label %one @@ -311,21 +311,21 @@ ; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP1]], 1 ; CHECK-NEXT: [[TMP3:%.*]] = shl i32 [[TMP1]], 31 ; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]] -; CHECK-NEXT: switch i32 [[TMP4]], label [[DEF:%.*]] [ +; CHECK-NEXT: switch i32 [[TMP4]], label [[COMMON_RET:%.*]] [ ; CHECK-NEXT: i32 6, label [[ONE:%.*]] ; CHECK-NEXT: i32 7, label [[TWO:%.*]] ; CHECK-NEXT: i32 0, label [[THREE:%.*]] ; CHECK-NEXT: i32 2, label [[THREE]] ; CHECK-NEXT: ] -; CHECK: def: -; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ] -; CHECK-NEXT: ret i32 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ], [ 8867, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i32 [[COMMON_RET_OP]] ; CHECK: one: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: two: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; CHECK: three: -; CHECK-NEXT: br label [[DEF]] +; CHECK-NEXT: br label [[COMMON_RET]] ; switch i32 %a, label %def [ i32 18, label %one diff --git a/llvm/test/Transforms/SimplifyCFG/suppress-zero-branch-weights.ll b/llvm/test/Transforms/SimplifyCFG/suppress-zero-branch-weights.ll --- a/llvm/test/Transforms/SimplifyCFG/suppress-zero-branch-weights.ll +++ b/llvm/test/Transforms/SimplifyCFG/suppress-zero-branch-weights.ll @@ -12,16 +12,16 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SEXT:%.*]] = sext i8 [[CONDITION:%.*]] to i32 ; CHECK-NEXT: switch i32 [[SEXT]], label [[DEFAULT:%.*]] [ -; CHECK-NEXT: i32 0, label [[A:%.*]] -; CHECK-NEXT: i32 127, label [[A]] -; CHECK-NEXT: i32 -128, label [[A]] -; CHECK-NEXT: i32 -1, label [[A]] +; CHECK-NEXT: i32 0, label [[COMMON_RET:%.*]] +; CHECK-NEXT: i32 127, label [[COMMON_RET]] +; CHECK-NEXT: i32 -128, label [[COMMON_RET]] +; CHECK-NEXT: i32 -1, label [[COMMON_RET]] ; CHECK-NEXT: ] -; CHECK: a: -; CHECK-NEXT: [[MERGE:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ], [ false, [[DEFAULT]] ] -; CHECK-NEXT: ret i1 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i1 [ false, [[DEFAULT]] ], [ true, [[ENTRY:%.*]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ] +; CHECK-NEXT: ret i1 [[COMMON_RET_OP]] ; CHECK: default: -; CHECK-NEXT: br label [[A]] +; CHECK-NEXT: br label [[COMMON_RET]] ; entry: %sext = sext i8 %condition to i32 diff --git a/llvm/test/Transforms/SimplifyCFG/switch-masked-bits.ll b/llvm/test/Transforms/SimplifyCFG/switch-masked-bits.ll --- a/llvm/test/Transforms/SimplifyCFG/switch-masked-bits.ll +++ b/llvm/test/Transforms/SimplifyCFG/switch-masked-bits.ll @@ -3,11 +3,11 @@ define i32 @test1(i32 %x) nounwind { ; CHECK-LABEL: @test1( -; CHECK-NEXT: a: +; CHECK-NEXT: common.ret: ; CHECK-NEXT: [[I:%.*]] = shl i32 [[X:%.*]], 1 ; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[I]], 24 -; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[COND]], i32 5, i32 0 -; CHECK-NEXT: ret i32 [[SPEC_SELECT]] +; CHECK-NEXT: [[DOT:%.*]] = select i1 [[COND]], i32 5, i32 0 +; CHECK-NEXT: ret i32 [[DOT]] ; %i = shl i32 %x, 1 switch i32 %i, label %a [ @@ -51,16 +51,16 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SEXT:%.*]] = sext i8 [[CONDITION:%.*]] to i32 ; CHECK-NEXT: switch i32 [[SEXT]], label [[DEFAULT:%.*]] [ -; CHECK-NEXT: i32 0, label [[A:%.*]] -; CHECK-NEXT: i32 127, label [[A]] -; CHECK-NEXT: i32 -128, label [[A]] -; CHECK-NEXT: i32 -1, label [[A]] +; CHECK-NEXT: i32 0, label [[COMMON_RET:%.*]] +; CHECK-NEXT: i32 127, label [[COMMON_RET]] +; CHECK-NEXT: i32 -128, label [[COMMON_RET]] +; CHECK-NEXT: i32 -1, label [[COMMON_RET]] ; CHECK-NEXT: ] -; CHECK: a: -; CHECK-NEXT: [[MERGE:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ], [ false, [[DEFAULT]] ] -; CHECK-NEXT: ret i1 [[MERGE]] +; CHECK: common.ret: +; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i1 [ false, [[DEFAULT]] ], [ true, [[ENTRY:%.*]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ] +; CHECK-NEXT: ret i1 [[COMMON_RET_OP]] ; CHECK: default: -; CHECK-NEXT: br label [[A]] +; CHECK-NEXT: br label [[COMMON_RET]] ; entry: %sext = sext i8 %condition to i32