diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8978,9 +8978,6 @@ HeaderVPBB->setName("vector.body"); - // Fold the last, empty block into its predecessor. - VPBB = VPBlockUtils::tryToMergeBlockIntoPredecessor(VPBB); - assert(VPBB && "expected to fold last (empty) block"); // After here, VPBB should not be used. VPBB = nullptr; @@ -9152,11 +9149,7 @@ VPlanTransforms::sinkScalarOperands(*Plan); VPlanTransforms::mergeReplicateRegions(*Plan); VPlanTransforms::removeRedundantExpandSCEVRecipes(*Plan); - - // Fold Exit block into its predecessor if possible. - // TODO: Fold block earlier once all VPlan transforms properly maintain a - // VPBasicBlock as exit. - VPBlockUtils::tryToMergeBlockIntoPredecessor(TopRegion->getExiting()); + VPlanTransforms::mergeBlocksIntoPredecessors(*Plan); assert(VPlanVerifier::verifyPlanIsValid(*Plan) && "VPlan is invalid"); return Plan; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2881,31 +2881,6 @@ To->removePredecessor(From); } - /// Try to merge \p Block into its single predecessor, if \p Block is a - /// VPBasicBlock and its predecessor has a single successor. Returns a pointer - /// to the predecessor \p Block was merged into or nullptr otherwise. - static VPBasicBlock *tryToMergeBlockIntoPredecessor(VPBlockBase *Block) { - auto *VPBB = dyn_cast(Block); - auto *PredVPBB = - dyn_cast_or_null(Block->getSinglePredecessor()); - if (!VPBB || !PredVPBB || PredVPBB->getNumSuccessors() != 1) - return nullptr; - - for (VPRecipeBase &R : make_early_inc_range(*VPBB)) - R.moveBefore(*PredVPBB, PredVPBB->end()); - VPBlockUtils::disconnectBlocks(PredVPBB, VPBB); - auto *ParentRegion = cast(Block->getParent()); - if (ParentRegion->getExiting() == Block) - ParentRegion->setExiting(PredVPBB); - SmallVector Successors(Block->successors()); - for (auto *Succ : Successors) { - VPBlockUtils::disconnectBlocks(Block, Succ); - VPBlockUtils::connectBlocks(PredVPBB, Succ); - } - delete Block; - return PredVPBB; - } - /// Return an iterator range over \p Range which only includes \p BlockTy /// blocks. The accesses are casted to \p BlockTy. template diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -40,6 +40,10 @@ static bool mergeReplicateRegions(VPlan &Plan); + /// Remove redundant VPBasicBlocks by merging them into their predecessor if + /// the predecessor has a single successor. + static bool mergeBlocksIntoPredecessors(VPlan &Plan); + /// Remove redundant casts of inductions. /// /// Such redundant casts are casts of induction variables that can be ignored, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -307,6 +307,34 @@ return Changed; } +bool VPlanTransforms::mergeBlocksIntoPredecessors(VPlan &Plan) { + SmallVector WorkList; + for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(depth_first( + VPBlockRecursiveTraversalWrapper(Plan.getEntry())))) { + auto *PredVPBB = + dyn_cast_or_null(VPBB->getSinglePredecessor()); + if (PredVPBB && PredVPBB->getNumSuccessors() == 1) + WorkList.push_back(VPBB); + } + + for (VPBasicBlock *VPBB : WorkList) { + VPBasicBlock *PredVPBB = cast(VPBB->getSinglePredecessor()); + for (VPRecipeBase &R : make_early_inc_range(*VPBB)) + R.moveBefore(*PredVPBB, PredVPBB->end()); + VPBlockUtils::disconnectBlocks(PredVPBB, VPBB); + auto *ParentRegion = cast_or_null(VPBB->getParent()); + if (ParentRegion && ParentRegion->getExiting() == VPBB) + ParentRegion->setExiting(PredVPBB); + SmallVector Successors(VPBB->successors()); + for (auto *Succ : Successors) { + VPBlockUtils::disconnectBlocks(VPBB, Succ); + VPBlockUtils::connectBlocks(PredVPBB, Succ); + } + delete VPBB; + } + return !WorkList.empty(); +} + void VPlanTransforms::removeRedundantInductionCasts(VPlan &Plan) { for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) { auto *IV = dyn_cast(&Phi); diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -23,9 +23,6 @@ ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -47,9 +44,6 @@ ; CHECK-NEXT: loop.1: ; CHECK-NEXT: WIDEN ir<%conv> = sext vp<[[PRED1]]> ; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0> ir<%conv> -; CHECK-NEXT: Successor(s): loop.1.split -; CHECK-EMPTY: -; CHECK-NEXT: loop.1.split: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -118,14 +112,8 @@ ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: ; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> ; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur> ir<%recur.next> -; CHECK-NEXT: Successor(s): loop.0.split -; CHECK-EMPTY: -; CHECK-NEXT: loop.0.split: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -194,9 +182,6 @@ ; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> ; CHECK-NEXT: EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: ; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> ; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur> ir<%recur.next> ; CHECK-NEXT: Successor(s): pred.srem @@ -272,9 +257,6 @@ ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -295,12 +277,6 @@ ; CHECK-NEXT: loop.1: ; CHECK-NEXT: WIDEN ir<%conv> = sext vp<[[PRED]]> ; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0> ir<%conv> -; CHECK-NEXT: Successor(s): loop.1.split -; CHECK-EMPTY: -; CHECK: loop.1.split: -; CHECK-NEXT: Successor(s): loop.2 -; CHECK-EMPTY: -; CHECK: loop.2: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK: pred.store: { @@ -378,12 +354,6 @@ ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: Successor(s): loop.1 -; CHECK-EMPTY: -; CHECK-NEXT: loop.1: ; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> ; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur> ir<%recur.next> ; CHECK-NEXT: Successor(s): pred.srem @@ -404,9 +374,6 @@ ; CHECK-NEXT: Successor(s): loop.1.split ; CHECK-EMPTY: ; CHECK-NEXT: loop.1.split: -; CHECK-NEXT: Successor(s): loop.2 -; CHECK-EMPTY: -; CHECK-NEXT: loop.2: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -477,14 +444,8 @@ ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1> ; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: ; CHECK-NEXT: CLONE ir<[[L]]> = load ir<%src> ; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn> ir<[[L]]> -; CHECK-NEXT: Successor(s): loop.0.split -; CHECK-EMPTY: -; CHECK-NEXT: loop.0.split: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -73,12 +73,6 @@ ; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir + vp<[[CAN_IV]]> * ir ; DBG-NEXT: vp<[[STEPS1:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir ; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; DBG-NEXT: Successor(s): cond.false -; DBG-EMPTY: -; DBG-NEXT: cond.false: -; DBG-NEXT: Successor(s): cond.false.0 -; DBG-EMPTY: -; DBG-NEXT: cond.false.0: ; DBG-NEXT: Successor(s): pred.store ; DBG-EMPTY: ; DBG-NEXT: pred.store: { @@ -100,9 +94,6 @@ ; DBG-NEXT: Successor(s): cond.false.1 ; DBG-EMPTY: ; DBG-NEXT: cond.false.1: -; DBG-NEXT: Successor(s): loop.latch -; DBG-EMPTY: -; DBG-NEXT: loop.latch: ; DBG-NEXT: EMIT vp<[[CAN_IV_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> ; DBG-NEXT: EMIT branch-on-count vp<[[CAN_IV_INC]]> vp<[[VEC_TC]]> ; DBG-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll @@ -16,9 +16,6 @@ ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%dst>, vp<[[STEPS:%.+]]> ; CHECK-NEXT: WIDEN store ir<%gep>, ir<%iv> -; CHECK-NEXT: Successor(s): loop.latch -; CHECK-EMPTY: -; CHECK-NEXT: loop.latch: ; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -206,9 +206,6 @@ ; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5> -; CHECK-NEXT: Successor(s): if.then -; CHECK-EMPTY: -; CHECK-NEXT: if.then: ; CHECK-NEXT: Successor(s): pred.udiv ; CHECK-EMPTY: ; CHECK-NEXT: pred.udiv: { @@ -227,9 +224,6 @@ ; CHECK-NEXT: Successor(s): if.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.0: -; CHECK-NEXT: Successor(s): for.inc -; CHECK-EMPTY: -; CHECK-NEXT: for.inc: ; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%cmp> ; CHECK-NEXT: BLEND %d = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/ir<%cmp> ; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<[[STEPS]]> @@ -404,13 +398,7 @@ ; CHECK-NEXT: WIDEN ir<%lsd> = load ir<%isd> ; CHECK-NEXT: WIDEN ir<%psd> = add ir<%lsd>, ir<23> ; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100> -; CHECK-NEXT: Successor(s): check -; CHECK-EMPTY: -; CHECK-NEXT: check: ; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200> -; CHECK-NEXT: Successor(s): if.then -; CHECK-EMPTY: -; CHECK-NEXT: if.then: ; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not ir<%cmp1>, !dbg /tmp/s.c:5:3 ; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]> ir<%cmp2> ir, !dbg /tmp/s.c:5:21 ; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]> ir<%cmp1> @@ -432,9 +420,6 @@ ; CHECK-NEXT: Successor(s): if.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.0: -; CHECK-NEXT: Successor(s): if.end -; CHECK-EMPTY: -; CHECK-NEXT: if.end: ; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp2> ; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]> vp<[[NOT2]]> ir ; CHECK-NEXT: BLEND %ysd.0 = vp<[[PHI]]>/vp<[[OR1]]> ir<%psd>/vp<[[SEL2]]> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -20,9 +20,6 @@ ; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2> ; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2> -; CHECK-NEXT: Successor(s): if.then - -; CHECK: if.then: ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -41,9 +38,6 @@ ; CHECK-NEXT: } ; CHECK: if.then.0: -; CHECK-NEXT: Successor(s): for.inc - -; CHECK: for.inc: ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -24,9 +24,6 @@ ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): loop.0 - -; CHECK: loop.0: ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -255,9 +252,6 @@ ; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr ir<%A>, ir<0> ; CHECK-NEXT: CLONE ir<%lv> = load ir<%gep.A.uniform> ; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k> -; CHECK-NEXT: Successor(s): loop.then -; CHECK-EMPTY: -; CHECK-NEXT: loop.then: ; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp> ; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]> vp<[[NOT2]]> ir ; CHECK-NEXT: Successor(s): pred.store @@ -279,9 +273,6 @@ ; CHECK-NEXT: Successor(s): loop.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.then.0: -; CHECK-NEXT: Successor(s): loop.latch -; CHECK-EMPTY: -; CHECK-NEXT: loop.latch: ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -328,9 +319,6 @@ ; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j> ; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> -; CHECK-NEXT: Successor(s): then.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0: ; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]> ir<%c.1> ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: @@ -352,9 +340,6 @@ ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: Successor(s): next.0 -; CHECK-EMPTY: -; CHECK-NEXT: next.0: ; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.1> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]> vp<[[NOT]]> ir ; CHECK-NEXT: BLEND %p = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> @@ -430,9 +415,6 @@ ; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> ; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> ; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j> -; CHECK-NEXT: Successor(s): then.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0: ; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]> ir<%c.0> ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: @@ -454,15 +436,9 @@ ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: Successor(s): next.0 -; CHECK-EMPTY: -; CHECK-NEXT: next.0: ; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]> vp<[[NOT]]> ir ; CHECK-NEXT: BLEND %p = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> -; CHECK-NEXT: Successor(s): then.1 -; CHECK-EMPTY: -; CHECK-NEXT: then.1: ; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]> vp<[[MASK3]]> ; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]> ir<%c.1> ir ; CHECK-NEXT: Successor(s): pred.store @@ -483,9 +459,6 @@ ; CHECK-NEXT: Successor(s): then.1.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.1.0: -; CHECK-NEXT: Successor(s): next.1 -; CHECK-EMPTY: -; CHECK-NEXT: next.1: ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -545,9 +518,6 @@ ; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> ; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: Successor(s): then.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0: ; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]> ir<%c.0> ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: @@ -569,20 +539,11 @@ ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: Successor(s): next.0 -; CHECK-EMPTY: -; CHECK-NEXT: next.0: ; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]> vp<[[NOT]]> ir ; CHECK-NEXT: BLEND %p = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> -; CHECK-NEXT: Successor(s): then.1 -; CHECK-EMPTY: -; CHECK-NEXT: then.1: ; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = or vp<[[MASK2]]> vp<[[MASK3]]> ; CHECK-NEXT: EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]> ir<%c.0> ir -; CHECK-NEXT: Successor(s): then.1.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.1.0: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -603,9 +564,6 @@ ; CHECK-NEXT: Successor(s): then.1.1 ; CHECK-EMPTY: ; CHECK-NEXT: then.1.1: -; CHECK-NEXT: Successor(s): next.1 -; CHECK-EMPTY: -; CHECK-NEXT: next.1: ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -664,15 +622,6 @@ ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: Successor(s): loop.1 -; CHECK-EMPTY: -; CHECK-NEXT: loop.1: -; CHECK-NEXT: Successor(s): loop.2 -; CHECK-EMPTY: -; CHECK-NEXT: loop.2: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -698,9 +647,6 @@ ; CHECK-EMPTY: ; CHECK-NEXT: loop.3: ; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: Successor(s): then.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0: ; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]> ; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]> ir<%c.0> ir ; CHECK-NEXT: Successor(s): pred.store @@ -721,9 +667,6 @@ ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: Successor(s): latch -; CHECK-EMPTY: -; CHECK-NEXT: latch: ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -779,12 +722,6 @@ ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: Successor(s): loop.1 -; CHECK-EMPTY: -; CHECK-NEXT: loop.1: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -867,9 +804,6 @@ ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: ; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for> vp<[[PRED]]> -; CHECK-NEXT: Successor(s): loop.1 -; CHECK-EMPTY: -; CHECK-NEXT: loop.1: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -925,12 +859,6 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: Successor(s): loop.then -; CHECK-EMPTY: -; CHECK-NEXT: loop.then: -; CHECK-NEXT: Successor(s): loop.then.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.then.0: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -953,9 +881,6 @@ ; CHECK-NEXT: Successor(s): loop.then.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.then.1: -; CHECK-NEXT: Successor(s): loop.latch -; CHECK-EMPTY: -; CHECK-NEXT: loop.latch: ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -999,14 +924,8 @@ ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> -; CHECK-NEXT: Successor(s): loop.body -; CHECK-EMPTY: -; CHECK-NEXT: loop.body: ; CHECK-NEXT: WIDEN ir<%0> = load ir<%gep> ; CHECK-NEXT: WIDEN ir<%pred> = fcmp oeq ir<%0>, ir<0.000000e+00> -; CHECK-NEXT: Successor(s): then -; CHECK-EMPTY: -; CHECK-NEXT: then: ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not ir<%pred> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: @@ -1026,9 +945,6 @@ ; CHECK-NEXT: Successor(s): then.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0: -; CHECK-NEXT: Successor(s): loop.latch -; CHECK-EMPTY: -; CHECK-NEXT: loop.latch: ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -1079,9 +995,6 @@ ; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> ; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -1146,9 +1059,6 @@ ; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr ir<%ptr.iv>, ir<-1> ; CHECK-NEXT: WIDEN ir<%l> = load ir<%ptr.iv.next> ; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0> -; CHECK-NEXT: Successor(s): if.then -; CHECK-EMPTY: -; CHECK-NEXT: if.then: ; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: @@ -1168,9 +1078,6 @@ ; CHECK-NEXT: Successor(s): if.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.0: -; CHECK-NEXT: Successor(s): loop.latch -; CHECK-EMPTY: -; CHECK-NEXT: loop.latch: ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors