diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8528,7 +8528,7 @@ // Note: first set Entry as region entry and then connect successors starting // from it in order, to propagate the "parent" of each VPBasicBlock. - VPBlockUtils::insertTwoBlocksAfter(Pred, Exiting, BlockInMask, Entry); + VPBlockUtils::insertTwoBlocksAfter(Pred, Exiting, Entry); VPBlockUtils::connectBlocks(Pred, Exiting); return Region; @@ -8661,7 +8661,7 @@ // CanonicalIVIncrement{NUW} VPInstruction to increment it by VF * UF and a // BranchOnCount VPInstruction to the latch. static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, DebugLoc DL, - bool HasNUW, bool IsVPlanNative) { + bool HasNUW) { Value *StartIdx = ConstantInt::get(IdxTy, 0); auto *StartV = Plan.getOrAddVPValue(StartIdx); @@ -8677,8 +8677,6 @@ CanonicalIVPHI->addOperand(CanonicalIVIncrement); VPBasicBlock *EB = TopRegion->getExitingBasicBlock(); - if (IsVPlanNative) - EB->setCondBit(nullptr); EB->appendRecipe(CanonicalIVIncrement); auto *BranchOnCount = @@ -8785,7 +8783,7 @@ getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()); addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), DLInst ? DLInst->getDebugLoc() : DebugLoc(), - !CM.foldTailByMasking(), false); + !CM.foldTailByMasking()); // Scan the body of the loop in a topological order to visit each basic block // after having visited its predecessor basic blocks. @@ -9089,8 +9087,14 @@ [this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); }, DeadInstructions, *PSE.getSE()); + // Remove the existing terminator of the exiting block of the top-most region. + // A BranchOnCount will be added instead when adding the canonical IV recipes. + auto *Term = + Plan->getVectorLoopRegion()->getExitingBasicBlock()->getTerminator(); + Term->eraseFromParent(); + addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), DebugLoc(), - true, true); + true); return Plan; } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -351,41 +351,6 @@ Loop *CurrentVectorLoop = nullptr; }; -/// VPUsers instance used by VPBlockBase to manage CondBit and the block -/// predicate. Currently VPBlockUsers are used in VPBlockBase for historical -/// reasons, but in the future the only VPUsers should either be recipes or -/// live-outs.VPBlockBase uses. -struct VPBlockUser : public VPUser { - VPBlockUser() : VPUser({}, VPUserID::Block) {} - - VPValue *getSingleOperandOrNull() { - if (getNumOperands() == 1) - return getOperand(0); - - return nullptr; - } - const VPValue *getSingleOperandOrNull() const { - if (getNumOperands() == 1) - return getOperand(0); - - return nullptr; - } - - void resetSingleOpUser(VPValue *NewVal) { - assert(getNumOperands() <= 1 && "Didn't expect more than one operand!"); - if (!NewVal) { - if (getNumOperands() == 1) - removeLastOperand(); - return; - } - - if (getNumOperands() == 1) - setOperand(0, NewVal); - else - addOperand(NewVal); - } -}; - /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph. /// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock. class VPBlockBase { @@ -406,16 +371,6 @@ /// List of successor blocks. SmallVector Successors; - /// Successor selector managed by a VPUser. For blocks with zero or one - /// successors, there is no operand. Otherwise there is exactly one operand - /// which is the branch condition. - VPBlockUser CondBitUser; - - /// If the block is predicated, its predicate is stored as an operand of this - /// VPUser to maintain the def-use relations. Otherwise there is no operand - /// here. - VPBlockUser PredicateUser; - /// VPlan containing the block. Can only be set on the entry block of the /// plan. VPlan *Plan = nullptr; @@ -561,20 +516,6 @@ return getEnclosingBlockWithPredecessors()->getSinglePredecessor(); } - /// \return the condition bit selecting the successor. - VPValue *getCondBit(); - /// \return the condition bit selecting the successor. - const VPValue *getCondBit() const; - /// Set the condition bit selecting the successor. - void setCondBit(VPValue *CV); - - /// \return the block's predicate. - VPValue *getPredicate(); - /// \return the block's predicate. - const VPValue *getPredicate() const; - /// Set the block's predicate. - void setPredicate(VPValue *Pred); - /// Set a given VPBlockBase \p Successor as the single successor of this /// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor. /// This VPBlockBase must have no successors. @@ -584,14 +525,11 @@ } /// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two - /// successors of this VPBlockBase. \p Condition is set as the successor - /// selector. This VPBlockBase is not added as predecessor of \p IfTrue or \p - /// IfFalse. This VPBlockBase must have no successors. - void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse, - VPValue *Condition) { + /// successors of this VPBlockBase. This VPBlockBase is not added as + /// predecessor of \p IfTrue or \p IfFalse. This VPBlockBase must have no + /// successors. + void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) { assert(Successors.empty() && "Setting two successors when others exist."); - assert(Condition && "Setting two successors without condition!"); - setCondBit(Condition); appendSuccessor(IfTrue); appendSuccessor(IfFalse); } @@ -608,11 +546,8 @@ /// Remove all the predecessor of this block. void clearPredecessors() { Predecessors.clear(); } - /// Remove all the successors of this block and set to null its condition bit - void clearSuccessors() { - Successors.clear(); - setCondBit(nullptr); - } + /// Remove all the successors of this block. + void clearSuccessors() { Successors.clear(); } /// The method which generates the output IR that correspond to this /// VPBlockBase, thereby "executing" the VPlan. @@ -821,6 +756,7 @@ CanonicalIVIncrement, CanonicalIVIncrementNUW, BranchOnCount, + BranchOnCond }; private: @@ -909,6 +845,7 @@ case Instruction::Unreachable: case Instruction::Fence: case Instruction::AtomicRMW: + case VPInstruction::BranchOnCond: case VPInstruction::BranchOnCount: return false; default: @@ -2098,6 +2035,14 @@ using VPBlockBase::print; // Get the print(raw_stream &O) version. #endif + /// If the block has multiple successors, return the branch recipe terminating + /// the block. If there are no or only a single successor, return nullptr; + VPRecipeBase *getTerminator(); + const VPRecipeBase *getTerminator() const; + + /// Returns true if the block is exiting it's parent region. + bool isExiting() const; + private: /// Create an IR BasicBlock to hold the output instructions generated by this /// VPBasicBlock, and return it. Update the CFGState accordingly. @@ -2785,9 +2730,8 @@ /// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p /// NewBlock as successor of \p BlockPtr and \p BlockPtr as predecessor of \p /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. \p BlockPtr's - /// successors are moved from \p BlockPtr to \p NewBlock and \p BlockPtr's - /// conditional bit is propagated to \p NewBlock. \p NewBlock must have - /// neither successors nor predecessors. + /// successors are moved from \p BlockPtr to \p NewBlock. \p NewBlock must + /// have neither successors nor predecessors. static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) { assert(NewBlock->getSuccessors().empty() && NewBlock->getPredecessors().empty() && @@ -2798,24 +2742,22 @@ disconnectBlocks(BlockPtr, Succ); connectBlocks(NewBlock, Succ); } - NewBlock->setCondBit(BlockPtr->getCondBit()); - BlockPtr->setCondBit(nullptr); connectBlocks(BlockPtr, NewBlock); } /// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p /// BlockPtr. Add \p IfTrue and \p IfFalse as succesors of \p BlockPtr and \p /// BlockPtr as predecessor of \p IfTrue and \p IfFalse. Propagate \p BlockPtr - /// parent to \p IfTrue and \p IfFalse. \p Condition is set as the successor - /// selector. \p BlockPtr must have no successors and \p IfTrue and \p IfFalse - /// must have neither successors nor predecessors. + /// parent to \p IfTrue and \p IfFalse. \p BlockPtr must have no successors + /// and \p IfTrue and \p IfFalse must have neither successors nor + /// predecessors. static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, - VPValue *Condition, VPBlockBase *BlockPtr) { + VPBlockBase *BlockPtr) { assert(IfTrue->getSuccessors().empty() && "Can't insert IfTrue with successors."); assert(IfFalse->getSuccessors().empty() && "Can't insert IfFalse with successors."); - BlockPtr->setTwoSuccessors(IfTrue, IfFalse, Condition); + BlockPtr->setTwoSuccessors(IfTrue, IfFalse); IfTrue->setPredecessors({BlockPtr}); IfFalse->setPredecessors({BlockPtr}); IfTrue->setParent(BlockPtr->getParent()); @@ -3035,7 +2977,6 @@ /// create a new one. VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE); - } // end namespace vputils } // end namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -188,28 +188,6 @@ return Parent->getEnclosingBlockWithPredecessors(); } -VPValue *VPBlockBase::getCondBit() { - return CondBitUser.getSingleOperandOrNull(); -} - -const VPValue *VPBlockBase::getCondBit() const { - return CondBitUser.getSingleOperandOrNull(); -} - -void VPBlockBase::setCondBit(VPValue *CV) { CondBitUser.resetSingleOpUser(CV); } - -VPValue *VPBlockBase::getPredicate() { - return PredicateUser.getSingleOperandOrNull(); -} - -const VPValue *VPBlockBase::getPredicate() const { - return PredicateUser.getSingleOperandOrNull(); -} - -void VPBlockBase::setPredicate(VPValue *CV) { - PredicateUser.resetSingleOpUser(CV); -} - void VPBlockBase::deleteCFG(VPBlockBase *Entry) { SmallVector Blocks(depth_first(Entry)); @@ -357,29 +335,6 @@ for (VPRecipeBase &Recipe : Recipes) Recipe.execute(*State); - VPValue *CBV; - if (EnableVPlanNativePath && (CBV = getCondBit())) { - assert(CBV->getUnderlyingValue() && - "Unexpected null underlying value for condition bit"); - - // Condition bit value in a VPBasicBlock is used as the branch selector. In - // the VPlan-native path case, since all branches are uniform we generate a - // branch instruction using the condition value from vector lane 0 and dummy - // successors. The successors are fixed later when the successor blocks are - // visited. - Value *NewCond = State->get(CBV, {0, 0}); - - // Replace the temporary unreachable terminator with the new conditional - // branch. - auto *CurrentTerminator = NewBB->getTerminator(); - assert(isa(CurrentTerminator) && - "Expected to replace unreachable terminator with conditional " - "branch."); - auto *CondBr = BranchInst::Create(NewBB, nullptr, NewCond); - CondBr->setSuccessor(0, nullptr); - ReplaceInstWithInst(CurrentTerminator, CondBr); - } - LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB); } @@ -428,6 +383,50 @@ return P; } +static bool hasConditionalTerminator(const VPBasicBlock *VPBB) { + if (VPBB->empty()) { + assert( + VPBB->getNumSuccessors() < 2 && + "block with multiple successors doesn't have a recipe as terminator"); + return false; + } + + const VPRecipeBase *R = &VPBB->back(); + auto *VPI = dyn_cast(R); + bool IsCondBranch = + isa(R) || + (VPI && (VPI->getOpcode() == VPInstruction::BranchOnCond || + VPI->getOpcode() == VPInstruction::BranchOnCount)); + + if (VPBB->getNumSuccessors() >= 2 || VPBB->isExiting()) { + assert(IsCondBranch && "block with multiple successors not terminated by " + "conditional branch recipe"); + + return true; + } + + assert( + !IsCondBranch && + "block with 0 or 1 successors terminated by conditional branch recipe"); + return false; +} + +VPRecipeBase *VPBasicBlock::getTerminator() { + if (hasConditionalTerminator(this)) + return &back(); + return nullptr; +} + +const VPRecipeBase *VPBasicBlock::getTerminator() const { + if (hasConditionalTerminator(this)) + return &back(); + return nullptr; +} + +bool VPBasicBlock::isExiting() const { + return getParent()->getExitingBasicBlock() == this; +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const { if (getSuccessors().empty()) { @@ -444,13 +443,6 @@ void VPBasicBlock::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << Indent << getName() << ":\n"; - if (const VPValue *Pred = getPredicate()) { - O << Indent << "BlockPredicate:"; - Pred->printAsOperand(O, SlotTracker); - if (const auto *PredInst = dyn_cast(Pred)) - O << " (" << PredInst->getParent()->getName() << ")"; - O << '\n'; - } auto RecipeIndent = Indent + " "; for (const VPRecipeBase &Recipe : *this) { @@ -459,14 +451,6 @@ } printSuccessors(O, Indent); - - if (const VPValue *CBV = getCondBit()) { - O << Indent << "CondBit: "; - CBV->printAsOperand(O, SlotTracker); - if (const auto *CBI = dyn_cast(CBV)) - O << " (" << CBI->getParent()->getName() << ")"; - O << '\n'; - } } #endif @@ -781,6 +765,26 @@ State.set(this, Next, Part); break; } + case VPInstruction::BranchOnCond: { + if (Part != 0) + break; + Value *Cond = State.get(getOperand(0), VPIteration(Part, 0)); + VPRegionBlock *ParentRegion = getParent()->getParent(); + VPBasicBlock *Header = ParentRegion->getEntryBasicBlock(); + + // Replace the temporary unreachable terminator with a new conditional + // branch, hooking it up to backward destination for exiting blocks now and + // to forward destination(s) later when they are created. + BranchInst *CondBr = + Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), nullptr); + + if (getParent()->isExiting()) + CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]); + + CondBr->setSuccessor(0, nullptr); + Builder.GetInsertBlock()->getTerminator()->eraseFromParent(); + break; + } case VPInstruction::BranchOnCount: { if (Part != 0) break; @@ -854,6 +858,9 @@ case VPInstruction::CanonicalIVIncrementNUW: O << "VF * UF +(nuw) "; break; + case VPInstruction::BranchOnCond: + O << "branch-on-cond"; + break; case VPInstruction::BranchOnCount: O << "branch-on-count "; break; diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -216,10 +216,13 @@ "Instruction shouldn't have been visited."); if (auto *Br = dyn_cast(Inst)) { - // Branch instruction is not explicitly represented in VPlan but we need - // to represent its condition bit when it's conditional. - if (Br->isConditional()) - getOrCreateVPOperand(Br->getCondition()); + // Conditional branch instruction are represented using BranchOnCond + // recipes. + if (Br->isConditional()) { + VPValue *Cond = getOrCreateVPOperand(Br->getCondition()); + VPBB->appendRecipe( + new VPInstruction(VPInstruction::BranchOnCond, {Cond})); + } // Skip the rest of the Instruction processing for Branch instructions. continue; @@ -310,10 +313,9 @@ // representing the condition bit in VPlan (which may be in another VPBB). assert(IRDef2VPValue.count(BrCond) && "Missing condition bit in IRDef2VPValue!"); - VPValue *VPCondBit = IRDef2VPValue[BrCond]; - // Link successors using condition bit. - VPBB->setTwoSuccessors(SuccVPBB0, SuccVPBB1, VPCondBit); + // Link successors. + VPBB->setTwoSuccessors(SuccVPBB0, SuccVPBB1); } else llvm_unreachable("Number of successors not supported."); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -27,8 +27,12 @@ ReversePostOrderTraversal> RPOT(Plan->getEntry()); for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) { + VPRecipeBase *Term = VPBB->getTerminator(); + auto EndIter = Term ? Term->getIterator() : VPBB->end(); // Introduce each ingredient into VPlan. - for (VPRecipeBase &Ingredient : llvm::make_early_inc_range(*VPBB)) { + for (VPRecipeBase &Ingredient : + make_early_inc_range(make_range(VPBB->begin(), EndIter))) { + VPValue *VPV = Ingredient.getVPSingleValue(); Instruction *Inst = cast(VPV->getUnderlyingValue()); if (DeadInstructions.count(Inst)) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -209,9 +209,6 @@ enum class VPUserID { Recipe, LiveOut, - // TODO: Currently VPUsers are used in VPBlockBase, but in the future the - // only VPUsers should either be recipes or live-outs. - Block }; private: diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -49,11 +49,14 @@ // Check block's parent. assert(VPB->getParent() == Region && "VPBlockBase has wrong parent"); + auto *VPBB = dyn_cast(VPB); // Check block's condition bit. - if (VPB->getNumSuccessors() > 1 || Region->getExitingBasicBlock() == VPB) - assert(VPB->getCondBit() && "Missing condition bit!"); + if (VPB->getNumSuccessors() > 1 || (VPBB && VPBB->isExiting())) + assert(VPBB && VPBB->getTerminator() && + "Block has multiple successors but doesn't " + "have a proper branch recipe!"); else - assert(!VPB->getCondBit() && "Unexpected condition bit!"); + assert((!VPBB || !VPBB->getTerminator()) && "Unexpected branch recipe!"); // Check block's successors. const auto &Successors = VPB->getSuccessors(); diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -32,7 +32,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> @@ -54,7 +53,6 @@ ; CHECK-NEXT: pred.srem.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.if: ; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V) @@ -128,7 +126,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> @@ -201,7 +198,6 @@ ; CHECK-NEXT: pred.srem.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.if: ; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V) @@ -278,7 +274,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) @@ -302,7 +297,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V) @@ -383,7 +377,6 @@ ; CHECK-NEXT: pred.srem.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.if: ; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> @@ -402,7 +395,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, vp<[[PRED]]> diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -58,7 +58,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[COND]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[COND]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll @@ -26,16 +26,16 @@ ; CHECK-NEXT: EMIT store ir<%add> ir<%gep.2> ; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv> ir<1> ; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next> ir<8> +; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec> ; CHECK-NEXT: No successors -; CHECK-NEXT: CondBit: ir<%inner.ec> (inner) ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): outer.latch ; CHECK-EMPTY: ; CHECK-NEXT: outer.latch: ; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv> ir<1> ; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next> ir<8> +; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec> ; CHECK-NEXT: No successors -; CHECK-NEXT: CondBit: ir<%outer.ec> (outer.latch) ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): exit ; CHECK-EMPTY: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -215,7 +215,6 @@ ; CHECK-NEXT: pred.udiv.entry: ; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp> ; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue -; CHECK-NEXT: CondBit: ir<%cmp> ; CHECK-EMPTY: ; CHECK-NEXT: pred.udiv.if: ; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V) @@ -420,7 +419,6 @@ ; CHECK-NEXT: pred.sdiv.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[OR1]]> ; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue -; CHECK-NEXT: CondBit: vp<[[OR1]]> (if.then) ; CHECK-EMPTY: ; CHECK-NEXT: pred.sdiv.if: ; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V) diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -31,7 +31,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK ir<%c> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: ir<%c> ; CHECK: pred.store.if: ; CHECK-NEXT: CLONE store ir<%tmp5>, ir<%tmp2> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -33,7 +33,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> @@ -100,7 +99,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> @@ -120,7 +118,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10> @@ -183,7 +180,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> @@ -204,7 +200,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> @@ -270,7 +265,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep.A.uniform> @@ -295,7 +289,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK2]]> (loop.then) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr ir<%B>, vp<[[STEPS]]> @@ -368,7 +361,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0) ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> @@ -395,7 +387,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[OR]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[OR]]> (next.0) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> @@ -474,7 +465,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0) ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> @@ -505,7 +495,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK4]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK4]]> (then.1) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> @@ -593,7 +582,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0) ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> @@ -623,7 +611,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK5]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK5]]> (then.1) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> @@ -711,7 +698,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> @@ -742,7 +728,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr ir<@c>, ir<0>, vp<[[STEPS]]> @@ -827,7 +812,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> @@ -893,7 +877,6 @@ ; CHECK-NEXT: pred.load.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> @@ -916,7 +899,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]> @@ -980,7 +962,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK ir<%c> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: ir<%c> ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src> @@ -1059,7 +1040,6 @@ ; CHECK-NEXT: pred.store.entry: ; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-NEXT: CondBit: vp<[[MASK]]> (then) ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp --- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -45,7 +45,6 @@ EXPECT_NE(nullptr, Entry->getSingleSuccessor()); EXPECT_EQ(0u, Entry->getNumPredecessors()); EXPECT_EQ(1u, Entry->getNumSuccessors()); - EXPECT_EQ(nullptr, Entry->getCondBit()); // Check that the region following the preheader is a single basic-block // region (loop). @@ -91,7 +90,6 @@ EXPECT_EQ(Instruction::ICmp, ICmp->getOpcode()); EXPECT_EQ(2u, ICmp->getNumOperands()); EXPECT_EQ(IndvarAdd, ICmp->getOperand(0)); - EXPECT_EQ(VecBB->getCondBit(), ICmp); #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) // Add an external value to check we do not print the list of external values, @@ -123,7 +121,6 @@ " EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\> ir\<1\>\l" + " EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\> ir\<%N\>\l" + "No successors\l" + - "CondBit: ir\<%exitcond\> (vector.body)\l" ] } N1 -> N3 [ label="" ltail=cluster_N2]