Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8926,15 +8926,16 @@ // followed by a region for the vector loop, followed by the middle block. The // skeleton vector loop region contains a header and latch block. VPBasicBlock *Preheader = new VPBasicBlock("vector.ph"); - auto Plan = std::make_unique(Preheader); VPBasicBlock *HeaderVPBB = new VPBasicBlock("vector.body"); VPBasicBlock *LatchVPBB = new VPBasicBlock("vector.latch"); VPBlockUtils::insertBlockAfter(LatchVPBB, HeaderVPBB); - auto *TopRegion = new VPRegionBlock(HeaderVPBB, LatchVPBB, "vector loop"); - VPBlockUtils::insertBlockAfter(TopRegion, Preheader); + auto *Loop = new VPCountableLoop(HeaderVPBB, LatchVPBB, "vector countable loop"); + VPBlockUtils::insertBlockAfter(Loop, Preheader); VPBasicBlock *MiddleVPBB = new VPBasicBlock("middle.block"); - VPBlockUtils::insertBlockAfter(MiddleVPBB, TopRegion); + VPBlockUtils::insertBlockAfter(MiddleVPBB, Loop); + auto *TopRegion = new VPRegionBlock(Preheader, MiddleVPBB, "vector loop"); + auto Plan = std::make_unique(TopRegion); // Don't use getDecisionAndClampRange here, because we don't know the UF // so this function is better to be conservative, rather than to split @@ -9035,9 +9036,12 @@ } else addUsersInExitBlock(HeaderVPBB, MiddleVPBB, OrigLoop, *Plan); - assert(isa(Plan->getVectorLoopRegion()) && + if (std::optional Bounds = + OrigLoop->getBounds(*PSE.getSE())) + Loop->setUpperBound(Plan->getVPValue(&Bounds->getFinalIVValue())); + assert(isa(Plan->getVectorLoopRegion()) && !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() && - "entry block must be set to a VPRegionBlock having a non-empty entry " + "entry block must be set to a VPLoopBase having a non-empty entry " "VPBasicBlock"); RecipeBuilder.fixHeaderPhis(); @@ -9050,7 +9054,7 @@ VPlanTransforms::removeRedundantInductionCasts(*Plan); // Adjust the recipes for any inloop reductions. - adjustRecipesForReductions(cast(TopRegion->getExiting()), Plan, + adjustRecipesForReductions(cast(Loop->getExiting()), Plan, RecipeBuilder, Range.Start); // Sink users of fixed-order recurrence past the recipe defining the previous Index: llvm/lib/Transforms/Vectorize/VPlan.h =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.h +++ llvm/lib/Transforms/Vectorize/VPlan.h @@ -472,7 +472,7 @@ /// that are actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPBlockBase objects. They are used for concrete /// type identification. - using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC }; + using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC, VPCountableLoopSC }; using VPBlocksTy = SmallVectorImpl; @@ -2038,6 +2038,7 @@ /// Returns a reference to the list of recipes. RecipeListTy &getRecipeList() { return Recipes; } + const RecipeListTy &getRecipeList() const { return Recipes; } /// Returns a pointer to a member of the recipe list. static RecipeListTy VPBasicBlock::*getSublistAccess(VPRecipeBase *) { @@ -2128,17 +2129,15 @@ public: VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, + const std::string &Name = "", bool IsReplicator = false); + + VPRegionBlock(unsigned RegionOpcode, VPBlockBase *Entry, VPBlockBase *Exiting, + const std::string &Name = ""); + + VPRegionBlock(unsigned RegionOpcode = VPRegionBlockSC, const std::string &Name = "", bool IsReplicator = false) - : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting), - IsReplicator(IsReplicator) { - assert(Entry->getPredecessors().empty() && "Entry block has predecessors."); - assert(Exiting->getSuccessors().empty() && "Exit block has successors."); - Entry->setParent(this); - Exiting->setParent(this); - } - VPRegionBlock(const std::string &Name = "", bool IsReplicator = false) - : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr), - IsReplicator(IsReplicator) {} + : VPBlockBase(RegionOpcode, Name), Entry(nullptr), Exiting(nullptr), + IsReplicator(IsReplicator){} ~VPRegionBlock() override { if (Entry) { @@ -2150,7 +2149,8 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPBlockBase *V) { - return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC; + return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC || + V->getVPBlockID() == VPCountableLoopSC; } const VPBlockBase *getEntry() const { return Entry; } @@ -2206,6 +2206,95 @@ #endif }; +/// A base class for all loop-like regions +class VPLoopBase : public VPRegionBlock { +public: + explicit VPLoopBase(unsigned LoopOpcode, VPBlockBase *Header, + VPBlockBase *Exiting, const std::string &Name = "") + : VPRegionBlock(LoopOpcode, Header, Exiting, Name) {} + + explicit VPLoopBase(unsigned LoopOpcode, const std::string &Name = "") + : VPRegionBlock(LoopOpcode, Name, false) {} + + /// Method to return header VPBB + VPBlockBase *getHeader() { return getEntry(); } + + /// Method to return header VPBB + const VPBlockBase *getHeader() const { return getEntry(); } + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPBlockBase *V) { + return V->getVPBlockID() == VPBlockBase::VPCountableLoopSC; + } +}; + +/// Countable loop representation in VPlan. +class VPCountableLoop final : public VPLoopBase { + VPValue *LowerBound; + VPValue *UpperBound; + +public: + explicit VPCountableLoop(VPBlockBase *Header, VPBlockBase *Exiting, + VPValue *LB, VPValue *UB, + const std::string &Name = ""); + + /// TODO: These constructors should be moved to private and only allowed to + /// be called by VPlan builder which may not convert lower and upper bounds + /// before countable loop is created. + explicit VPCountableLoop(VPBasicBlock *Header, VPBasicBlock *Exiting, + const std::string &Name = "") + : VPCountableLoop(Header, Exiting, /*LowerBound=*/nullptr, + /*UpperBound=*/nullptr, Name) {} + + explicit VPCountableLoop(const std::string &Name = "") + : VPLoopBase(VPCountableLoopSC, Name) {} + + /// Method to get IV of the loop + VPValue *getCanonicalIV(); + + /// Method to get IV of the loop + const VPValue *getCanonicalIV() const; + + /// Return lower bound of the countable loop + VPValue *getLowerBound() { return LowerBound; } + + /// Return lower bound of the countable loop + const VPValue *getLowerBound() const { return LowerBound; } + + /// Return upper bound of the countable loop + VPValue *getUpperBound() { return UpperBound; } + + /// Return upper bound of the countable loop + const VPValue *getUpperBound() const { return UpperBound; } + + /// Set lower bound of the loop + void setLowerBound(VPValue *LB) { LowerBound = LB; } + + /// Set upper bound of the loop + void setUpperBound(VPValue *UB) { UpperBound = UB; } + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPBlockBase *V) { + return V->getVPBlockID() == VPBlockBase::VPCountableLoopSC; + } + + /// The method which generates the output IR instructions that correspond to + /// this VPCountableLoop, thereby "executing" the VPlan. + void execute(VPTransformState *State) override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print this VPLoopRegion to \p O (recursively), prefixing all lines with + /// \p Indent. \p SlotTracker is used to print unnamed VPValue's using + /// consequtive numbers. + /// + /// Note that the numbering is applied to the whole VPlan, so printing + /// individual regions is consistent with the whole VPlan printing. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; + using VPBlockBase::print; // Get the print(raw_stream &O) version. +#endif +}; + /// VPlan models a candidate for vectorization, encoding various decisions take /// to produce efficient output IR, including which branches, basic-blocks and /// output IR instructions to generate, and their cost. VPlan holds a @@ -2384,22 +2473,42 @@ return map_range(Operands, Fn); } - /// Returns the VPRegionBlock of the vector loop. - VPRegionBlock *getVectorLoopRegion() { - return cast(getEntry()->getSingleSuccessor()); - } - const VPRegionBlock *getVectorLoopRegion() const { - return cast(getEntry()->getSingleSuccessor()); + /// Returns the VPLoopBase of the VPlan + /// TODO: These function should be moved to vputils or removed completely as + /// VPlan may contain multiple loops + VPLoopBase *getVectorLoopRegion() { + return const_cast( + const_cast(this)->getVectorLoopRegion()); + } + + /// Returns the VPLoopBase of the VPlan + /// TODO: These function should be moved to vputils or removed completely as + /// VPlan may contain multiple loops + const VPLoopBase *getVectorLoopRegion() const { + assert(isa(Entry) && + "Entry of the VPlan must be a type of VPRegionBlock"); + // Ad-hoc approach that expects + // VPRegionBlock { + // preheader: + // ... + // VPLoopBase { + // ... + // } + // } + VPBlockBase *Loop = + cast(Entry)->getEntry()->getSingleSuccessor(); + assert(isa(Loop) && + "VPLoopBase is expected as a successor of the preheader"); + return cast(Loop); } /// Returns the canonical induction recipe of the vector loop. + /// TODO: These function should be moved to vputils or removed completely as + /// VPlan may contain multiple VPLoops VPCanonicalIVPHIRecipe *getCanonicalIV() { - VPBasicBlock *EntryVPBB = getVectorLoopRegion()->getEntryBasicBlock(); - if (EntryVPBB->empty()) { - // VPlan native path. - EntryVPBB = cast(EntryVPBB->getSingleSuccessor()); - } - return cast(&*EntryVPBB->begin()); + VPLoopBase *Loop = getVectorLoopRegion(); + VPValue *IV = cast(Loop)->getCanonicalIV(); + return cast(IV->getDefiningRecipe()); } /// Find and return the VPActiveLaneMaskPHIRecipe from the header - there Index: llvm/lib/Transforms/Vectorize/VPlan.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.cpp +++ llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -517,27 +517,8 @@ ReversePostOrderTraversal> RPOT(Entry); - if (!isReplicator()) { - // Create and register the new vector loop. - Loop *PrevLoop = State->CurrentVectorLoop; - State->CurrentVectorLoop = State->LI->AllocateLoop(); - BasicBlock *VectorPH = State->CFG.VPBB2IRBB[getPreheaderVPBB()]; - Loop *ParentLoop = State->LI->getLoopFor(VectorPH); - - // Insert the new loop into the loop nest and register the new basic blocks - // before calling any utilities such as SCEV that require valid LoopInfo. - if (ParentLoop) - ParentLoop->addChildLoop(State->CurrentVectorLoop); - else - State->LI->addTopLevelLoop(State->CurrentVectorLoop); - - // Visit the VPBlocks connected to "this", starting from it. - for (VPBlockBase *Block : RPOT) { - LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n'); - Block->execute(State); - } - - State->CurrentVectorLoop = PrevLoop; + if (!isReplicator() && !getParent()) { + getPlan()->getVectorLoopRegion()->execute(State); return; } @@ -673,7 +654,8 @@ State->Builder.SetInsertPoint(VectorPreHeader->getTerminator()); // Generate code in the loop pre-header and body. - for (VPBlockBase *Block : vp_depth_first_shallow(Entry)) + for (VPBlockBase *Block : + vp_depth_first_shallow(cast(Entry)->getEntry())) Block->execute(State); VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock(); @@ -1138,3 +1120,113 @@ Preheader->appendRecipe(Step); return Step; } + +VPValue *VPCountableLoop::getCanonicalIV() { + return const_cast( + const_cast(this)->getCanonicalIV()); +} + +const VPValue *VPCountableLoop::getCanonicalIV() const { + for (const VPRecipeBase &R : + getHeader()->getEntryBasicBlock()->getRecipeList()) + if (auto *VPIV = dyn_cast(&R)) + return VPIV; + return nullptr; +} + +void VPCountableLoop::execute(VPTransformState *State) { + ReversePostOrderTraversal> RPOT( + getHeader()); + // Create and register the new vector loop. + Loop *PrevLoop = State->CurrentVectorLoop; + State->CurrentVectorLoop = State->LI->AllocateLoop(); + BasicBlock *VectorPH = + State->CFG.VPBB2IRBB[cast(getSinglePredecessor())]; + Loop *ParentLoop = State->LI->getLoopFor(VectorPH); + + // Insert the new loop into the loop nest and register the new basic blocks + // before calling any utilities such as SCEV that require valid LoopInfo. + if (ParentLoop) + ParentLoop->addChildLoop(State->CurrentVectorLoop); + else + State->LI->addTopLevelLoop(State->CurrentVectorLoop); + + // Visit the VPBlocks connected to "this", starting from it. + for (VPBlockBase *Block : RPOT) { + LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n'); + Block->execute(State); + } + + State->CurrentVectorLoop = PrevLoop; +} + +VPRegionBlock::VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, + const std::string &Name, bool IsReplicator) + : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting), + IsReplicator(IsReplicator) { + assert(Entry->getPredecessors().empty() && "Entry block has predecessors."); + assert(Exiting->getSuccessors().empty() && "Exit block has successors."); + Entry->setParent(this); + Exiting->setParent(this); + ReversePostOrderTraversal> RPOT( + Entry); + for (VPBlockBase *VPBB : RPOT) + VPBB->setParent(this); +} + +VPRegionBlock::VPRegionBlock(unsigned RegionOpcode, VPBlockBase *Entry, + VPBlockBase *Exiting, const std::string &Name) + : VPBlockBase(RegionOpcode, Name), Entry(Entry), Exiting(Exiting), + IsReplicator(false) { + assert(Entry->getPredecessors().empty() && "Entry block has predecessors."); + // Single exiting block may not exist for loops with multple exits + assert((!Exiting || Exiting->getSuccessors().empty()) && + "Exit block has successors."); + Entry->setParent(this); + if (Exiting) + Exiting->setParent(this); + ReversePostOrderTraversal> RPOT( + Entry); + for (VPBlockBase *VPBB : RPOT) + VPBB->setParent(this); +} + +VPCountableLoop::VPCountableLoop(VPBlockBase *Header, VPBlockBase *Exiting, + VPValue *LB, VPValue *UB, + const std::string &Name) + : VPLoopBase(VPCountableLoopSC, Header, Exiting, Name), LowerBound(LB), + UpperBound(UB) { + ReversePostOrderTraversal> RPOT( + Header); + for (VPBlockBase *VPBB : RPOT) + VPBB->setParent(this); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPCountableLoop::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << getName(); + if (const VPValue *IV = getCanonicalIV()) { + O << ' '; + IV->printAsOperand(O, SlotTracker); + O << " = "; + if (const VPValue *LB = getLowerBound()) + O << *LB; + else + O << '0'; + if (const VPValue *UB = getUpperBound()) + O << ", " << *UB; + else + O << ", inf"; + } + O << " {\n"; + auto NewIndent = Indent + " "; + for (auto *BlockBase : vp_depth_first_shallow(getHeader())) { + O << '\n'; + BlockBase->print(O, NewIndent, SlotTracker); + } + O << Indent << "}\n"; + + printSuccessors(O, Indent); +} +#endif Index: llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h =================================================================== --- llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h +++ llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h @@ -59,7 +59,7 @@ /// Build plain CFG for TheLoop. Return the pre-header VPBasicBlock connected /// to a new VPRegionBlock (TopRegion) enclosing the plain CFG. - VPBasicBlock *buildPlainCFG(); + VPBlockBase *buildPlainCFG(); public: VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P) Index: llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -75,7 +75,7 @@ /// Build plain CFG for TheLoop. Return the pre-header VPBasicBlock connected /// to a new VPRegionBlock (TopRegion) enclosing the plain CFG. - VPBasicBlock *buildPlainCFG(); + VPBlockBase *buildPlainCFG(); }; } // anonymous namespace @@ -123,8 +123,8 @@ if (CurrentLoop) { auto Iter = Loop2Region.insert({CurrentLoop, nullptr}); if (Iter.second) - Iter.first->second = new VPRegionBlock( - CurrentLoop->getHeader()->getName().str(), false /*isReplicator*/); + Iter.first->second = + new VPCountableLoop(CurrentLoop->getHeader()->getName().str()); ParentR = Iter.first->second; } @@ -254,7 +254,7 @@ } // Main interface to build the plain CFG. -VPBasicBlock *PlainCFGBuilder::buildPlainCFG() { +VPBlockBase *PlainCFGBuilder::buildPlainCFG() { // 1. Scan the body of the loop in a topological order to visit each basic // block after having visited its predecessor basic blocks. Create a VPBB for // each BB and link it to its successor and predecessor VPBBs. Note that @@ -362,6 +362,7 @@ VPBasicBlock *ExitVPBB = getOrCreateVPBB(L->getExitBlock()); VPBlockUtils::disconnectBlocks(ExitingVPBB, ExitVPBB); Region->setExiting(ExitingVPBB); + // TODO: Set lower and upper bound for the loop VPBlockUtils::connectBlocks(Region, ExitVPBB); // Queue sub-loops for processing. @@ -372,10 +373,10 @@ // VPlan operands. fixPhiNodes(); - return ThePreheaderVPBB; + return new VPRegionBlock(ThePreheaderVPBB, LoopExitVPBB); } -VPBasicBlock *VPlanHCFGBuilder::buildPlainCFG() { +VPBlockBase *VPlanHCFGBuilder::buildPlainCFG() { PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan); return PCFGBuilder.buildPlainCFG(); } @@ -383,11 +384,11 @@ // Public interface to build a H-CFG. void VPlanHCFGBuilder::buildHierarchicalCFG() { // Build Top Region enclosing the plain CFG and set it as VPlan entry. - VPBasicBlock *EntryVPBB = buildPlainCFG(); + VPBlockBase *EntryVPBB = buildPlainCFG(); Plan.setEntry(EntryVPBB); LLVM_DEBUG(Plan.setName("HCFGBuilder: Plain CFG\n"); dbgs() << Plan); - VPRegionBlock *TopRegion = Plan.getVectorLoopRegion(); + VPRegionBlock *TopRegion = cast(Plan.getEntry()); Verifier.verifyHierarchicalCFG(TopRegion); // Compute plain CFG dom tree for VPLInfo. Index: llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "VPlanTransforms.h" +#include "VPlan.h" #include "VPlanDominatorTree.h" #include "VPRecipeBuilder.h" #include "VPlanCFG.h" @@ -719,7 +720,7 @@ SmallVector RecurrencePhis; for (VPRecipeBase &R : - Plan.getVectorLoopRegion()->getEntry()->getEntryBasicBlock()->phis()) + Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) if (auto *FOR = dyn_cast(&R)) RecurrencePhis.push_back(FOR); Index: llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -51,7 +51,10 @@ auto *VPBB = dyn_cast(VPB); // Check block's condition bit. - if (VPB->getNumSuccessors() > 1 || (VPBB && VPBB->isExiting())) + if (VPB->getNumSuccessors() == 0 && !Region->getParent()) + assert(!VPBB->getTerminator() && + "Unexpected branch recipe in VPlan's exiting basic block"); + else if (VPB->getNumSuccessors() > 1 || (VPBB && VPBB->isExiting())) assert(VPBB && VPBB->getTerminator() && "Block has multiple successors but doesn't " "have a proper branch recipe!"); Index: llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll @@ -10,28 +10,37 @@ ; VPLANS-LABEL: Checking a loop in 'simple_memset' ; VPLANS: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' { -; VPLANS-NEXT: Live-in vp<[[TC:%[0-9]+]]> = original trip-count -; VPLANS-EMPTY: -; VPLANS-NEXT: vector.ph: -; VPLANS-NEXT: EMIT vp<[[VF:%[0-9]+]]> = VF * Part + ir<0> -; VPLANS-NEXT: EMIT vp<[[NEWTC:%[0-9]+]]> = TC > VF ? TC - VF : 0 vp<[[TC]]> -; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%[0-9]+]]> = active lane mask vp<[[VF]]> vp<[[TC]]> -; VPLANS-NEXT: Successor(s): vector loop +; VPLANS-NEXT: Live-in vp<%1> = original trip-count ; VPLANS-EMPTY: ; VPLANS-NEXT: vector loop: { -; VPLANS-NEXT: vector.body: -; VPLANS-NEXT: EMIT vp<[[INDV:%[0-9]+]]> = CANONICAL-INDUCTION -; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%[0-9]+]]> -; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1> -; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]> -; VPLANS-NEXT: WIDEN store ir<%gep>, ir<%val>, vp<[[LANEMASK_PHI]]> -; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV]]> -; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]> vp<[[NEWTC]]> -; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = VF * UF + vp<[[INDV]]> -; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]> -; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]> +; VPLANS-NEXT: vector.ph: +; VPLANS-NEXT: EMIT vp<%2> = VF * Part + ir<0> +; VPLANS-NEXT: EMIT vp<%3> = TC > VF ? TC - VF : 0 vp<%1> +; VPLANS-NEXT: EMIT vp<%4> = active lane mask vp<%2> vp<%1> +; VPLANS-NEXT: Successor(s): vector countable loop +; VPLANS-EMPTY: +; VPLANS-NEXT: vector countable loop vp<%5> = 0, ir<%n> { +; VPLANS-EMPTY: +; VPLANS-NEXT: vector.body: +; VPLANS-NEXT: EMIT vp<%5> = CANONICAL-INDUCTION +; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<%6> = phi vp<%4>, vp<%10> +; VPLANS-NEXT: vp<%7> = SCALAR-STEPS vp<%5>, ir<1> +; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<%7> +; VPLANS-NEXT: WIDEN store ir<%gep>, ir<%val>, vp<%6> +; VPLANS-NEXT: EMIT vp<%9> = VF * Part + vp<%5> +; VPLANS-NEXT: EMIT vp<%10> = active lane mask vp<%9> vp<%3> +; VPLANS-NEXT: EMIT vp<%11> = VF * UF + vp<%5> +; VPLANS-NEXT: EMIT vp<%12> = not vp<%10> +; VPLANS-NEXT: EMIT branch-on-cond vp<%12> +; VPLANS-NEXT: No successors +; VPLANS-NEXT: } +; VPLANS-NEXT: Successor(s): middle.block +; VPLANS-EMPTY: +; VPLANS-NEXT: middle.block: ; VPLANS-NEXT: No successors ; VPLANS-NEXT: } +; VPLANS-NEXT: No successors +; VPLANS-NEXT: } define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-LABEL: @simple_memset( Index: llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -9,25 +9,34 @@ ; CHECK-NOT: LV: Found {{.*}} scalar instruction: %ptr.iv.2.next = getelementptr inbounds i8, ptr %ptr.iv.2, i64 1 ; ; CHECK: VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: vector loop: { +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop ; CHECK-EMPTY: -; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: EMIT ir<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8 -; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 -; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr ir<%ptr.iv.2>, ir<1> -; CHECK-NEXT: WIDEN store ir<%ptr.iv.1>, ir<%ptr.iv.2.next> -; CHECK-NEXT: WIDEN ir<%lv> = load ir<%ptr.iv.2> -; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1> -; CHECK-NEXT: WIDEN store ir<%ptr.iv.2>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<%N> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: EMIT ir<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8 +; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 +; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr ir<%ptr.iv.2>, ir<1> +; CHECK-NEXT: WIDEN store ir<%ptr.iv.1>, ir<%ptr.iv.2.next> +; CHECK-NEXT: WIDEN ir<%lv> = load ir<%ptr.iv.2> +; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1> +; CHECK-NEXT: WIDEN store ir<%ptr.iv.2>, ir<%add> +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors ; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: No successors +; CHECK-NEXT: } ; In the test below the pointer phi %ptr.iv.2 is used as ; 1. As a uniform address for the load, and Index: llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll @@ -10,52 +10,60 @@ ; CHECK-LABEL: LV: Checking a loop in 'test_v4_v4m' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> -; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<%2> +; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> +; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<%2> +; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<%2> +; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<%2> +; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -65,52 +73,59 @@ ; CHECK-LABEL: LV: Checking a loop in 'test_v2_v4m' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<%2> +; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<%2> +; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } - -; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir) (using library function: foo_vector_fixed4_mask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<%2> +; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir) (using library function: foo_vector_fixed4_mask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<%2> +; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -119,52 +134,59 @@ ; CHECK-LABEL: LV: Checking a loop in 'test_v2_v4' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<%2> +; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<%2> +; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } - -; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<%2> +; CHECK-NEXT: WIDEN ir<%load> = load ir<%gep> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<%2> +; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } Index: llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -7,54 +7,62 @@ ; CHECK-LABEL: LV: Checking a loop in 'test' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%l> = load ir<%gep.src> -; CHECK-NEXT: WIDEN ir<%conv> = fpext ir<%l> -; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64) -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<%2> +; CHECK-NEXT: WIDEN ir<%l> = load ir<%gep.src> +; CHECK-NEXT: WIDEN ir<%conv> = fpext ir<%l> +; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64) +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<%2> +; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> +; CHECK-NEXT: EMIT vp<%9> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%9> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%l> = load ir<%gep.src> -; CHECK-NEXT: WIDEN ir<%conv> = fpext ir<%l> -; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic) -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<%2> +; CHECK-NEXT: WIDEN ir<%l> = load ir<%gep.src> +; CHECK-NEXT: WIDEN ir<%conv> = fpext ir<%l> +; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic) +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<%2> +; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> +; CHECK-NEXT: EMIT vp<%9> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%9> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; Index: llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -51,27 +51,34 @@ ; CHECK-NEXT: LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK: vector.ph: -; CHECK-NEXT: Successor(s): vector loop -; CHECK: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%i.0> = add vp<[[STEPS]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%B>, ir<%idxprom> -; CHECK-NEXT: WIDEN ir<%1> = load ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr ir<%A>, ir<%idxprom> -; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%add9> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-EMPTY: +; CHECK-NEXT: vector loop: { +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = DERIVED-IV ir<%n> + vp<%1> * ir<-1> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<-1> +; CHECK-NEXT: CLONE ir<%i.0> = add vp<%3>, ir<-1> +; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%B>, ir<%idxprom> +; CHECK-NEXT: WIDEN ir<%1> = load ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr ir<%A>, ir<%idxprom> +; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%add9> +; CHECK-NEXT: EMIT vp<%10> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%10> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] @@ -183,27 +190,34 @@ ; CHECK-NEXT: LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK: vector.ph: -; CHECK-NEXT: Successor(s): vector loop -; CHECK: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%i.0> = add vp<[[STEPS]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%B>, ir<%idxprom> -; CHECK-NEXT: WIDEN ir<%1> = load ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00> -; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr ir<%A>, ir<%idxprom> -; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%conv1> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-EMPTY: +; CHECK-NEXT: vector loop: { +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = DERIVED-IV ir<%n> + vp<%1> * ir<-1> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<-1> +; CHECK-NEXT: CLONE ir<%i.0> = add vp<%3>, ir<-1> +; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%B>, ir<%idxprom> +; CHECK-NEXT: WIDEN ir<%1> = load ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00> +; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr ir<%A>, ir<%idxprom> +; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%conv1> +; CHECK-NEXT: EMIT vp<%10> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%10> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] Index: llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -5,30 +5,34 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-LABEL: 'test_chained_first_order_recurrences_1' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%for.1.next> = load ir<%gep.ptr> -; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1> ir<%for.1.next> -; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice ir<%for.2> vp<[[FOR1_SPLICE]]> -; CHECK-NEXT: WIDEN ir<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: WIDEN store ir<%gep.ptr>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1000> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<%7> +; CHECK-NEXT: vp<%4> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr ir<%ptr>, vp<%4> +; CHECK-NEXT: WIDEN ir<%for.1.next> = load ir<%gep.ptr> +; CHECK-NEXT: EMIT vp<%7> = first-order splice ir<%for.1> ir<%for.1.next> +; CHECK-NEXT: EMIT vp<%8> = first-order splice ir<%for.2> vp<%7> +; CHECK-NEXT: WIDEN ir<%add> = add vp<%7>, vp<%8> +; CHECK-NEXT: WIDEN store ir<%gep.ptr>, ir<%add> +; CHECK-NEXT: EMIT vp<%10> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%10> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -54,37 +58,39 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-LABEL: 'test_chained_first_order_recurrences_3' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%for.1.next> = load ir<%gep.ptr> -; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1> ir<%for.1.next> -; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice ir<%for.2> vp<[[FOR1_SPLICE]]> -; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice ir<%for.3> vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: WIDEN ir<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<[[FOR3_SPLICE]]> -; CHECK-NEXT: WIDEN store ir<%gep.ptr>, ir<%add.2> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1000> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<%8> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.3> = phi ir<33>, vp<%9> +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr ir<%ptr>, vp<%5> +; CHECK-NEXT: WIDEN ir<%for.1.next> = load ir<%gep.ptr> +; CHECK-NEXT: EMIT vp<%8> = first-order splice ir<%for.1> ir<%for.1.next> +; CHECK-NEXT: EMIT vp<%9> = first-order splice ir<%for.2> vp<%8> +; CHECK-NEXT: EMIT vp<%10> = first-order splice ir<%for.3> vp<%9> +; CHECK-NEXT: WIDEN ir<%add.1> = add vp<%8>, vp<%9> +; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<%10> +; CHECK-NEXT: WIDEN store ir<%gep.ptr>, ir<%add.2> +; CHECK-NEXT: EMIT vp<%13> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%13> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } - -; CHECK-NOT: vector.body: ; entry: br label %loop Index: llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -9,68 +9,72 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize { ; CHECK-LABEL: sink_replicate_region_1 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): pred.load -; CHECK-EMPTY: -; CHECK-NEXT: pred.load: { -; CHECK-NEXT: pred.load.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) -; CHECK-NEXT: Successor(s): pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN ir<%conv> = sext vp<[[PRED1]]> -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0> ir<%conv> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.2 +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, ir<20001> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: EMIT vp<%6> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: Successor(s): pred.load +; CHECK-EMPTY: +; CHECK-NEXT: pred.load: { +; CHECK-NEXT: pred.load.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.if: +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) +; CHECK-NEXT: Successor(s): pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%lv> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.0 +; CHECK-EMPTY: +; CHECK-NEXT: loop.0: +; CHECK-NEXT: WIDEN ir<%conv> = sext vp<%9> +; CHECK-NEXT: EMIT vp<%11> = first-order splice ir<%0> ir<%conv> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%11>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%16> = ir<%rem> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.2 +; CHECK-EMPTY: +; CHECK-NEXT: loop.2: +; CHECK-NEXT: EMIT vp<%17> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%17> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -98,49 +102,53 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-LABEL: sink_replicate_region_2 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur> ir<%recur.next> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.1 +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, ir<20001> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%5> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> +; CHECK-NEXT: EMIT vp<%7> = first-order splice ir<%recur> ir<%recur.next> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%5> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%7>, ir<%x> +; CHECK-NEXT: vp<%9> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<%9> +; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%13> = ir<%rem> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.1 +; CHECK-EMPTY: +; CHECK-NEXT: loop.1: +; CHECK-NEXT: EMIT vp<%14> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%14> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -166,49 +174,53 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-LABEL: sink_replicate_region_3_reduction ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> -; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> -; CHECK-NEXT: EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]> vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur> ir<%recur.next> -; CHECK-NEXT: Successor(s): pred.srem -; CHECK-EMPTY: -; CHECK-NEXT: pred.srem: { -; CHECK-NEXT: pred.srem.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.srem.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V) -; CHECK-NEXT: Successor(s): pred.srem.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.srem.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, ir<20001> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> +; CHECK-NEXT: EMIT vp<%5> = WIDEN-CANONICAL-INDUCTION vp<%2> +; CHECK-NEXT: EMIT vp<%6> = icmp ule vp<%5> vp<%1> +; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> +; CHECK-NEXT: EMIT vp<%8> = first-order splice ir<%recur> ir<%recur.next> +; CHECK-NEXT: Successor(s): pred.srem +; CHECK-EMPTY: +; CHECK-NEXT: pred.srem: { +; CHECK-NEXT: pred.srem.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.srem.if: +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%8>, ir<%x> (S->V) +; CHECK-NEXT: Successor(s): pred.srem.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.srem.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%10> = ir<%rem> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.0 +; CHECK-EMPTY: +; CHECK-NEXT: loop.0: +; CHECK-NEXT: WIDEN ir<%add> = add vp<%10>, ir<%recur.next> +; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> +; CHECK-NEXT: EMIT vp<%13> = select vp<%6> ir<%and.red.next> ir<%and.red> +; CHECK-NEXT: EMIT vp<%14> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%14> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<%recur.next> -; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> -; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]> ir<%and.red.next> ir<%and.red> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out i32 %res = ir<%and.red.next> @@ -239,72 +251,76 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr %ptr, ptr noalias %dst) optsize { ; CHECK-LABEL: sink_replicate_region_4_requires_split_at_end_of_block ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: Successor(s): pred.load -; CHECK-EMPTY: -; CHECK-NEXT: pred.load: { -; CHECK-NEXT: pred.load.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) -; CHECK-NEXT: Successor(s): pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN ir<%conv> = sext vp<[[PRED]]> -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0> ir<%conv> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> -; CHECK-NEXT: REPLICATE ir<%conv.lv.2> = sext ir<%lv.2> -; CHECK-NEXT: REPLICATE ir<%add.1> = add ir<%conv>, ir<%rem> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%add> = add ir<%add.1>, ir<%conv.lv.2> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%rem> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.2> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.3 +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, ir<20001> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: EMIT vp<%6> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<%5> +; CHECK-NEXT: Successor(s): pred.load +; CHECK-EMPTY: +; CHECK-NEXT: pred.load: { +; CHECK-NEXT: pred.load.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.if: +; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) +; CHECK-NEXT: Successor(s): pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%lv> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.0 +; CHECK-EMPTY: +; CHECK-NEXT: loop.0: +; CHECK-NEXT: WIDEN ir<%conv> = sext vp<%9> +; CHECK-NEXT: EMIT vp<%11> = first-order splice ir<%0> ir<%conv> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%11>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> +; CHECK-NEXT: REPLICATE ir<%conv.lv.2> = sext ir<%lv.2> +; CHECK-NEXT: REPLICATE ir<%add.1> = add ir<%conv>, ir<%rem> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%add> = add ir<%add.1>, ir<%conv.lv.2> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%19> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%20> = ir<%lv.2> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.3 +; CHECK-EMPTY: +; CHECK-NEXT: loop.3: +; CHECK-NEXT: EMIT vp<%21> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%21> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK: loop.3: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -336,52 +352,56 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias %dst.2, i32 %x, i8 %y) optsize { ; CHECK-LABEL: sink_replicate_region_after_replicate_region ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur> ir<%recur.next> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep> -; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem.div> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.3 +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, WIDEN ir<%recur.next> = sext ir<%y> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%5> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: WIDEN ir<%recur.next> = sext ir<%y> +; CHECK-NEXT: EMIT vp<%7> = first-order splice ir<%recur> ir<%recur.next> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%5> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: vp<%8> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<%7>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<%8> +; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep> +; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<%8> +; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%15> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%16> = ir<%rem.div> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.3 +; CHECK-EMPTY: +; CHECK-NEXT: loop.3: +; CHECK-NEXT: EMIT vp<%17> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%17> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK-NEXT: loop.3: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -409,49 +429,53 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias %dst) { ; CHECK-LABEL: need_new_block_after_sinking_pr56146 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<[[L:%.+]]> -; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1> -; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]> -; CHECK-NEXT: CLONE ir<[[L]]> = load ir<%src> -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn> ir<[[L]]> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[CMP]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = ir<%val> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, ir<3> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<%l> +; CHECK-NEXT: vp<%4> = DERIVED-IV ir<2> + vp<%2> * ir<1> +; CHECK-NEXT: EMIT vp<%5> = WIDEN-CANONICAL-INDUCTION vp<%2> +; CHECK-NEXT: EMIT vp<%6> = icmp ule vp<%5> vp<%1> +; CHECK-NEXT: CLONE ir<%l> = load ir<%src> +; CHECK-NEXT: EMIT vp<%8> = first-order splice ir<%.pn> ir<%l> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<%8>, ir<%x> +; CHECK-NEXT: vp<%10> = SCALAR-STEPS vp<%4>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<%10> +; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%13> = ir<%val> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.1 +; CHECK-EMPTY: +; CHECK-NEXT: loop.1: +; CHECK-NEXT: EMIT vp<%14> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%14> vp<%0> ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.1 +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; Index: llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -37,41 +37,50 @@ ; Check for crash exposed by D76992. ; CHECK-LABEL: 'test' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13> -; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20> -; CHECK-NEXT: Successor(s): pred.store +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, ir<14> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13> +; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20> +; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[COND]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%4> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep> -; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: vp<%7> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<%7> +; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep> +; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.0 +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successor +; CHECK-NEXT: loop.0: +; CHECK-NEXT: EMIT vp<%10> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%10> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: No successors ; CHECK-NEXT: } define void @test(ptr %ptr) { entry: Index: llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -5,23 +5,32 @@ ; DBG-LABEL: 'test_scalarize_call' ; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { -; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; DBG-EMPTY: -; DBG-NEXT: vector.ph: -; DBG-NEXT: Successor(s): vector loop +; DBG-NEXT: Live-in vp<%0> = vector-trip-count ; DBG-EMPTY: ; DBG-NEXT: vector loop: { -; DBG-NEXT: vector.body: -; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%start> + vp<[[CAN_IV]]> * ir<1> -; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) -; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%dst>, vp<[[IV_STEPS]]> -; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx> -; DBG-NEXT: EMIT vp<[[INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; DBG-NEXT: EMIT branch-on-count vp<[[INC]]> vp<[[VEC_TC]]> +; DBG-NEXT: vector.ph: +; DBG-NEXT: Successor(s): vector countable loop +; DBG-EMPTY: +; DBG-NEXT: vector countable loop vp<%1> = 0, ir<1000> { +; DBG-EMPTY: +; DBG-NEXT: vector.body: +; DBG-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; DBG-NEXT: vp<%2> = DERIVED-IV ir<%start> + vp<%1> * ir<1> +; DBG-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> +; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<%3>, ir<65535>) +; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%dst>, vp<%3> +; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx> +; DBG-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; DBG-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; DBG-NEXT: No successors +; DBG-NEXT: } +; DBG-NEXT: Successor(s): middle.block +; DBG-EMPTY: +; DBG-NEXT: middle.block: ; DBG-NEXT: No successors ; DBG-NEXT: } +; DBG-NEXT: No successors +; DBG-NEXT: } ; define void @test_scalarize_call(i32 %start, ptr %dst) { ; CHECK-LABEL: @test_scalarize_call( @@ -61,49 +70,54 @@ ; DBG-LABEL: 'test_scalarize_with_branch_cond' - -; DBG: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; DBG-EMPTY: -; DBG-NEXT: vector.ph: -; DBG-NEXT: Successor(s): vector loop +; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { +; DBG-NEXT: Live-in vp<%0> = vector-trip-count ; DBG-EMPTY: ; DBG-NEXT: vector loop: { -; DBG-NEXT: vector.body: -; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir + vp<[[CAN_IV]]> * ir -; DBG-NEXT: vp<[[STEPS1:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir -; DBG-NEXT: Successor(s): pred.store -; DBG-EMPTY: -; DBG-NEXT: pred.store: { -; DBG-NEXT: pred.store.entry: -; DBG-NEXT: BRANCH-ON-MASK vp<[[STEPS1]]> -; DBG-NEXT: Successor(s): pred.store.if, pred.store.continue -; DBG-EMPTY: -; DBG-NEXT: pred.store.if: -; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; DBG-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<[[STEPS2]]> -; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src> -; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS2]]> -; DBG-NEXT: CLONE store ir<%l>, ir<%gep.dst> -; DBG-NEXT: Successor(s): pred.store.continue -; DBG-EMPTY: -; DBG-NEXT: pred.store.continue: -; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = ir<%l> +; DBG-NEXT: vector.ph: +; DBG-NEXT: Successor(s): vector countable loop +; DBG-EMPTY: +; DBG-NEXT: vector countable loop vp<%1> = 0, ir<1000> { +; DBG-EMPTY: +; DBG-NEXT: vector.body: +; DBG-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; DBG-NEXT: vp<%2> = DERIVED-IV ir + vp<%1> * ir +; DBG-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir +; DBG-NEXT: Successor(s): pred.store +; DBG-EMPTY: +; DBG-NEXT: pred.store: { +; DBG-NEXT: pred.store.entry: +; DBG-NEXT: BRANCH-ON-MASK vp<%3> +; DBG-NEXT: Successor(s): pred.store.if, pred.store.continue +; DBG-EMPTY: +; DBG-NEXT: pred.store.if: +; DBG-NEXT: vp<%4> = SCALAR-STEPS vp<%1>, ir<1> +; DBG-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<%4> +; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src> +; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<%4> +; DBG-NEXT: CLONE store ir<%l>, ir<%gep.dst> +; DBG-NEXT: Successor(s): pred.store.continue +; DBG-EMPTY: +; DBG-NEXT: pred.store.continue: +; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%l> +; DBG-NEXT: No successors +; DBG-NEXT: } +; DBG-NEXT: Successor(s): cond.false.1 +; DBG-EMPTY: +; DBG-NEXT: cond.false.1: +; DBG-NEXT: EMIT vp<%10> = VF * UF +(nuw) vp<%1> +; DBG-NEXT: EMIT branch-on-count vp<%10> vp<%0> ; DBG-NEXT: No successors ; DBG-NEXT: } -; DBG-NEXT: Successor(s): cond.false.1 +; DBG-NEXT: Successor(s): middle.block ; DBG-EMPTY: -; DBG-NEXT: cond.false.1: -; DBG-NEXT: EMIT vp<[[CAN_IV_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; DBG-NEXT: EMIT branch-on-count vp<[[CAN_IV_INC]]> vp<[[VEC_TC]]> +; DBG-NEXT: middle.block: ; DBG-NEXT: No successors ; DBG-NEXT: } -; DBG-NEXT: Successor(s): middle.block -; DBG-EMPTY: -; DBG-NEXT: middle.block: ; DBG-NEXT: No successors ; DBG-NEXT: } + define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) { ; CHECK-LABEL: @test_scalarize_with_branch_cond( ; CHECK: vector.body: @@ -167,26 +181,30 @@ ; DBG-LABEL: 'first_order_recurrence_using_induction' ; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { -; DBG-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; DBG-EMPTY: -; DBG-NEXT: vector.ph: -; DBG-NEXT: Successor(s): vector loop +; DBG-NEXT: Live-in vp<%0> = vector-trip-count ; DBG-EMPTY: ; DBG-NEXT: vector loop: { -; DBG-NEXT: vector.body: -; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]> -; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<1> (truncated to i32) -; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for> vp<[[SCALAR_STEPS]]> -; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst> -; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]> vp<[[VTC]]> +; DBG-NEXT: vector.ph: +; DBG-NEXT: Successor(s): vector countable loop +; DBG-EMPTY: +; DBG-NEXT: vector countable loop vp<%1> = 0, inf { +; DBG-EMPTY: +; DBG-NEXT: vector.body: +; DBG-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<%4> +; DBG-NEXT: vp<%3> = DERIVED-IV ir<0> + vp<%1> * ir<1> (truncated to i32) +; DBG-NEXT: vp<%4> = SCALAR-STEPS vp<%3>, ir<1> +; DBG-NEXT: EMIT vp<%5> = first-order splice ir<%for> vp<%4> +; DBG-NEXT: CLONE store vp<%5>, ir<%dst> +; DBG-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; DBG-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; DBG-NEXT: No successors +; DBG-NEXT: } +; DBG-NEXT: Successor(s): middle.block +; DBG-EMPTY: +; DBG-NEXT: middle.block: ; DBG-NEXT: No successors ; DBG-NEXT: } -; DBG-NEXT: Successor(s): middle.block -; DBG-EMPTY: -; DBG-NEXT: middle.block: ; DBG-NEXT: No successors ; DBG-NEXT: } Index: llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -7,33 +7,43 @@ ; Verify that -vplan-print-in-dot-format option works. define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounwind uwtable { -; CHECK: digraph VPlan { -; CHECK-NEXT: graph [labelloc=t, fontsize=30; label="Vectorization Plan\nInitial VPlan for VF=\{4\},UF\>=1"] -; CHECK-NEXT: node [shape=rect, fontname=Courier, fontsize=30] -; CHECK-NEXT: edge [fontname=Courier, fontsize=30] -; CHECK-NEXT: compound=true -; CHECK-NEXT: N0 [label = -; CHECK-NEXT: "vector.ph:\l" + -; CHECK-NEXT: "Successor(s): vector loop\l" -; CHECK-NEXT: ] -; CHECK-NEXT: N0 -> N1 [ label="" lhead=cluster_N2] -; CHECK-NEXT: subgraph cluster_N2 { -; CHECK-NEXT: fontname=Courier -; CHECK-NEXT: label="\ vector loop" -; CHECK-NEXT: N1 [label = -; CHECK-NEXT: "vector.body:\l" + -; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION\l" + -; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" + -; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, vp\<[[STEPS]]\>\l" + -; CHECK-NEXT: " WIDEN ir\<%lv\> = load ir\<%arrayidx\>\l" + -; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + -; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr ir\<%x\>, vp\<[[STEPS]]\>\l" + -; CHECK-NEXT: " WIDEN store ir\<%arrayidx2\>, ir\<%call\>\l" + -; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT:%.+]]\> = VF * UF +(nuw) vp\<[[CAN_IV]]\>\l" + -; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\> vp\<{{.+}}\>\l" + -; CHECK-NEXT: "No successors\l" -; CHECK-NEXT: ] -; +; CHECK: digraph VPlan { +; CHECK-NEXT: graph [labelloc=t, fontsize=30; label="Vectorization Plan\nInitial VPlan for VF=\{4\},UF\>=1"] +; CHECK-NEXT: node [shape=rect, fontname=Courier, fontsize=30] +; CHECK-NEXT: edge [fontname=Courier, fontsize=30] +; CHECK-NEXT: compound=true +; CHECK-NEXT: subgraph cluster_N0 { +; CHECK-NEXT: fontname=Courier +; CHECK-NEXT: label="\ vector loop" +; CHECK-NEXT: N1 [label = +; CHECK-NEXT: "vector.ph:\l" + +; CHECK-NEXT: "Successor(s): vector countable loop\l" +; CHECK-NEXT: ] +; CHECK-NEXT: N1 -> N2 [ label="" lhead=cluster_N3] +; CHECK-NEXT: subgraph cluster_N3 { +; CHECK-NEXT: fontname=Courier +; CHECK-NEXT: label="\ vector countable loop" +; CHECK-NEXT: N2 [label = +; CHECK-NEXT: "vector.body:\l" + +; CHECK-NEXT: " EMIT vp\<%1\> = CANONICAL-INDUCTION\l" + +; CHECK-NEXT: " vp\<%2\> = SCALAR-STEPS vp\<%1\>, ir\<1\>\l" + +; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, vp\<%2\>\l" + +; CHECK-NEXT: " WIDEN ir\<%lv\> = load ir\<%arrayidx\>\l" + +; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + +; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr ir\<%x\>, vp\<%2\>\l" + +; CHECK-NEXT: " WIDEN store ir\<%arrayidx2\>, ir\<%call\>\l" + +; CHECK-NEXT: " EMIT vp\<%7\> = VF * UF +(nuw) vp\<%1\>\l" + +; CHECK-NEXT: " EMIT branch-on-count vp\<%7\> vp\<%0\>\l" + +; CHECK-NEXT: "No successors\l" +; CHECK-NEXT: ] +; CHECK-NEXT: } +; CHECK-NEXT: N2 -> N4 [ label="" ltail=cluster_N3] +; CHECK-NEXT: N4 [label = +; CHECK-NEXT: "middle.block:\l" + +; CHECK-NEXT: "No successors\l" +; CHECK-NEXT: ] +; CHECK-NEXT: } +; CHECK-NEXT: } entry: %cmp6 = icmp sgt i64 %n, 0 br i1 %cmp6, label %for.body, label %for.end Index: llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll +++ llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll @@ -4,25 +4,29 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-LABEL: LV: Checking a loop in 'iv_no_binary_op_in_descriptor' ; CHECK: VPlan 'Initial VPlan for VF={8},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next.p, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%dst>, vp<[[STEPS:%.+]]> -; CHECK-NEXT: WIDEN store ir<%gep>, ir<%iv> -; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]> vp<[[VEC_TC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next.p, ir<1> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%dst>, vp<%3> +; CHECK-NEXT: WIDEN store ir<%gep>, ir<%iv> +; CHECK-NEXT: EMIT vp<%5> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%5> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; Index: llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll +++ llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll @@ -7,39 +7,44 @@ define void @foo(i64 %n) { ; CHECK: VPlan 'HCFGBuilder: Plain CFG -; CHECK-NEXT: { -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): outer.header +; CHECK-NEXT: for UF>=1' { +; CHECK-NEXT: : { +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): outer.header ; CHECK-EMPTY: -; CHECK-NEXT: outer.header: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next> -; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2> ir<0> ir<%outer.iv> -; CHECK-NEXT: EMIT store ir<%outer.iv> ir<%gep.1> -; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv> ir<%n> -; CHECK-NEXT: Successor(s): inner +; CHECK-NEXT: outer.header { ; CHECK-EMPTY: -; CHECK-NEXT: inner: { -; CHECK-NEXT: inner: -; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next> -; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr> ir<0> ir<%inner.iv> ir<%outer.iv> -; CHECK-NEXT: EMIT store ir<%add> ir<%gep.2> -; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv> ir<1> -; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next> ir<8> -; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): outer.latch +; CHECK-NEXT: vector.body: +; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next> +; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2> ir<0> ir<%outer.iv> +; CHECK-NEXT: EMIT store ir<%outer.iv> ir<%gep.1> +; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv> ir<%n> +; CHECK-NEXT: Successor(s): inner ; CHECK-EMPTY: -; CHECK-NEXT: outer.latch: -; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv> ir<1> -; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next> ir<8> -; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): exit +; CHECK-NEXT: inner { +; CHECK-EMPTY: +; CHECK-NEXT: inner: +; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next> +; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr> ir<0> ir<%inner.iv> ir<%outer.iv> +; CHECK-NEXT: EMIT store ir<%add> ir<%gep.2> +; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv> ir<1> +; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next> ir<8> +; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): outer.latch ; CHECK-EMPTY: -; CHECK-NEXT: exit: +; CHECK-NEXT: outer.latch: +; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv> ir<1> +; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next> ir<8> +; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): exit +; CHECK-EMPTY: +; CHECK-NEXT: exit: +; CHECK-NEXT: No successors +; CHECK-NEXT: } ; CHECK-NEXT: No successors ; CHECK-NEXT: } entry: Index: llvm/test/Transforms/LoopVectorize/vplan-printing.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -9,27 +9,31 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounwind uwtable { ; CHECK-LABEL: Checking a loop in 'print_call_and_memory' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) -; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<%n> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<%2> +; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) (using vector intrinsic) +; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%x>, vp<%2> +; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%call> +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -55,30 +59,34 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ptr %z) nounwind uwtable { ; CHECK-LABEL: Checking a loop in 'print_widen_gep_and_select' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv> -; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z> -; CHECK-NEXT: WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01> -; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel> -; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<%n> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv> +; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z> +; CHECK-NEXT: WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01> +; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel> +; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%x>, vp<%3> +; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%add> +; CHECK-NEXT: EMIT vp<%10> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%10> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -106,26 +114,30 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-LABEL: Checking a loop in 'print_reduction' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> -; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<%n> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<%3> +; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out float %red.next.lcssa = ir<%red.next> @@ -151,26 +163,30 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr noalias %dst) { ; CHECK-LABEL: Checking a loop in 'print_reduction_with_invariant_store' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> -; CHECK-NEXT: vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<[[IV]]> -; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> -; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<%n> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<%3> +; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop) +; CHECK-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%7> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -195,46 +211,50 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-LABEL: Checking a loop in 'print_replicate_predicated_phi' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5> -; CHECK-NEXT: Successor(s): pred.udiv -; CHECK-EMPTY: -; CHECK-NEXT: pred.udiv: { -; CHECK-NEXT: pred.udiv.entry: -; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp> -; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.udiv.if: -; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V) -; CHECK-NEXT: Successor(s): pred.udiv.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.udiv.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%tmp4> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<%n> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5> +; CHECK-NEXT: Successor(s): pred.udiv +; CHECK-EMPTY: +; CHECK-NEXT: pred.udiv: { +; CHECK-NEXT: pred.udiv.entry: +; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp> +; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.udiv.if: +; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<%3> (S->V) +; CHECK-NEXT: Successor(s): pred.udiv.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.udiv.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%tmp4> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): if.then.0 +; CHECK-EMPTY: +; CHECK-NEXT: if.then.0: +; CHECK-NEXT: EMIT vp<%7> = not ir<%cmp> +; CHECK-NEXT: BLEND %d = ir<0>/vp<%7> vp<%6>/ir<%cmp> +; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<%3> +; CHECK-NEXT: WIDEN store ir<%idx>, ir<%d> +; CHECK-NEXT: EMIT vp<%10> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%10> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): if.then.0 -; CHECK-EMPTY: -; CHECK-NEXT: if.then.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%cmp> -; CHECK-NEXT: BLEND %d = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/ir<%cmp> -; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN store ir<%idx>, ir<%d> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -268,36 +288,40 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-LABEL: Checking a loop in 'print_interleave_groups' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<4> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<4> -; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr ir<@AB>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0> -; CHECK-NEXT: ir<%AB.0> = load from index 0 -; CHECK-NEXT: ir<%AB.1> = load from index 1 -; CHECK-NEXT: ir<%AB.3> = load from index 3 -; CHECK-NEXT: CLONE ir<%iv.plus.3> = add vp<[[STEPS]]>, ir<3> -; CHECK-NEXT: WIDEN ir<%add> = add ir<%AB.0>, ir<%AB.1> -; CHECK-NEXT: CLONE ir<%gep.CD.3> = getelementptr ir<@CD>, ir<0>, ir<%iv.plus.3> -; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at , ir<%gep.CD.3> -; CHECK-NEXT: store ir<%add> to index 0 -; CHECK-NEXT: store ir<1> to index 1 -; CHECK-NEXT: store ir<2> to index 2 -; CHECK-NEXT: store ir<%AB.3> to index 3 -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1024> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = DERIVED-IV ir<0> + vp<%1> * ir<4> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<4> +; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr ir<@AB>, ir<0>, vp<%3> +; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0> +; CHECK-NEXT: ir<%AB.0> = load from index 0 +; CHECK-NEXT: ir<%AB.1> = load from index 1 +; CHECK-NEXT: ir<%AB.3> = load from index 3 +; CHECK-NEXT: CLONE ir<%iv.plus.3> = add vp<%3>, ir<3> +; CHECK-NEXT: WIDEN ir<%add> = add ir<%AB.0>, ir<%AB.1> +; CHECK-NEXT: CLONE ir<%gep.CD.3> = getelementptr ir<@CD>, ir<0>, ir<%iv.plus.3> +; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at , ir<%gep.CD.3> +; CHECK-NEXT: store ir<%add> to index 0 +; CHECK-NEXT: store ir<1> to index 1 +; CHECK-NEXT: store ir<2> to index 2 +; CHECK-NEXT: store ir<%AB.3> to index 3 +; CHECK-NEXT: EMIT vp<%11> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%11> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -335,33 +359,38 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-LABEL: Checking a loop in 'print_fmuladd_strict' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%l.a> = load ir<%arrayidx> -; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%l.b> = load ir<%arrayidx2> -; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a> ir<%l.b> -; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<%n> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<%3> +; CHECK-NEXT: WIDEN ir<%l.a> = load ir<%arrayidx> +; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%b>, vp<%3> +; CHECK-NEXT: WIDEN ir<%l.b> = load ir<%arrayidx2> +; CHECK-NEXT: EMIT vp<%8> = fmul nnan ninf nsz ir<%l.a> ir<%l.b> +; CHECK-NEXT: REDUCE ir<%muladd> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<%8>) +; CHECK-NEXT: EMIT vp<%10> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%10> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out float %muladd.lcssa = ir<%muladd> -; CHECK-NEXT:} +; CHECK-NEXT: } +; entry: br label %for.body @@ -385,54 +414,58 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 { ; CHECK-LABEL: Checking a loop in 'debug_loc_vpinstruction' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%isd> = getelementptr ir<%asd>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%lsd> = load ir<%isd> -; CHECK-NEXT: WIDEN ir<%psd> = add ir<%lsd>, ir<23> -; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100> -; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200> -; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not ir<%cmp1>, !dbg /tmp/s.c:5:3 -; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]> ir<%cmp2> ir, !dbg /tmp/s.c:5:21 -; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]> ir<%cmp1> -; CHECK-NEXT: Successor(s): pred.sdiv -; CHECK-EMPTY: -; CHECK-NEXT: pred.sdiv: { -; CHECK-NEXT: pred.sdiv.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[OR1]]> -; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.sdiv.if: -; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V) -; CHECK-NEXT: Successor(s): pred.sdiv.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.sdiv.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = ir<%sd1> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): if.then.0 -; CHECK-EMPTY: -; CHECK-NEXT: if.then.0: -; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp2> -; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]> vp<[[NOT2]]> ir -; CHECK-NEXT: BLEND %ysd.0 = vp<[[PHI]]>/vp<[[OR1]]> ir<%psd>/vp<[[SEL2]]> -; CHECK-NEXT: WIDEN store ir<%isd>, ir<%ysd.0> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors -; CHECK-NEXT:} -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<128> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%isd> = getelementptr ir<%asd>, vp<%2> +; CHECK-NEXT: WIDEN ir<%lsd> = load ir<%isd> +; CHECK-NEXT: WIDEN ir<%psd> = add ir<%lsd>, ir<23> +; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100> +; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200> +; CHECK-NEXT: EMIT vp<%8> = not ir<%cmp1>, !dbg /tmp/s.c:5:3 +; CHECK-NEXT: EMIT vp<%9> = select vp<%8> ir<%cmp2> ir, !dbg /tmp/s.c:5:21 +; CHECK-NEXT: EMIT vp<%10> = or vp<%9> ir<%cmp1> +; CHECK-NEXT: Successor(s): pred.sdiv +; CHECK-EMPTY: +; CHECK-NEXT: pred.sdiv: { +; CHECK-NEXT: pred.sdiv.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%10> +; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.sdiv.if: +; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V) +; CHECK-NEXT: Successor(s): pred.sdiv.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.sdiv.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%12> = ir<%sd1> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): if.then.0 +; CHECK-EMPTY: +; CHECK-NEXT: if.then.0: +; CHECK-NEXT: EMIT vp<%13> = not ir<%cmp2>, !dbg /tmp/s.c:5:21 +; CHECK-NEXT: EMIT vp<%14> = select vp<%8> vp<%13> ir, !dbg /tmp/s.c:5:21 +; CHECK-NEXT: BLEND %ysd.0 = vp<%12>/vp<%10> ir<%psd>/vp<%14> +; CHECK-NEXT: WIDEN store ir<%isd>, ir<%ysd.0> +; CHECK-NEXT: EMIT vp<%16> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%16> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: No successors +; CHECK-NEXT: } ; CHECK-NEXT: No successors -; CHECK-NEXT:} +; CHECK-NEXT: } ; entry: br label %loop @@ -470,30 +503,34 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-LABEL: Checking a loop in 'print_expand_scev' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: EMIT vp<[[EXP_SCEV:%.+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060)) -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION\l" + -; CHECK-NEXT: " %iv = phi %iv.next, 0\l" + -; CHECK-NEXT: " ir<%v2>, vp<[[EXP_SCEV]]> -; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * vp<[[EXP_SCEV]]> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, vp<[[EXP_SCEV]]> -; CHECK-NEXT: WIDEN ir<%v3> = add ir<%v2>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep> -; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]> vp<[[VTC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<%1> = EXPAND SCEV (1 + (%y /u 492802768830814060)) +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION\l" + +; CHECK-NEXT: " %iv = phi %iv.next, 0\l" + +; CHECK-NEXT: " ir<%v2>, vp<%1> +; CHECK-NEXT: vp<%4> = DERIVED-IV ir<0> + vp<%2> * vp<%1> +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%4>, vp<%1> +; CHECK-NEXT: WIDEN ir<%v3> = add ir<%v2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<%5> +; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep> +; CHECK-NEXT: EMIT vp<%9> = VF * UF +(nuw) vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%9> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -520,26 +557,30 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-LABEL: Checking a loop in 'print_exit_value' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off> -; CHECK-NEXT: WIDEN store ir<%gep>, ir<0> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<1000> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<%3> +; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off> +; CHECK-NEXT: WIDEN store ir<%gep>, ir<0> +; CHECK-NEXT: EMIT vp<%6> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%6> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out i32 %lcssa = ir<%add> Index: llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -8,43 +8,48 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-LABEL: sink_with_sideeffects ; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2> -; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2> -; CHECK-NEXT: Successor(s): pred.store - -; CHECK: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK ir<%c> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue - -; CHECK: pred.store.if: -; CHECK-NEXT: CLONE ir<%tmp4> = zext ir<%tmp3> -; CHECK-NEXT: CLONE ir<%tmp5> = trunc ir<%tmp4> -; CHECK-NEXT: CLONE store ir<%tmp5>, ir<%tmp2> -; CHECK-NEXT: Successor(s): pred.store.continue - -; CHECK: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<0> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<%2> +; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2> +; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK ir<%c> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: CLONE ir<%tmp4> = zext ir<%tmp3> +; CHECK-NEXT: CLONE ir<%tmp5> = trunc ir<%tmp4> +; CHECK-NEXT: CLONE store ir<%tmp5>, ir<%tmp2> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): if.then.0 +; CHECK-EMPTY: +; CHECK-NEXT: if.then.0: +; CHECK-NEXT: EMIT vp<%9> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%9> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } - -; CHECK: if.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; Index: llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -11,42 +11,52 @@ ; CHECK-LABEL: LV: Checking a loop in 'sink1' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): pred.store - -; CHECK: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue - -; CHECK: pred.store.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> -; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10> -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]] -; CHECK-NEXT: REPLICATE ir<%mul> = mul ir<2>, ir<%add> -; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue - -; CHECK: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%4> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> +; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%mul> = mul ir<2>, ir<%add> +; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%12> = ir<%lv.b> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.1 +; CHECK-EMPTY: +; CHECK-NEXT: loop.1: +; CHECK-NEXT: EMIT vp<%13> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%13> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } - -; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -73,58 +83,69 @@ } ; CHECK-LABEL: LV: Checking a loop in 'sink2' -; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): pred.load - -; CHECK: pred.load: { -; CHECK-NEXT: pred.load.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue - -; CHECK: pred.load.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> -; CHECK-NEXT: Successor(s): pred.load.continue - -; CHECK: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> -; CHECK-NEXT: No successors -; CHECK-NEXT: } - -; CHECK: loop.0: -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<2> -; CHECK-NEXT: Successor(s): pred.store - -; CHECK: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue - -; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue - -; CHECK: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: Successor(s): pred.load +; CHECK-EMPTY: +; CHECK-NEXT: pred.load: { +; CHECK-NEXT: pred.load.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%4> +; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.if: +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> +; CHECK-NEXT: Successor(s): pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%8> = ir<%lv.b> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.0 +; CHECK-EMPTY: +; CHECK-NEXT: loop.0: +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<2> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%4> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE ir<%add> = add vp<%8>, ir<10> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.1 +; CHECK-EMPTY: +; CHECK-NEXT: loop.1: +; CHECK-NEXT: EMIT vp<%13> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%13> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } - -; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -152,57 +173,68 @@ ; CHECK-LABEL: LV: Checking a loop in 'sink3' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): pred.load - -; CHECK: pred.load: { -; CHECK-NEXT: pred.load.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue - -; CHECK: pred.load.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) -; CHECK-NEXT: Successor(s): pred.load.continue - -; CHECK: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> -; CHECK-NEXT: No successors -; CHECK-NEXT: } - -; CHECK: loop.0: -; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<10> -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add> -; CHECK-NEXT: Successor(s): pred.store - -; CHECK: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue - -; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue - -; CHECK: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: Successor(s): pred.load +; CHECK-EMPTY: +; CHECK-NEXT: pred.load: { +; CHECK-NEXT: pred.load.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%4> +; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.if: +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) +; CHECK-NEXT: Successor(s): pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%8> = ir<%lv.b> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.0 +; CHECK-EMPTY: +; CHECK-NEXT: loop.0: +; CHECK-NEXT: WIDEN ir<%add> = add vp<%8>, ir<10> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%4> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.1 +; CHECK-EMPTY: +; CHECK-NEXT: loop.1: +; CHECK-NEXT: EMIT vp<%13> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%13> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } - -; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -232,45 +264,54 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-LABEL: LV: Checking a loop in 'uniform_gep' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 21, %iv.next, ir<1> -; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<21> + vp<[[CAN_IV]]> * ir<1> -; CHECK-NEXT: EMIT vp<[[WIDE_CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_CAN_IV]]> vp<[[BTC]]> -; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr ir<%A>, ir<0> -; CHECK-NEXT: CLONE ir<%lv> = load ir<%gep.A.uniform> -; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k> -; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]> vp<[[NOT2]]> ir -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr ir<%B>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, ir<32> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 21, %iv.next, ir<1> +; CHECK-NEXT: vp<%4> = DERIVED-IV ir<21> + vp<%2> * ir<1> +; CHECK-NEXT: EMIT vp<%5> = WIDEN-CANONICAL-INDUCTION vp<%2> +; CHECK-NEXT: EMIT vp<%6> = icmp ule vp<%5> vp<%1> +; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr ir<%A>, ir<0> +; CHECK-NEXT: CLONE ir<%lv> = load ir<%gep.A.uniform> +; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k> +; CHECK-NEXT: EMIT vp<%10> = not ir<%cmp> +; CHECK-NEXT: EMIT vp<%11> = select vp<%6> vp<%10> ir +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%11> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: vp<%12> = SCALAR-STEPS vp<%4>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr ir<%B>, vp<%12> +; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.then.0 +; CHECK-EMPTY: +; CHECK-NEXT: loop.then.0: +; CHECK-NEXT: EMIT vp<%15> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%15> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.then.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -301,64 +342,73 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg1' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]> ir<%c.1> ir -; CHECK-NEXT: Successor(s): pred.load -; CHECK-EMPTY: -; CHECK-NEXT: pred.load: { -; CHECK-NEXT: pred.load.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> -; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) -; CHECK-NEXT: Successor(s): pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): then.0.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.1> -; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]> vp<[[NOT]]> ir -; CHECK-NEXT: BLEND %p = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> -; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]> vp<[[MASK3]]> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[OR]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> +; CHECK-NEXT: EMIT vp<%7> = select vp<%4> ir<%c.1> ir +; CHECK-NEXT: Successor(s): pred.load +; CHECK-EMPTY: +; CHECK-NEXT: pred.load: { +; CHECK-NEXT: pred.load.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%7> +; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.if: +; CHECK-NEXT: vp<%8> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<%8> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) +; CHECK-NEXT: Successor(s): pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%lv.b> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): then.0.0 +; CHECK-EMPTY: +; CHECK-NEXT: then.0.0: +; CHECK-NEXT: EMIT vp<%12> = not ir<%c.1> +; CHECK-NEXT: EMIT vp<%13> = select vp<%4> vp<%12> ir +; CHECK-NEXT: BLEND %p = ir<0>/vp<%13> vp<%11>/vp<%7> +; CHECK-NEXT: EMIT vp<%15> = or vp<%7> vp<%13> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%15> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): next.0.1 +; CHECK-EMPTY: +; CHECK-NEXT: next.0.1: +; CHECK-NEXT: EMIT vp<%18> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%18> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): next.0.1 -; CHECK-EMPTY: -; CHECK-NEXT: next.0.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -395,66 +445,75 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg2' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> -; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]> ir<%c.0> ir -; CHECK-NEXT: Successor(s): pred.load -; CHECK-EMPTY: -; CHECK-NEXT: pred.load: { -; CHECK-NEXT: pred.load.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> -; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) -; CHECK-NEXT: Successor(s): pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): then.0.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0> -; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]> vp<[[NOT]]> ir -; CHECK-NEXT: BLEND %p = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> -; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]> vp<[[MASK3]]> -; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]> ir<%c.1> ir -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK4]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> +; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> +; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j> +; CHECK-NEXT: EMIT vp<%8> = select vp<%4> ir<%c.0> ir +; CHECK-NEXT: Successor(s): pred.load +; CHECK-EMPTY: +; CHECK-NEXT: pred.load: { +; CHECK-NEXT: pred.load.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%8> +; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.if: +; CHECK-NEXT: vp<%9> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<%9> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) +; CHECK-NEXT: Successor(s): pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%12> = ir<%lv.b> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): then.0.0 +; CHECK-EMPTY: +; CHECK-NEXT: then.0.0: +; CHECK-NEXT: EMIT vp<%13> = not ir<%c.0> +; CHECK-NEXT: EMIT vp<%14> = select vp<%4> vp<%13> ir +; CHECK-NEXT: BLEND %p = ir<0>/vp<%14> vp<%12>/vp<%8> +; CHECK-NEXT: EMIT vp<%16> = or vp<%8> vp<%14> +; CHECK-NEXT: EMIT vp<%17> = select vp<%16> ir<%c.1> ir +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%17> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): then.1.1 +; CHECK-EMPTY: +; CHECK-NEXT: then.1.1: +; CHECK-NEXT: EMIT vp<%20> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%20> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): then.1.1 -; CHECK-EMPTY: -; CHECK-NEXT: then.1.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -498,67 +557,76 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg3' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> -; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]> ir<%c.0> ir -; CHECK-NEXT: Successor(s): pred.load -; CHECK-EMPTY: -; CHECK-NEXT: pred.load: { -; CHECK-NEXT: pred.load.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> -; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> -; CHECK-NEXT: Successor(s): pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): then.0.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0> -; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]> vp<[[NOT]]> ir -; CHECK-NEXT: BLEND %p = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> -; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = or vp<[[MASK2]]> vp<[[MASK3]]> -; CHECK-NEXT: EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]> ir<%c.0> ir -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK5]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<0>, ir<%gep.a> -; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr ir<@c>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.c> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> +; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> +; CHECK-NEXT: EMIT vp<%7> = select vp<%4> ir<%c.0> ir +; CHECK-NEXT: Successor(s): pred.load +; CHECK-EMPTY: +; CHECK-NEXT: pred.load: { +; CHECK-NEXT: pred.load.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%7> +; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.if: +; CHECK-NEXT: vp<%8> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<%8> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) +; CHECK-NEXT: Successor(s): pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%lv.b> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): then.0.0 +; CHECK-EMPTY: +; CHECK-NEXT: then.0.0: +; CHECK-NEXT: EMIT vp<%12> = not ir<%c.0> +; CHECK-NEXT: EMIT vp<%13> = select vp<%4> vp<%12> ir +; CHECK-NEXT: BLEND %p = ir<0>/vp<%13> vp<%11>/vp<%7> +; CHECK-NEXT: EMIT vp<%15> = or vp<%7> vp<%13> +; CHECK-NEXT: EMIT vp<%16> = select vp<%15> ir<%c.0> ir +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%16> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<0>, ir<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr ir<@c>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.c> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): then.1.2 +; CHECK-EMPTY: +; CHECK-NEXT: then.1.2: +; CHECK-NEXT: EMIT vp<%21> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%21> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): then.1.2 -; CHECK-EMPTY: -; CHECK-NEXT: then.1.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -601,66 +669,75 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'merge_3_replicate_region' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> -; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr ir<@c>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c> -; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.b> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.3 -; CHECK-EMPTY: -; CHECK-NEXT: loop.3: -; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]> ir<%c.0> ir -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr ir<@c>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1> -; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: vp<%4> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: EMIT vp<%5> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%5> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<%4> +; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> (S->V) +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<%4> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) +; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr ir<@c>, ir<0>, vp<%4> +; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c> +; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%13> = ir<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%14> = ir<%lv.b> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.3 +; CHECK-EMPTY: +; CHECK-NEXT: loop.3: +; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> +; CHECK-NEXT: WIDEN ir<%mul> = mul vp<%13>, vp<%14> +; CHECK-NEXT: EMIT vp<%17> = select vp<%5> ir<%c.0> ir +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%17> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr ir<@c>, ir<0>, vp<%4> +; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): then.0.4 +; CHECK-EMPTY: +; CHECK-NEXT: then.0.4: +; CHECK-NEXT: EMIT vp<%20> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%20> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): then.0.4 -; CHECK-EMPTY: -; CHECK-NEXT: then.0.4: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -700,42 +777,51 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-LABEL: LV: Checking a loop in 'update_2_uses_in_same_recipe_in_merged_block' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> -; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a> -; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%4> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> +; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a> +; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%10> = ir<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%div> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.2 +; CHECK-EMPTY: +; CHECK-NEXT: loop.2: +; CHECK-NEXT: EMIT vp<%12> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%12> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.2 -; CHECK-EMPTY: -; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -761,60 +847,69 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-LABEL: LV: Checking a loop in 'recipe_in_merge_candidate_used_by_first_order_recurrence' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: Successor(s): pred.load -; CHECK-EMPTY: -; CHECK-NEXT: pred.load: { -; CHECK-NEXT: pred.load.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.load.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = ir<%lv.a> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.0 -; CHECK-EMPTY: -; CHECK-NEXT: loop.0: -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for> vp<[[PRED]]> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]> -; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> -; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<%9> +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<1> +; CHECK-NEXT: EMIT vp<%6> = icmp ule ir<%iv> vp<%1> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<%5> +; CHECK-NEXT: Successor(s): pred.load +; CHECK-EMPTY: +; CHECK-NEXT: pred.load: { +; CHECK-NEXT: pred.load.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.if: +; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> (S->V) +; CHECK-NEXT: Successor(s): pred.load.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.load.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%lv.a> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.0 +; CHECK-EMPTY: +; CHECK-NEXT: loop.0: +; CHECK-NEXT: EMIT vp<%10> = first-order splice ir<%for> vp<%9> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<%10>, vp<%9> +; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%13> = ir<%div> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.2 +; CHECK-EMPTY: +; CHECK-NEXT: loop.2: +; CHECK-NEXT: EMIT vp<%14> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%14> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div> +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.2 -; CHECK-EMPTY: -; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -841,38 +936,47 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-LABEL: LV: Checking a loop in 'update_multiple_users' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK ir<%c> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src> -; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1> -; CHECK-NEXT: REPLICATE ir<%cmp> = icmp ir<%l1>, ir<0> -; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2> -; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%l1> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, ir<999> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK ir<%c> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src> +; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1> +; CHECK-NEXT: REPLICATE ir<%cmp> = icmp ir<%l1>, ir<0> +; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2> +; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%7> = ir<%l1> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.then.1 +; CHECK-EMPTY: +; CHECK-NEXT: loop.then.1: +; CHECK-NEXT: EMIT vp<%8> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%8> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.then.1 -; CHECK-EMPTY: -; CHECK-NEXT: loop.then.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -904,39 +1008,48 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-LABEL: LV: Checking a loop in 'sinking_requires_duplication' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%0> = load ir<%gep> -; CHECK-NEXT: WIDEN ir<%pred> = fcmp oeq ir<%0>, ir<0.000000e+00> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not ir<%pred> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%2> = SCALAR-STEPS vp<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<%2> +; CHECK-NEXT: WIDEN ir<%0> = load ir<%gep> +; CHECK-NEXT: WIDEN ir<%pred> = fcmp oeq ir<%0>, ir<0.000000e+00> +; CHECK-NEXT: EMIT vp<%6> = not ir<%pred> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%addr>, vp<%2> +; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): then.0 +; CHECK-EMPTY: +; CHECK-NEXT: then.0: +; CHECK-NEXT: EMIT vp<%9> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%9> vp<%0> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): then.0 -; CHECK-EMPTY: -; CHECK-NEXT: then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -971,47 +1084,51 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr noalias %dst) optsize { ; CHECK-LABEL: LV: Checking a loop in 'merge_with_dead_gep_between_regions' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count +; CHECK-NEXT: Live-in vp<%1> = backedge-taken count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> -; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr ir<%src>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%l>, ir<%gep.dst> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = ir<%l> +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%2> = 0, ir<0> { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<%3> = DERIVED-IV ir<%n> + vp<%2> * ir<-1> +; CHECK-NEXT: EMIT vp<%4> = WIDEN-CANONICAL-INDUCTION vp<%2> +; CHECK-NEXT: EMIT vp<%5> = icmp ule vp<%4> vp<%1> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%5> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: vp<%6> = SCALAR-STEPS vp<%3>, ir<-1> +; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr ir<%src>, vp<%6> +; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<%6> +; CHECK-NEXT: REPLICATE store ir<%l>, ir<%gep.dst> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%11> = ir<%l> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): loop.1 +; CHECK-EMPTY: +; CHECK-NEXT: loop.1: +; CHECK-NEXT: EMIT vp<%12> = VF * UF + vp<%2> +; CHECK-NEXT: EMIT branch-on-count vp<%12> vp<%0> ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): loop.1 +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -1036,44 +1153,48 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-LABEL: LV: Checking a loop in 'ptr_induction_remove_dead_recipe' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop +; CHECK-NEXT: Live-in vp<%0> = vector-trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1 -; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr ir<%ptr.iv>, ir<-1> -; CHECK-NEXT: WIDEN ir<%l> = load ir<%ptr.iv.next> -; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0> -; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1> -; CHECK-NEXT: Successor(s): pred.store -; CHECK-EMPTY: -; CHECK-NEXT: pred.store: { -; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<[[NEG]]> -; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%ptr.iv.next> = getelementptr ir<%ptr.iv>, ir<-1> -; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next> -; CHECK-NEXT: Successor(s): pred.store.continue -; CHECK-EMPTY: -; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector countable loop +; CHECK-EMPTY: +; CHECK-NEXT: vector countable loop vp<%1> = 0, inf { +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION +; CHECK-NEXT: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1 +; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr ir<%ptr.iv>, ir<-1> +; CHECK-NEXT: WIDEN ir<%l> = load ir<%ptr.iv.next> +; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0> +; CHECK-NEXT: EMIT vp<%6> = not ir<%c.1> +; CHECK-NEXT: Successor(s): pred.store +; CHECK-EMPTY: +; CHECK-NEXT: pred.store: { +; CHECK-NEXT: pred.store.entry: +; CHECK-NEXT: BRANCH-ON-MASK vp<%6> +; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.if: +; CHECK-NEXT: REPLICATE ir<%ptr.iv.next> = getelementptr ir<%ptr.iv>, ir<-1> +; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next> +; CHECK-NEXT: Successor(s): pred.store.continue +; CHECK-EMPTY: +; CHECK-NEXT: pred.store.continue: +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): if.then.0 +; CHECK-EMPTY: +; CHECK-NEXT: if.then.0: +; CHECK-NEXT: EMIT vp<%9> = VF * UF +(nuw) vp<%1> +; CHECK-NEXT: EMIT branch-on-count vp<%9> vp<%0> ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): if.then.0 +; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: -; CHECK-NEXT: if.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]> +; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; Index: llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp =================================================================== --- llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -105,32 +105,36 @@ node [shape=rect, fontname=Courier, fontsize=30] edge [fontname=Courier, fontsize=30] compound=true - N0 [label = - "vector.ph:\l" + - "Successor(s): for.body\l" - ] - N0 -> N1 [ label="" lhead=cluster_N2] - subgraph cluster_N2 { + subgraph cluster_N0 { fontname=Courier - label="\ for.body" + label="\ " N1 [label = - "vector.body:\l" + - " WIDEN-PHI ir\<%indvars.iv\> = phi ir\<0\>, ir\<%indvars.iv.next\>\l" + - " EMIT ir\<%arr.idx\> = getelementptr ir\<%A\> ir\<%indvars.iv\>\l" + - " EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" + - " EMIT ir\<%res\> = add ir\<%l1\> ir\<10\>\l" + - " EMIT store ir\<%res\> ir\<%arr.idx\>\l" + - " EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\> ir\<1\>\l" + - " EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\> ir\<%N\>\l" + - " EMIT branch-on-cond ir\<%exitcond\>\l" + + "vector.ph:\l" + + "Successor(s): for.body\l" + ] + N1 -> N2 [ label="" lhead=cluster_N3] + subgraph cluster_N3 { + fontname=Courier + label="\ for.body" + N2 [label = + "vector.body:\l" + + " WIDEN-PHI ir\<%indvars.iv\> = phi ir\<0\>, ir\<%indvars.iv.next\>\l" + + " EMIT ir\<%arr.idx\> = getelementptr ir\<%A\> ir\<%indvars.iv\>\l" + + " EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" + + " EMIT ir\<%res\> = add ir\<%l1\> ir\<10\>\l" + + " EMIT store ir\<%res\> ir\<%arr.idx\>\l" + + " EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\> ir\<1\>\l" + + " EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\> ir\<%N\>\l" + + " EMIT branch-on-cond ir\<%exitcond\>\l" + + "No successors\l" + ] + } + N2 -> N4 [ label="" ltail=cluster_N3] + N4 [label = + "for.end:\l" + "No successors\l" ] } - N1 -> N3 [ label="" ltail=cluster_N2] - N3 [label = - "for.end:\l" + - "No successors\l" - ] } )"; EXPECT_EQ(ExpectedStr, FullDump); Index: llvm/unittests/Transforms/Vectorize/VPlanTestBase.h =================================================================== --- llvm/unittests/Transforms/Vectorize/VPlanTestBase.h +++ llvm/unittests/Transforms/Vectorize/VPlanTestBase.h @@ -76,7 +76,7 @@ auto Plan = std::make_unique(); VPlanHCFGBuilder HCFGBuilder(LI->getLoopFor(LoopHeader), LI.get(), *Plan); - VPBasicBlock *EntryVPBB = HCFGBuilder.buildPlainCFG(); + VPBlockBase *EntryVPBB = HCFGBuilder.buildPlainCFG(); Plan->setEntry(EntryVPBB); return Plan; }