diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -289,6 +289,8 @@ getDecisionAndClampRange(const std::function &Predicate, VFRange &Range); + VPlanPtr clone(VPlan &OriginalPlan); + protected: /// Collect the instructions from the original loop that would be trivially /// dead in the vectorized loop if generated. diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7776,7 +7776,9 @@ CM.selectUserVectorizationFactor(VF); CM.collectInLoopReductions(); auto InitialPlan = buildVPlan({VF, VF}); - buildVPlansWithVPRecipes(VF, VF, *InitialPlan); + VPlanPredicator Pred(*InitialPlan); + Pred.predicate(); + buildVPlansWithVPRecipes(UserVF, UserVF, *InitialPlan); LLVM_DEBUG(printPlans(dbgs())); return {{VF, 0}}; } @@ -7798,6 +7800,8 @@ CM.collectInLoopReductions(); auto InitialPlan = buildVPlan({ElementCount::getFixed(1), MaxVF}); + VPlanPredicator Pred(*InitialPlan); + Pred.predicate(); buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxVF, *InitialPlan); LLVM_DEBUG(printPlans(dbgs())); if (MaxVF.isScalar()) @@ -8651,9 +8655,9 @@ return new VPWidenRecipe(*I, make_range(Operands.begin(), Operands.end())); } -VPBasicBlock *VPRecipeBuilder::handleReplication( - Instruction *I, VFRange &Range, VPBasicBlock *VPBB, - VPlanPtr &Plan) { +std::pair +VPRecipeBuilder::handleReplication(Instruction *I, VFRange &Range, + VPBasicBlock *VPBB, VPlanPtr &Plan) { bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange( [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); }, Range); @@ -8662,10 +8666,12 @@ [&](ElementCount VF) { return CM.isScalarWithPredication(I, VF); }, Range); - auto *Recipe = new VPReplicateRecipe(I, Plan->mapToVPValues(I->operands()), - IsUniform, IsPredicated); + VPRecipeBase *Recipe = new VPReplicateRecipe( + I, Plan->mapToVPValues(I->operands()), IsUniform, IsPredicated); setRecipe(I, Recipe); - Plan->addVPValue(I, Recipe); + Plan->getVPValue(I)->replaceAllUsesWith(Recipe->getVPValue(0)); + Plan->removeVPValueFor(I); + Plan->addVPValue(I, Recipe->getVPValue(0)); // Find if I uses a predicated instruction. If so, it will use its scalar // value. Avoid hoisting the insert-element which packs the scalar value into @@ -8684,23 +8690,29 @@ // Finalize the recipe for Instr, first if it is not predicated. if (!IsPredicated) { LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n"); - VPBB->appendRecipe(Recipe); - return VPBB; + return {VPBB, Recipe}; } LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n"); - assert(VPBB->getSuccessors().empty() && - "VPBB has successors when handling predicated replication."); - // Record predicated instructions for above packing optimizations. - VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan); + SmallVector Succs(VPBB->getSuccessors().begin(), + VPBB->getSuccessors().end()); + for (auto *Succ : VPBB->getSuccessors()) + VPBlockUtils::disconnectBlocks(VPBB, Succ); + VPBlockBase *Region; + std::tie(Region, Recipe) = createReplicateRegion(I, Recipe, Plan); VPBlockUtils::insertBlockAfter(Region, VPBB); auto *RegSucc = new VPBasicBlock(); VPBlockUtils::insertBlockAfter(RegSucc, Region); - return RegSucc; + + for (auto *Succ : Succs) + VPBlockUtils::connectBlocks(RegSucc, Succ); + + return {RegSucc, Recipe}; } -VPRegionBlock *VPRecipeBuilder::createReplicateRegion(Instruction *Instr, - VPRecipeBase *PredRecipe, - VPlanPtr &Plan) { +std::pair +VPRecipeBuilder::createReplicateRegion(Instruction *Instr, + VPRecipeBase *PredRecipe, + VPlanPtr &Plan) { // Instructions marked for predication are replicated and placed under an // if-then construct to prevent side-effects. @@ -8728,7 +8740,7 @@ VPBlockUtils::insertTwoBlocksAfter(Pred, Exit, BlockInMask, Entry); VPBlockUtils::connectBlocks(Pred, Exit); - return Region; + return {Region, PHIRecipe ? PHIRecipe : PredRecipe}; } VPRecipeOrVPValueTy @@ -8783,6 +8795,78 @@ return toVPRecipeResult(tryToWiden(Instr, Operands)); } +VPlanPtr LoopVectorizationPlanner::clone(VPlan &OriginalPlan) { + VPBlockBase *EntryBlock = OriginalPlan.getEntry()->getEntryBasicBlock(); + + auto Plan = std::make_unique(); + DenseMap Old2NewBlocks; + auto *TopRegion = new VPRegionBlock("TopRegion"); + Old2NewBlocks[OriginalPlan.getEntry()] = TopRegion; + + // Clone blocks of the original plan, add them to cloned plan. + for (VPBlockBase *Base : depth_first(EntryBlock)) { + assert(isa(Base) && + "VPlans without nested regions can be cloned at the moment"); + Old2NewBlocks[Base] = new VPBasicBlock(Base->getName()); + Old2NewBlocks[Base]->setParent( + cast(Old2NewBlocks[Base->getParent()])); + } + + Plan->setEntry(TopRegion); + TopRegion->setEntry( + Old2NewBlocks[OriginalPlan.getEntry()->getEntryBasicBlock()]); + + // Connect blocks in cloned plan. + for (VPBlockBase *Base : depth_first(EntryBlock)) { + for (auto *Succ : Base->getSuccessors()) { + VPBlockUtils::connectBlocks(Old2NewBlocks[Base], Old2NewBlocks[Succ]); + } + } + + DenseMap Old2New; + // Update operands of U to use the version in the cloned plan. + auto RemapOperands = [&Old2New, &Plan](VPRecipeBase *U) { + for (unsigned I = 0, E = U->getNumOperands(); I != E; ++I) { + VPValue *OldOp = U->getOperand(I); + VPValue *NewOp = nullptr; + if (OldOp->getDef()) { + NewOp = Old2New[OldOp]; + assert(NewOp && "trying to map operand that was not defined"); + } else { + NewOp = Plan->getOrAddVPValue(OldOp->getLiveInIRValue()); + } + U->setOperand(I, NewOp); + } + }; + + // Clone each recipe in the original plan and add it to the cloned plan. + SmallVector PhisToFix; + for (VPBlockBase *Base : depth_first(EntryBlock)) { + VPBasicBlock *OrigVPBB = cast(Base); + VPBasicBlock *VPBB = cast(Old2NewBlocks[OrigVPBB]); + for (VPRecipeBase &OrigR : *OrigVPBB) { + VPRecipeBase *NewI = OrigR.clone(); + if (isa(NewI)) + PhisToFix.push_back(NewI); + else + RemapOperands(NewI); + + for (unsigned I = 0, E = NewI->getNumDefinedValues(); I != E; ++I) { + VPValue *NewDef = NewI->getVPValue(I); + Old2New[OrigR.getVPValue(I)] = NewDef; + Plan->addVPValue(NewDef->getUnderlyingValue(), NewDef); + } + + VPBB->appendRecipe(NewI); + } + } + + for (VPRecipeBase *R : PhisToFix) + RemapOperands(R); + + return Plan; +} + void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF, VPlan &OriginalPlan) { @@ -8868,10 +8952,14 @@ for (VPBlockBase *Base : depth_first(TopRegion->getEntryBasicBlock())) { VPBasicBlock *OriginalVPBB = Base->getEntryBasicBlock(); + + // Condition bits set by predicator is not used for now. Drop it. + OriginalVPBB->setCondBit(nullptr); + for (VPRecipeBase &Ingredient : make_early_inc_range(reverse(*OriginalVPBB))) { VPInstruction *VPInst = dyn_cast(&Ingredient); - if (!VPInst) + if (!VPInst || !VPInst->getUnderlyingValue()) continue; Instruction *Instr = VPInst->getUnderlyingInstr(); @@ -8886,6 +8974,26 @@ } } + SmallVector ToRemove; + // Currently the update instruction of induction PHIs is part of the + // instructions to remove. So the use in the VPWidenPHIRecipe will be replaced + // by DummyVal, but the recipe won't be removed. Replace it with a new phi + // node that only has the start value as operand. + for (VPUser *U : DummyVal->users()) { + VPRecipeBase *R = cast(U); + auto *VPhi = cast(R); + auto *NewP = + new VPWidenPHIRecipe(cast(VPhi->getUnderlyingValue())); + // Not all PHIs set the start value so far. + if (VPhi->getStartValue()) + NewP->addOperand(VPhi->getStartValue()); + NewP->insertBefore(VPhi); + VPhi->getVPValue()->replaceAllUsesWith(NewP); + ToRemove.push_back(R); + } + for (VPRecipeBase *R : ToRemove) + R->eraseFromParent(); + for (auto &Entry : SinkAfter) { VPRecipeBase *Sink = Inst2VPInst.find(Entry.first)->second; VPRecipeBase *Target = Inst2VPInst.find(Entry.second)->second; @@ -8950,93 +9058,102 @@ // visit each basic block after having visited its predecessor basic blocks. // --------------------------------------------------------------------------- - // Create a dummy pre-entry VPBasicBlock to start building the VPlan. - auto Plan = std::make_unique(); - - VPBasicBlock *Header = OriginalPlan.getEntry()->getEntryBasicBlock(); + auto Plan = clone(OriginalPlan); + VPBasicBlock *Header = Plan->getEntry()->getEntryBasicBlock(); ReversePostOrderTraversal RPOT(Header); VPBasicBlock *VPBB = nullptr; for (VPBlockBase *Base : RPOT) { - VPBasicBlock *OriginalVPBB = Base->getEntryBasicBlock(); // Relevant instructions from basic block BB will be grouped into VPRecipe // ingredients and fill a new VPBasicBlock. unsigned VPBBsForBB = 0; - auto *FirstVPBBForBB = new VPBasicBlock(OriginalVPBB->getName()); - if (VPBB) - VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB); - else - Plan->setEntry(FirstVPBBForBB); - VPBB = FirstVPBBForBB; + VPBB = cast(Base); Builder.setInsertPoint(VPBB); - // Introduce each ingredient into VPlan. - for (auto I = OriginalVPBB->begin(), E = OriginalVPBB->end(); I != E;) { - VPRecipeBase *Ingredient = &*I++; - // First, clone & add the recipes introduced earlier. - if (isa(Ingredient)) { - auto *C = Ingredient->clone(); - for (unsigned I = 0, E = C->getNumDefinedValues(); I != E; ++I) { - auto *NewDef = C->getVPValue(I); - auto *UV = NewDef->getUnderlyingValue(); - Plan->addVPValue(UV, NewDef); - Ingredient->getVPValue(I)->replaceAllUsesWith(NewDef); + auto ReplaceAllUsesWith = [&Plan](VPDef *Old, VPDef *New) { + for (auto &NewV : enumerate(New->definedValues())) { + VPValue *OldV = Old->getVPValue(NewV.index()); + if (auto *UV = OldV->getUnderlyingValue()) { + Plan->removeVPValueFor(UV); + Plan->addVPValue(UV, NewV.value()); } - for (unsigned I = 0, E = Ingredient->getNumOperands(); I != E; ++I) - C->setOperand(I, Plan->getOrAddVPValue( - Ingredient->getOperand(I)->getLiveInIRValue())); - VPBB->appendRecipe(C); + OldV->replaceAllUsesWith(NewV.value()); + } + }; + auto FirstNonPhi = VPBB->getFirstNonPhi(); + auto I = VPBB->begin(), E = VPBB->end(); + while (I != E && (isa(*I) || + isa(*I))) { + VPRecipeBase *Ingredient = &*I++; + if (isa(Ingredient)) + continue; + PHINode *Phi = cast( + cast(Ingredient)->getUnderlyingInstr()); + Builder.setInsertPoint(Ingredient->getParent(), FirstNonPhi); + auto RecipeOrValue = RecipeBuilder.tryToCreateWidenRecipe( + Phi, {Ingredient->op_begin(), Ingredient->getNumOperands()}, Range, + Plan); + // If Instr can be simplified to an existing VPValue, use it. + if (RecipeOrValue.is()) { + Ingredient->getVPValue()->replaceAllUsesWith( + RecipeOrValue.get()); + Plan->removeVPValueFor(Phi); + Plan->addVPValue(Phi, RecipeOrValue.get()); + Ingredient->eraseFromParent(); continue; } + // Otherwise, add the new recipe. + VPRecipeBase *Recipe = RecipeOrValue.get(); + RecipeBuilder.setRecipe(Phi, Recipe); + if (isa(Recipe)) + VPBB->insert(Recipe, FirstNonPhi); + else + Recipe->insertBefore(Ingredient); + ReplaceAllUsesWith(Ingredient, Recipe); + Ingredient->eraseFromParent(); + } + + // Introduce each ingredient into VPlan. + while (I != E) { + VPRecipeBase *Ingredient = &*I++; VPInstruction *VPInst = dyn_cast(Ingredient); - Instruction *Instr = - VPInst - ? VPInst->getUnderlyingInstr() - : cast( - cast(Ingredient)->getUnderlyingInstr()); - - // First filter out irrelevant instructions, to ensure no recipes are - // built for them. - if (isa(Instr)) + if (!VPInst || !VPInst->getUnderlyingValue()) continue; - - SmallVector Operands; - auto *Phi = dyn_cast(Instr); - if (Phi && Phi->getParent() == OrigLoop->getHeader()) { - Operands.push_back(Plan->getOrAddVPValue( - Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()))); - } else { - auto OpRange = Plan->mapToVPValues(Instr->operands()); - Operands = {OpRange.begin(), OpRange.end()}; - } + Instruction *Instr = nullptr; + Instr = VPInst->getUnderlyingInstr(); + + assert(!isa(Instr) && !isa(Instr) && + "Branch and PHI instruction must be removed earlier"); + auto InsertPt = Ingredient->getIterator(); + if (Base != VPBB) + InsertPt = VPBB->end(); + Builder.setInsertPoint(VPBB, InsertPt); + VPRecipeBase *Recipe = nullptr; if (auto RecipeOrValue = RecipeBuilder.tryToCreateWidenRecipe( - Instr, Operands, Range, Plan)) { - // If Instr can be simplified to an existing VPValue, use it. - if (RecipeOrValue.is()) { - Plan->addVPValue(Instr, RecipeOrValue.get()); - continue; - } - // Otherwise, add the new recipe. - VPRecipeBase *Recipe = RecipeOrValue.get(); - for (auto *Def : Recipe->definedValues()) { - auto *UV = Def->getUnderlyingValue(); - Plan->addVPValue(UV, Def); - } - + Instr, {Ingredient->op_begin(), Ingredient->getNumOperands()}, + Range, Plan)) { + // Add the new recipe. + Recipe = RecipeOrValue.get(); RecipeBuilder.setRecipe(Instr, Recipe); - VPBB->appendRecipe(Recipe); - continue; - } + VPBB->insert(Recipe, InsertPt); + } else { - // Otherwise, if all widening options failed, Instruction is to be - // replicated. This may create a successor for VPBB. - VPBasicBlock *NextVPBB = - RecipeBuilder.handleReplication(Instr, Range, VPBB, Plan); - if (NextVPBB != VPBB) { - VPBB = NextVPBB; - VPBB->setName(OriginalVPBB->getName() + "." + Twine(VPBBsForBB++)); + // Otherwise, if all widening options failed, Instruction is to be + // replicated. This may create a successor for VPBB. + auto Res = RecipeBuilder.handleReplication(Instr, Range, VPBB, Plan); + VPBasicBlock *NextVPBB = Res.first; + Recipe = Res.second; + if (NextVPBB != VPBB) { + VPBB = NextVPBB; + VPBB->setName(Base->getName() + "." + Twine(VPBBsForBB++)); + } else { + VPBB->insert(Recipe, InsertPt); + } } + + ReplaceAllUsesWith(Ingredient, Recipe); + Ingredient->eraseFromParent(); } } diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -153,8 +153,9 @@ /// Create a replicating region for instruction \p I that requires /// predication. \p PredRecipe is a VPReplicateRecipe holding \p I. - VPRegionBlock *createReplicateRegion(Instruction *I, VPRecipeBase *PredRecipe, - VPlanPtr &Plan); + std::pair + createReplicateRegion(Instruction *I, VPRecipeBase *PredRecipe, + VPlanPtr &Plan); /// Build a VPReplicationRecipe for \p I and enclose it within a Region if it /// is predicated. \return \p VPBB augmented with this new recipe if \p I is @@ -162,9 +163,9 @@ /// Region. Update the packing decision of predicated instructions if they /// feed \p I. Range.End may be decreased to ensure same recipe behavior from /// \p Range.Start to \p Range.End. - VPBasicBlock *handleReplication( - Instruction *I, VFRange &Range, VPBasicBlock *VPBB, - VPlanPtr &Plan); + std::pair + handleReplication(Instruction *I, VFRange &Range, VPBasicBlock *VPBB, + VPlanPtr &Plan); }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -682,6 +682,12 @@ return true; } + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPUser *D) { + // All VPDefs are also VPRecipeBases. + return true; + } + /// Clone this recipe. virtual VPRecipeBase *clone() = 0; }; @@ -727,7 +733,6 @@ protected: void setUnderlyingInstr(Instruction *I) { setUnderlyingValue(I); } - public: VPInstruction(unsigned Opcode, ArrayRef Operands) : VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands), @@ -1963,6 +1968,8 @@ return getVPValue(V); } + bool hasVPValue(Value *V) { return Value2VPValue.count(V); } + void removeVPValueFor(Value *V) { Value2VPValue.erase(V); } /// Return the VPLoopInfo analysis for this VPlan. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1013,7 +1013,11 @@ void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << Indent << "WIDEN-PHI " << VPlanIngredient(getUnderlyingValue()); + O << Indent << "WIDEN-PHI "; + printAsOperand(O, SlotTracker); + O << " = phi "; + printOperands(O, SlotTracker); + O << " (original " << VPlanIngredient(getUnderlyingValue()) << ")"; } void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp @@ -135,8 +135,8 @@ // Blocks that dominate region exit inherit the predicate from the region. // Return after setting the predicate. if (VPDomTree.dominates(CurrBlock, Region->getExit())) { - VPValue *RegionBP = Region->getPredicate(); - CurrBlock->setPredicate(RegionBP); + // VPValue *RegionBP = Region->getPredicate(); + // CurrBlock->setPredicate(RegionBP); return; } @@ -155,6 +155,7 @@ // Skip back-edges if (VPBlockUtils::isBackEdge(PredBlock, CurrBlock, VPLI)) continue; + continue; VPValue *IncomingPredicate = nullptr; unsigned NumPredSuccsNoBE = @@ -177,10 +178,10 @@ } // Logically OR all incoming predicates by building the Predicate Tree. - VPValue *Predicate = genPredicateTree(IncomingPredicates); + // VPValue *Predicate = genPredicateTree(IncomingPredicates); - // Now update the block's predicate with the new one. - CurrBlock->setPredicate(Predicate); + //// Now update the block's predicate with the new one. + // CurrBlock->setPredicate(Predicate); } // Generate all predicates needed for Region. @@ -208,6 +209,7 @@ ReversePostOrderTraversal RPOT(Region->getEntry()); VPBlockBase *PrevBlock = nullptr; + auto *Exit = Region->getExit(); for (VPBlockBase *CurrBlock : RPOT) { // TODO: Handle nested regions once we start generating the same. assert(!isa(CurrBlock) && "Nested region not expected"); @@ -216,11 +218,17 @@ // and CurrBlock skipping loop headers and latches to keep intact loop // header predecessors and loop latch successors. if (PrevBlock && !VPLI->isLoopHeader(CurrBlock) && - !VPBlockUtils::blockIsLoopLatch(PrevBlock, VPLI)) { + !VPBlockUtils::blockIsLoopLatch(PrevBlock, VPLI) && CurrBlock != Exit && + PrevBlock != Exit) { LLVM_DEBUG(dbgs() << "Linearizing: " << PrevBlock->getName() << "->" << CurrBlock->getName() << "\n"); + SmallVector Succs(PrevBlock->getSuccessors().begin(), + PrevBlock->getSuccessors().end()); + for (auto *Succ : Succs) { + VPBlockUtils::disconnectBlocks(PrevBlock, Succ); + } PrevBlock->clearSuccessors(); CurrBlock->clearPredecessors(); VPBlockUtils::connectBlocks(PrevBlock, CurrBlock); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -73,13 +73,13 @@ // for multiple underlying IRs (Polly?) by providing a new VPlan front-end, // back-end and analysis information for the new IR. +public: // Set \p Val as the underlying Value of this VPValue. void setUnderlyingValue(Value *Val) { assert(!UnderlyingVal && "Underlying Value is already set."); UnderlyingVal = Val; } -public: /// Return the underlying Value attached to this VPValue. Value *getUnderlyingValue() { return UnderlyingVal; } const Value *getUnderlyingValue() const { return UnderlyingVal; } diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -37,11 +37,13 @@ ; Check for crash exposed by D76992. ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: loop: -; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next -; CHECK-NEXT: WIDEN ir<%cond0> = icmp ir<%iv>, ir<13> -; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20> -; CHECK-NEXT: No successor +; CHECK-NEXT: TopRegion: { +; CHECK-NEXT: loop: +; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next +; CHECK-NEXT: WIDEN ir<%cond0> = icmp ir<%iv>, ir<13> +; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20> +; CHECK-NEXT: No successor +; CHECK-NEXT: } ; CHECK-NEXT: } define void @test() { entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -7,7 +7,7 @@ ; Verify that -vplan-print-in-dot-format option works. define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable { -; CHECK: N0 [label = +; CHECK: N1 [label = ; CHECK-NEXT: "for.body:\l" + ; CHECK-NEXT: " WIDEN-INDUCTION %iv = phi %iv.next, 0\l" + ; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, ir\<%iv\>\l" + diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -9,6 +9,7 @@ define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable { ; CHECK-LABEL: Checking a loop in "print_call_and_memory" ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: TopRegion: { ; CHECK-NEXT: for.body: ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0 ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv> @@ -41,6 +42,7 @@ define void @print_widen_gep_and_select(i64 %n, float* noalias %y, float* noalias %x, float* %z) nounwind uwtable { ; CHECK-LABEL: Checking a loop in "print_widen_gep_and_select" ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: TopRegion: { ; CHECK-NEXT: for.body: ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0 ; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv> @@ -77,6 +79,7 @@ define float @print_reduction(i64 %n, float* noalias %y) { ; CHECK-LABEL: Checking a loop in "print_reduction" ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: TopRegion: { ; CHECK-NEXT: for.body: ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0 ; CHECK-NEXT: WIDEN-PHI %red = phi %red.next, 0.000000e+00 @@ -106,6 +109,7 @@ define void @print_replicate_predicated_phi(i64 %n, i64* %x) { ; CHECK-LABEL: Checking a loop in "print_replicate_predicated_phi" ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: TopRegion: { ; CHECK-NEXT: for.body: ; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next ; CHECK-NEXT: WIDEN ir<%cmp> = icmp ir<%i>, ir<5>