diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3101,32 +3101,12 @@ BrInst->setDebugLoc(ScalarLatchTerm->getDebugLoc()); ReplaceInstWithInst(LoopMiddleBlock->getTerminator(), BrInst); - // We intentionally don't let SplitBlock to update LoopInfo since - // LoopVectorBody should belong to another loop than LoopVectorPreHeader. - // LoopVectorBody is explicitly added to the correct place few lines later. - BasicBlock *LoopVectorBody = - SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT, - nullptr, nullptr, Twine(Prefix) + "vector.body"); - // Update dominator for loop exit. if (!Cost->requiresScalarEpilogue(VF)) // If there is an epilogue which must run, there's no edge from the // middle block to exit blocks and thus no need to update the immediate // dominator of the exit blocks. DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock); - - // Create and register the new vector loop. - Loop *Lp = LI->AllocateLoop(); - Loop *ParentLoop = OrigLoop->getParentLoop(); - - // Insert the new loop into the loop nest and register the new basic blocks - // before calling any utilities such as SCEV that require valid LoopInfo. - if (ParentLoop) { - ParentLoop->addChildLoop(Lp); - } else { - LI->addTopLevelLoop(Lp); - } - Lp->addBasicBlockToLoop(LoopVectorBody, *LI); } void InnerLoopVectorizer::createInductionResumeValues( @@ -8645,8 +8625,6 @@ auto *CanonicalIVPHI = new VPCanonicalIVPHIRecipe(StartV, DL); VPRegionBlock *TopRegion = Plan.getVectorLoopRegion(); VPBasicBlock *Header = TopRegion->getEntryBasicBlock(); - if (IsVPlanNative) - Header = cast(Header->getSingleSuccessor()); Header->insert(CanonicalIVPHI, Header->begin()); auto *CanonicalIVIncrement = @@ -8733,7 +8711,9 @@ VPBasicBlock *LatchVPBB = new VPBasicBlock("vector.latch"); VPBlockUtils::insertBlockAfter(LatchVPBB, HeaderVPBB); auto *TopRegion = new VPRegionBlock(HeaderVPBB, LatchVPBB, "vector loop"); - auto Plan = std::make_unique(TopRegion); + VPBasicBlock *Preheader = new VPBasicBlock("vector.ph"); + VPBlockUtils::insertBlockAfter(TopRegion, Preheader); + auto Plan = std::make_unique(Preheader); Instruction *DLInst = getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()); @@ -9505,10 +9485,14 @@ } LastInduction->setName("vec.ind.next"); - VecInd->addIncoming(SteppedStart, State.CFG.VectorPreHeader); + BasicBlock *VectorPreHeader = + State.CFG.VPBB2IRBB[getParent() + ->getSingleHierarchicalPredecessor() + ->getExitBasicBlock()]; + VecInd->addIncoming(SteppedStart, VectorPreHeader); // Add induction update using an incorrect block temporarily. The phi node // will be fixed after VPlan execution. - VecInd->addIncoming(LastInduction, State.CFG.VectorPreHeader); + VecInd->addIncoming(LastInduction, VectorPreHeader); } void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { @@ -9563,7 +9547,12 @@ Type *ScStValueType = ScalarStartValue->getType(); PHINode *NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi", CanonicalIV); - NewPointerPhi->addIncoming(ScalarStartValue, State.CFG.VectorPreHeader); + + BasicBlock *VectorPreHeader = + State.CFG.VPBB2IRBB[getParent() + ->getSingleHierarchicalPredecessor() + ->getExitBasicBlock()]; + NewPointerPhi->addIncoming(ScalarStartValue, VectorPreHeader); // A pointer induction, performed by using a gep const DataLayout &DL = Phi->getModule()->getDataLayout(); @@ -9578,7 +9567,7 @@ IndDesc.getElementType(), NewPointerPhi, State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind", &*State.Builder.GetInsertPoint()); - NewPointerPhi->addIncoming(InductionGEP, State.CFG.VectorPreHeader); + NewPointerPhi->addIncoming(InductionGEP, VectorPreHeader); // Create UF many actual address geps that use the pointer // phi as base and a vectorized version of the step value diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2645,10 +2645,10 @@ /// Returns the VPRegionBlock of the vector loop. VPRegionBlock *getVectorLoopRegion() { - return cast(getEntry()); + return cast(getEntry()->getSingleSuccessor()); } const VPRegionBlock *getVectorLoopRegion() const { - return cast(getEntry()); + return cast(getEntry()->getSingleSuccessor()); } /// Returns the canonical induction recipe of the vector loop. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -307,6 +307,8 @@ // 1. Create an IR basic block, or reuse the last one if possible. // The last IR basic block is reused, as an optimization, in three cases: // A. the first VPBB reuses the loop header BB - when PrevVPBB is null; + // A2. the current VPBB is not the entry block of the vector loop region + // (i.e. not the header block of the vector loop). // B. when the current VPBB has a single (hierarchical) predecessor which // is PrevVPBB and the latter has a single (hierarchical) successor; and // C. when the current VPBB is an entry of a region replica - where PrevVPBB @@ -315,8 +317,10 @@ if (PrevVPBB && /* A */ !((SingleHPred = getSingleHierarchicalPredecessor()) && SingleHPred->getExitBasicBlock() == PrevVPBB && - PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */ - !(Replica && getPredecessors().empty())) { /* C */ + PrevVPBB->getSingleHierarchicalSuccessor() && + this != getPlan()->getVectorLoopRegion()->getEntryBasicBlock() /* A2 */ + ) && /* B */ + !(Replica && getPredecessors().empty())) { /* C */ NewBB = createEmptyBasicBlock(State->CFG); State->Builder.SetInsertPoint(NewBB); // Temporarily terminate with unreachable until CFG is rewired. @@ -451,16 +455,21 @@ ReversePostOrderTraversal RPOT(Entry); if (!isReplicator()) { + // Create and register the new vector loop. + State->CurrentVectorLoop = State->LI->AllocateLoop(); + Loop *ParentLoop = State->LI->getLoopFor(State->CFG.VectorPreHeader); + + // Insert the new loop into the loop nest and register the new basic blocks + // before calling any utilities such as SCEV that require valid LoopInfo. + if (ParentLoop) { + ParentLoop->addChildLoop(State->CurrentVectorLoop); + } else { + State->LI->addTopLevelLoop(State->CurrentVectorLoop); + } + // Visit the VPBlocks connected to "this", starting from it. for (VPBlockBase *Block : RPOT) { if (EnableVPlanNativePath) { - // The inner loop vectorization path does not represent loop preheader - // and exit blocks as part of the VPlan. In the VPlan-native path, skip - // vectorizing loop preheader block. In future, we may replace this - // check with the check for loop preheader. - if (Block->getNumPredecessors() == 0) - continue; - // Skip vectorizing loop exit block. In future, we may replace this // check with the check for loop exit. if (Block->getNumSuccessors() == 0) @@ -908,24 +917,20 @@ BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB; State->CFG.VectorPreHeader = VectorPreHeaderBB; - BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor(); - assert(VectorHeaderBB && "Loop preheader does not have a single successor."); - - Loop *L = State->LI->getLoopFor(VectorHeaderBB); - State->CurrentVectorLoop = L; // Remove the edge between Header and Latch to allow other connections. // Temporarily terminate with unreachable until CFG is rewired. // Note: this asserts the generated code's assumption that // getFirstInsertionPt() can be dereferenced into an Instruction. - VectorHeaderBB->getTerminator()->eraseFromParent(); - State->Builder.SetInsertPoint(VectorHeaderBB); + Instruction *OldTerm = VectorPreHeaderBB->getTerminator(); + State->Builder.SetInsertPoint(VectorPreHeaderBB); UnreachableInst *Terminator = State->Builder.CreateUnreachable(); - State->Builder.SetInsertPoint(Terminator); + Terminator->setDebugLoc(OldTerm->getDebugLoc()); + OldTerm->eraseFromParent(); // 1. Generate code in loop body. State->CFG.PrevVPBB = nullptr; - State->CFG.PrevBB = VectorHeaderBB; + State->CFG.PrevBB = VectorPreHeaderBB; for (VPBlockBase *Block : depth_first(Entry)) Block->execute(State); @@ -1004,10 +1009,13 @@ } } + BasicBlock *VectorHeaderBB = State->CFG.VPBB2IRBB[Header]; // We do not attempt to preserve DT for outer loop vectorization currently. - if (!EnableVPlanNativePath) + if (!EnableVPlanNativePath) { + State->DT->addNewBlock(VectorHeaderBB, VectorPreHeaderBB); updateDominatorTree(State->DT, VectorHeaderBB, VectorLatchBB, State->CFG.LastBB); + } } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -1429,7 +1437,10 @@ Value *Start = getStartValue()->getLiveInIRValue(); PHINode *EntryPart = PHINode::Create( Start->getType(), 2, "index", &*State.CFG.PrevBB->getFirstInsertionPt()); - EntryPart->addIncoming(Start, State.CFG.VectorPreHeader); + + VPBasicBlock *PreheaderVPBB = + getParent()->getSingleHierarchicalPredecessor()->getExitBasicBlock(); + EntryPart->addIncoming(Start, State.CFG.VPBB2IRBB[PreheaderVPBB]); EntryPart->setDebugLoc(DL); for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) State.set(this, EntryPart, Part); @@ -1514,10 +1525,13 @@ PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init"); } + VPBasicBlock *PreheaderVPBB = + getParent()->getSingleHierarchicalPredecessor()->getExitBasicBlock(); + BasicBlock *VectorPreHeader = State.CFG.VPBB2IRBB[PreheaderVPBB]; // Create a phi node for the new recurrence. PHINode *EntryPart = PHINode::Create( VecTy, 2, "vector.recur", &*State.CFG.PrevBB->getFirstInsertionPt()); - EntryPart->addIncoming(VectorInit, State.CFG.VectorPreHeader); + EntryPart->addIncoming(VectorInit, VectorPreHeader); State.set(this, EntryPart, 0); } @@ -1553,6 +1567,10 @@ State.set(this, EntryPart, Part); } + VPBasicBlock *PreheaderVPBB = + getParent()->getSingleHierarchicalPredecessor()->getExitBasicBlock(); + BasicBlock *VectorPreHeader = State.CFG.VPBB2IRBB[PreheaderVPBB]; + // Reductions do not have to start at zero. They can start with // any loop invariant values. VPValue *StartVPV = getStartValue(); @@ -1567,7 +1585,7 @@ Iden = StartV; } else { IRBuilderBase::InsertPointGuard IPBuilder(Builder); - Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator()); + Builder.SetInsertPoint(VectorPreHeader->getTerminator()); StartV = Iden = Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident"); } @@ -1578,7 +1596,7 @@ if (!ScalarPHI) { Iden = Builder.CreateVectorSplat(State.VF, Iden); IRBuilderBase::InsertPointGuard IPBuilder(Builder); - Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator()); + Builder.SetInsertPoint(VectorPreHeader->getTerminator()); Constant *Zero = Builder.getInt32(0); StartV = Builder.CreateInsertElement(Iden, StartV, Zero); } @@ -1589,7 +1607,7 @@ // Make sure to add the reduction start value only to the // first unroll part. Value *StartVal = (Part == 0) ? StartV : Iden; - cast(EntryPart)->addIncoming(StartVal, State.CFG.VectorPreHeader); + cast(EntryPart)->addIncoming(StartVal, VectorPreHeader); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h @@ -57,7 +57,7 @@ /// Build plain CFG for TheLoop. Return a new VPRegionBlock (TopRegion) /// enclosing the plain CFG. - VPRegionBlock *buildPlainCFG(); + VPBasicBlock *buildPlainCFG(); public: VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P) diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -60,7 +60,8 @@ SmallVector PhisToFix; // Utility functions. - void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB); + void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB, + BasicBlock *LatchBB); void fixPhiNodes(); VPBasicBlock *getOrCreateVPBB(BasicBlock *BB); #ifndef NDEBUG @@ -74,17 +75,22 @@ : TheLoop(Lp), LI(LI), Plan(P) {} // Build the plain CFG and return its Top Region. - VPRegionBlock *buildPlainCFG(); + VPBasicBlock *buildPlainCFG(); }; } // anonymous namespace // Set predecessors of \p VPBB in the same order as they are in \p BB. \p VPBB // must have no predecessors. -void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) { +void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB, + BasicBlock *HeaderBB) { SmallVector VPBBPreds; // Collect VPBB predecessors. - for (BasicBlock *Pred : predecessors(BB)) - VPBBPreds.push_back(getOrCreateVPBB(Pred)); + for (BasicBlock *Pred : predecessors(BB)) { + if (Pred == HeaderBB) + continue; + auto *PredVPBB = getOrCreateVPBB(Pred); + VPBBPreds.push_back(PredVPBB); + } VPBB->setPredecessors(VPBBPreds); } @@ -238,7 +244,7 @@ } // Main interface to build the plain CFG. -VPRegionBlock *PlainCFGBuilder::buildPlainCFG() { +VPBasicBlock *PlainCFGBuilder::buildPlainCFG() { // 1. Create the Top Region. It will be the parent of all VPBBs. TopRegion = new VPRegionBlock("TopRegion", false /*isReplicator*/); @@ -264,8 +270,6 @@ } // Create empty VPBB for Loop H so that we can link PH->H. VPBlockBase *HeaderVPBB = getOrCreateVPBB(TheLoop->getHeader()); - // Preheader's predecessors will be set during the loop RPO traversal below. - PreheaderVPBB->setOneSuccessor(HeaderVPBB); HeaderVPBB->setName("vector.body"); LoopBlocksRPO RPO(TheLoop); @@ -302,15 +306,23 @@ // representing the condition bit in VPlan (which may be in another VPBB). assert(IRDef2VPValue.count(BrCond) && "Missing condition bit in IRDef2VPValue!"); - VPValue *VPCondBit = IRDef2VPValue[BrCond]; - // Link successors using condition bit. - VPBB->setTwoSuccessors(SuccVPBB0, SuccVPBB1, VPCondBit); + if (TI->getSuccessor(0) == TheLoop->getHeader()) + VPBB->setOneSuccessor(SuccVPBB1); + else if (TI->getSuccessor(1) == TheLoop->getHeader()) + VPBB->setOneSuccessor(SuccVPBB0); + else { + VPValue *VPCondBit = IRDef2VPValue[BrCond]; + // Link successors using condition bit. + VPBB->setTwoSuccessors(SuccVPBB0, SuccVPBB1, VPCondBit); + } } else llvm_unreachable("Number of successors not supported."); - // Set VPBB predecessors in the same order as they are in the incoming BB. - setVPBBPredsFromBB(VPBB, BB); + if (BB != TheLoop->getHeader()) { + // Set VPBB predecessors in the same order as they are in the incoming BB. + setVPBBPredsFromBB(VPBB, BB, nullptr); + } } // 3. Process outermost loop exit. We created an empty VPBB for the loop @@ -322,7 +334,7 @@ createVPInstructionsForVPBB(LoopExitVPBB, LoopExitBB); // Loop exit was already set as successor of the loop exiting BB. // We only set its predecessor VPBB now. - setVPBBPredsFromBB(LoopExitVPBB, LoopExitBB); + setVPBBPredsFromBB(LoopExitVPBB, LoopExitBB, nullptr); // 4. The whole CFG has been built at this point so all the input Values must // have a VPlan couterpart. Fix VPlan phi nodes by adding their corresponding @@ -331,12 +343,16 @@ // 5. Final Top Region setup. Set outermost loop pre-header and single exit as // Top Region entry and exit. - TopRegion->setEntry(PreheaderVPBB); + TopRegion->setEntry(HeaderVPBB); TopRegion->setExit(LoopExitVPBB); - return TopRegion; + VPBlockUtils::insertBlockAfter(TopRegion, PreheaderVPBB); + TopRegion->setParent(nullptr); + PreheaderVPBB->setParent(nullptr); + + return PreheaderVPBB; } -VPRegionBlock *VPlanHCFGBuilder::buildPlainCFG() { +VPBasicBlock *VPlanHCFGBuilder::buildPlainCFG() { PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan); return PCFGBuilder.buildPlainCFG(); } @@ -344,10 +360,11 @@ // Public interface to build a H-CFG. void VPlanHCFGBuilder::buildHierarchicalCFG() { // Build Top Region enclosing the plain CFG and set it as VPlan entry. - VPRegionBlock *TopRegion = buildPlainCFG(); - Plan.setEntry(TopRegion); + VPBasicBlock *EntryVPBB = buildPlainCFG(); + Plan.setEntry(EntryVPBB); LLVM_DEBUG(Plan.setName("HCFGBuilder: Plain CFG\n"); dbgs() << Plan); + auto *TopRegion = cast(EntryVPBB->getSingleSuccessor()); Verifier.verifyHierarchicalCFG(TopRegion); // Compute plain CFG dom tree for VPLInfo. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -23,12 +23,12 @@ GetIntOrFpInductionDescriptor, SmallPtrSetImpl &DeadInstructions, ScalarEvolution &SE) { - auto *TopRegion = cast(Plan->getEntry()); + auto *TopRegion = cast(Plan->getVectorLoopRegion()); ReversePostOrderTraversal RPOT(TopRegion->getEntry()); for (VPBlockBase *Base : RPOT) { // Do not widen instructions in pre-header and exit blocks. - if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0) + if (Base->getNumSuccessors() == 0) continue; VPBasicBlock *VPBB = Base->getEntryBasicBlock(); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -10,6 +10,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll --- a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll @@ -19,6 +19,7 @@ ; CHECK-NOT: LV: Found uniform instruction: %tmp3 = getelementptr inbounds %data, %data* %d, i64 0, i32 0, i64 %i ; CHECK-NOT: LV: Found uniform instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] ; CHECK-NOT: LV: Found uniform instruction: %i.next = add nuw nsw i64 %i, 5 +; CHECK: define void @PR31671( ; CHECK: vector.ph: ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x float> poison, float %x, i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x float> [[BROADCAST_SPLATINSERT]], <16 x float> poison, <16 x i32> zeroinitializer diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll --- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll @@ -139,13 +139,13 @@ ; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[N]], -1 ; CHECK-NEXT: [[TMP19:%.*]] = zext i32 [[TMP18]] to i64 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH8:%.*]], label [[VECTOR_PH10:%.*]] -; CHECK: vector.ph10: +; CHECK: vector.ph9: ; CHECK-NEXT: [[N_RND_UP11:%.*]] = add nuw nsw i64 [[TMP19]], 4 ; CHECK-NEXT: [[N_VEC13:%.*]] = and i64 [[N_RND_UP11]], 8589934588 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT19:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT18]], <4 x i64> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY9:%.*]] -; CHECK: vector.body9: +; CHECK: vector.body19: ; CHECK-NEXT: [[INDEX38:%.*]] = phi i64 [ 0, [[VECTOR_PH10]] ], [ [[INDEX_NEXT37:%.*]], [[PRED_STORE_CONTINUE36:%.*]] ] ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX38]] ; CHECK-NEXT: [[BROADCAST_SPLATINSERT27:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX38]], i64 0 diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -13,6 +13,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -98,6 +101,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -168,6 +174,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -238,6 +247,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -333,6 +345,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -41,6 +41,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -7,7 +7,17 @@ ; Verify that -vplan-print-in-dot-format option works. define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable { -; CHECK: subgraph cluster_N0 { +; CHECK: digraph VPlan { +; CHECK-NEXT: graph [labelloc=t, fontsize=30; label="Vectorization Plan\nInitial VPlan for VF=\{4\},UF\>=1"] +; CHECK-NEXT: node [shape=rect, fontname=Courier, fontsize=30] +; CHECK-NEXT: edge [fontname=Courier, fontsize=30] +; CHECK-NEXT: compound=true +; CHECK-NEXT: N0 [label = +; CHECK-NEXT: "vector.ph:\l" + +; CHECK-NEXT: "Successor(s): vector loop\l" +; CHECK-NEXT: ] +; CHECK-NEXT: N0 -> N1 [ label="" lhead=cluster_N2] +; CHECK-NEXT: subgraph cluster_N2 { ; CHECK-NEXT: fontname=Courier ; CHECK-NEXT: label="\ vector loop" ; CHECK-NEXT: N1 [label = diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -11,6 +11,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -51,6 +54,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -96,6 +102,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -133,6 +142,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -207,6 +219,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -267,6 +282,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -307,6 +325,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -10,6 +10,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -15,6 +15,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -82,6 +85,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -162,6 +168,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -244,6 +253,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -335,6 +347,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -437,6 +452,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -554,6 +572,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -665,6 +686,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -781,6 +805,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -849,6 +876,9 @@ ; CHECK-EMPTY: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -932,6 +962,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION @@ -1003,6 +1036,9 @@ ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h --- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h +++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h @@ -76,8 +76,8 @@ auto Plan = std::make_unique(); VPlanHCFGBuilder HCFGBuilder(LI->getLoopFor(LoopHeader), LI.get(), *Plan); - VPRegionBlock *TopRegion = HCFGBuilder.buildPlainCFG(); - Plan->setEntry(TopRegion); + VPBasicBlock *EntryVPBB = HCFGBuilder.buildPlainCFG(); + Plan->setEntry(EntryVPBB); return Plan; } };