Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1736,7 +1736,8 @@ /// A set containing all BasicBlocks that are known to present after /// vectorization as a predicated block. - SmallPtrSet PredicatedBBsAfterVectorization; + DenseMap> + PredicatedBBsAfterVectorization; /// Records whether it is allowed to have the original scalar loop execute at /// least once. This may be needed as a fallback loop in case runtime @@ -6099,6 +6100,8 @@ // map will indicate that we've analyzed it already. ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF]; + PredicatedBBsAfterVectorization[VF].clear(); + // Find all the instructions that are scalar with predication in the loop and // determine if it would be better to not if-convert the blocks they are in. // If so, we also record the instructions to scalarize. @@ -6116,7 +6119,7 @@ computePredInstDiscount(&I, ScalarCosts, VF) >= 0) ScalarCostsVF.insert(ScalarCosts.begin(), ScalarCosts.end()); // Remember that BB will remain after vectorization. - PredicatedBBsAfterVectorization.insert(BB); + PredicatedBBsAfterVectorization[VF].insert(BB); } } } @@ -6976,8 +6979,8 @@ bool ScalarPredicatedBB = false; BranchInst *BI = cast(I); if (VF.isVector() && BI->isConditional() && - (PredicatedBBsAfterVectorization.count(BI->getSuccessor(0)) || - PredicatedBBsAfterVectorization.count(BI->getSuccessor(1)))) + (PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(0)) || + PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(1)))) ScalarPredicatedBB = true; if (ScalarPredicatedBB) { Index: llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll @@ -0,0 +1,32 @@ +; RUN: opt -S -loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -debug -disable-output <%s 2>%t +; RUN: cat %t | FileCheck %s + +target triple = "aarch64-unknown-linux-gnu" + +define i32 @uniform_load(i64 %n, i64 %boff, i64 %coff, ptr readonly %b, ptr readnone %c, ptr %d) #0 { +; CHECK: LAA: Found a loop in uniform_load: for.body +; CHECK-NOT: LV: Vector loop of width vscale x 2 costs: Invalid +; CHECK-NOT: LV: Vector loop of width vscale x 4 costs: Invalid +; CHECK: LV: Selecting VF: vscale x 4 +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ 1, %entry ], [ %indvars.iv.next, %for.body ] + %idxprom = add i64 %indvars.iv, %boff + %arrayidx = getelementptr inbounds float, ptr %b, i64 %idxprom + %load1 = load float, ptr %arrayidx, align 4 + %load2 = load float, ptr %d, align 4 + %add12 = fadd fast float %load1, %load2 + %idxprom2 = add i64 %indvars.iv, %coff + %arrayidx2 = getelementptr inbounds float, ptr %c, i64 %idxprom2 + store float %add12, ptr %arrayidx2, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret i32 0 +} + +attributes #0 = { vscale_range(1,16) "target-features"="+sve" }