diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1263,6 +1263,11 @@ /// are the selected vectorization factor and the cost of the selected VF. unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost); + /// \return true when the loop contains a predicated instruction that requires + /// scalarization. + bool loopHasScalarWithPredication(ElementCount VF, + bool StateFoldTailByMasking); + /// Memory access instruction may be vectorized in more than one way. /// Form of instruction after vectorization depends on cost. /// This function takes cost-based decisions for Load/Store instructions @@ -5597,6 +5602,17 @@ return ElementCount::getScalable(0); } + // Check the loop body. All instructions are in the loop body and not in the + // tail when the scalar epilogue is not allowed. + if (loopHasScalarWithPredication(MaxScalableVF, !isScalarEpilogueAllowed())) { + reportVectorizationInfo( + "Scalable vectorization not supported for predicated operations" + "operations found in this loop. Using fixed-width " + "vectorization instead.", + "ScalableVFUnfeasible", ORE, TheLoop); + return ElementCount::getScalable(0); + } + if (Legal->isSafeForAnyVectorWidth()) return MaxScalableVF; @@ -7749,6 +7765,18 @@ return false; } +bool LoopVectorizationCostModel::loopHasScalarWithPredication( + ElementCount VF, bool StateFoldTailByMasking) { + for (BasicBlock *BB : TheLoop->blocks()) { + // For each instruction in the old loop. + for (Instruction &I : BB->instructionsWithoutDebug()) { + if (!Legal->canVectorizeWithPredication(&I, StateFoldTailByMasking)) + return true; + } + } + return false; +} + void LoopVectorizationCostModel::collectValuesToIgnore() { // Ignore ephemeral values. CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore); diff --git a/llvm/test/Transforms/LoopVectorize/scalable-predicate-instruction.ll b/llvm/test/Transforms/LoopVectorize/scalable-predicate-instruction.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/scalable-predicate-instruction.ll @@ -0,0 +1,104 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -loop-vectorize -force-target-supports-scalable-vectors -S | FileCheck %s +; RUN: opt < %s -loop-vectorize -force-target-supports-scalable-vectors -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s --check-prefix=CHECK-PREFER-PREDICATE +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + +; This test corresponds to the following function +; for (long long i = 0; i < n; i++) { +; if (cond[i]) +; a[i] /= b[i]; +; } + +; Scalarizing the division cannot be done for scalable vectors at the moment +; when the loop needs predication +; Future implementation of llvm.vp could allow this to happen + +define void @loop_needs_predication_with_scalableVF(i32* %a, i32* %b, i32* %cond, i64 %n) #0 { +; CHECK-LABEL: @loop_needs_predication_with_scalableVF +; CHECK-NOT: sdiv +; CHECK-PREFER-PREDICATE-NOT: sdiv +; +entry: + %cmp8 = icmp sgt i64 %n, 0 + br i1 %cmp8, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.inc, %entry + ret void + +for.body: ; preds = %entry, %for.inc + %i.09 = phi i64 [ %inc, %for.inc ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, i32* %cond, i64 %i.09 + %0 = load i32, i32* %arrayidx, align 4 + %tobool.not = icmp eq i32 %0, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: ; preds = %for.body + %arrayidx1 = getelementptr inbounds i32, i32* %b, i64 %i.09 + %1 = load i32, i32* %arrayidx1, align 4 + %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %i.09 + %2 = load i32, i32* %arrayidx2, align 4 + %div = sdiv i32 %2, %1 + store i32 %div, i32* %arrayidx2, align 4 + br label %for.inc + +for.inc: ; preds = %for.body, %if.then + %inc = add nuw nsw i64 %i.09, 1 + %exitcond.not = icmp eq i64 %inc, %n + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0 +} + + +; +; This test corresponds to the following function +; for (long long i = 0; i < n; i++) { +; a[i] /= b[i]; +; } + +; Scalarization not possible in the main loop when there is no predication and +; epilogue should not be able to allow scalarization +; otherwise it should be able to vectorize + +define void @no_epillogue_allowed_with_scalableVF(i32* %a, i32* %b, i32* %cond, i64 %n) #0 { +; CHECK-LABEL: @no_epillogue_allowed_with_scalableVF +; CHECK: sdiv +; CHECK-PREFER-PREDICATE-NOT: sdiv + +entry: + %cmp6 = icmp sgt i64 %n, 0 + br i1 %cmp6, label %entry.for.body_crit_edge, label %for.cond.cleanup, !llvm.loop !0 + +entry.for.body_crit_edge: ; preds = %entry + %inc.1 = add nuw nsw i64 0, 1 + %arrayidx1.1 = getelementptr inbounds i32, i32* %a, i64 0 + %arrayidx.1 = getelementptr inbounds i32, i32* %b, i64 0 + br label %for.body, !llvm.loop !0 + +for.cond.cleanup: ; preds = %for.body, %entry + ret void + +for.body: ; preds = %entry.for.body_crit_edge, %tail + %arrayidx.phi = phi i32* [ %arrayidx.0, %tail ], [ %arrayidx.1, %entry.for.body_crit_edge ] + %arrayidx1.phi = phi i32* [ %arrayidx1.0, %tail ], [ %arrayidx1.1, %entry.for.body_crit_edge ] + %inc.phi = phi i64 [ %inc.0, %tail ], [ %inc.1, %entry.for.body_crit_edge ] + %0 = load i32, i32* %arrayidx.phi, align 4 + %1 = load i32, i32* %arrayidx1.phi, align 4 + %div = sdiv i32 %1, %0 + store i32 %div, i32* %arrayidx1.phi, align 4 + br label %tail, !llvm.loop !0 + +tail: ; preds = %for.body + %inc.0 = add nuw nsw i64 %inc.phi, 1 + %arrayidx1.0 = getelementptr inbounds i32, i32* %a, i64 %inc.phi + %arrayidx.0 = getelementptr inbounds i32, i32* %b, i64 %inc.phi + %exitcond.not = icmp eq i64 %inc.0, %n + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0 +} + +attributes #0 = {"target-cpu"="generic"} + + +!0 = distinct !{!0, !1, !2, !3, !4} +!1 = !{!"llvm.loop.vectorize.width", i32 4} +!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} +!3 = !{!"llvm.loop.interleave.count", i32 1} +!4 = !{!"llvm.loop.vectorize.enable", i1 true}