diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1263,6 +1263,11 @@ /// are the selected vectorization factor and the cost of the selected VF. unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost); + /// \return true when the loop contains a predicated instruction that requires + /// scalarization. + bool loopHasScalarWithPredication(ElementCount VF, + bool StateFoldTailByMasking); + /// Memory access instruction may be vectorized in more than one way. /// Form of instruction after vectorization depends on cost. /// This function takes cost-based decisions for Load/Store instructions @@ -5597,6 +5602,17 @@ return ElementCount::getScalable(0); } + // Check the loop body. All instructions are in the loop body and not in the + // tail when the scalar epilogue is not allowed. + if (loopHasScalarWithPredication(MaxScalableVF, !isScalarEpilogueAllowed())) { + reportVectorizationInfo( + "Scalable vectorization not supported for predicated operations" + "operations found in this loop. Using fixed-width " + "vectorization instead.", + "ScalableVFUnfeasible", ORE, TheLoop); + return ElementCount::getScalable(0); + } + if (Legal->isSafeForAnyVectorWidth()) return MaxScalableVF; @@ -7749,6 +7765,18 @@ return false; } +bool LoopVectorizationCostModel::loopHasScalarWithPredication( + ElementCount VF, bool StateFoldTailByMasking) { + for (BasicBlock *BB : TheLoop->blocks()) { + // For each instruction in the old loop. + for (Instruction &I : BB->instructionsWithoutDebug()) { + if (!Legal->canVectorizeWithPredication(&I, StateFoldTailByMasking)) + return true; + } + } + return false; +} + void LoopVectorizationCostModel::collectValuesToIgnore() { // Ignore ephemeral values. CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore); diff --git a/llvm/test/Transforms/LoopVectorize/scalable-predicate-instruction.ll b/llvm/test/Transforms/LoopVectorize/scalable-predicate-instruction.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/scalable-predicate-instruction.ll @@ -0,0 +1,97 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -loop-vectorize -force-target-supports-scalable-vectors -S | FileCheck %s +; RUN: opt < %s -loop-vectorize -force-target-supports-scalable-vectors -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s --check-prefix=CHECK-PREFER-PREDICATE +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + +; The test loop_needs_predication_with_scalableVF corresponds +; to the following function +; for (long long i = 0; i < 1024; i++) { +; if (cond[i]) +; a[i] /= b[i]; +; } + +; Scalarizing the division cannot be done for scalable vectors at the moment +; when the loop needs predication +; Future implementation of llvm.vp could allow this to happen + +define void @loop_needs_predication_with_scalableVF(i32* %a, i32* %b, i32* %cond) #0 { +; CHECK-LABEL: @loop_needs_predication_with_scalableVF +; CHECK-NOT: sdiv +; CHECK-PREFER-PREDICATE-NOT: sdiv +; +entry: + br label %for.body + +for.cond.cleanup: ; preds = %for.inc, %entry + ret void + +for.body: ; preds = %entry, %for.inc + %i.09 = phi i64 [ %inc, %for.inc ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, i32* %cond, i64 %i.09 + %0 = load i32, i32* %arrayidx, align 4 + %tobool.not = icmp eq i32 %0, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: ; preds = %for.body + %arrayidx1 = getelementptr inbounds i32, i32* %b, i64 %i.09 + %1 = load i32, i32* %arrayidx1, align 4 + %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %i.09 + %2 = load i32, i32* %arrayidx2, align 4 + %div = sdiv i32 %2, %1 + store i32 %div, i32* %arrayidx2, align 4 + br label %for.inc + +for.inc: ; preds = %for.body, %if.then + %inc = add nuw nsw i64 %i.09, 1 + %exitcond.not = icmp eq i64 %inc, 1024 + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0 +} + + +; +; This test no_epillogue_allowed_with_scalableVF corresponds +; to the following function +; for (long long i = 0; i < 1024; i++) { +; a[i] /= b[i]; +; } + +; Scalarization not possible in the main loop when there is no predication and +; epilogue should not be able to allow scalarization +; otherwise it could be able to vectorize, but will not because +; "Max legal vector width too small, scalable vectorization unfeasible.." + +define void @no_epillogue_allowed_with_scalableVF(i32* %a, i32* %b) #0 { +; CHECK-LABEL: @no_epillogue_allowed_with_scalableVF +; CHECK-NOT: sdiv +; CHECK-PREFER-PREDICATE-NOT: sdiv + +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv + %0 = load i32, i32* %arrayidx, align 4 + %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %iv + %1 = load i32, i32* %arrayidx2, align 4 + %sdiv = sdiv i32 %1, %0 + %2 = add nuw nsw i64 %iv, 8 + %arrayidx5 = getelementptr inbounds i32, i32* %a, i64 %2 + store i32 %sdiv, i32* %arrayidx5, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, 1024 + br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !0 + +exit: + ret void + +} + +attributes #0 = {"target-cpu"="generic"} + + +!0 = distinct !{!0, !1, !2, !3, !4} +!1 = !{!"llvm.loop.vectorize.width", i32 4} +!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} +!3 = !{!"llvm.loop.interleave.count", i32 1} +!4 = !{!"llvm.loop.vectorize.enable", i1 true}