Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -332,10 +332,6 @@ cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference.")); -// FIXME: When loop hints are passed which allow reordering of FP operations, -// we still choose to use strict reductions with this flag. We should instead -// use the default behaviour of vectorizing with unordered reductions if -// reordering is allowed. cl::opt EnableStrictReductions( "enable-strict-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " @@ -558,6 +554,8 @@ /// Fix the non-induction PHIs in the OrigPHIsToFix vector. void fixNonInductionPHIs(VPTransformState &State); + bool useOrderedReductions(RecurrenceDescriptor &RdxDesc); + /// Create a broadcast instruction. This method generates a broadcast /// instruction (shuffle) for loop invariant values and for the induction /// value. If this is the induction variable then we extend it to N, N+1, ... @@ -1306,6 +1304,11 @@ /// outside. In loop reductions are collected into InLoopReductionChains. void collectInLoopReductions(); + bool useOrderedReductions(RecurrenceDescriptor &RdxDesc) { + return EnableStrictReductions && !Hints->allowReordering() && + RdxDesc.isOrdered(); + } + /// \returns The smallest bitwidth each instruction can be represented with. /// The vector equivalents of these instructions should be truncated to this /// type. @@ -4308,10 +4311,6 @@ LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock); } -static bool useOrderedReductions(RecurrenceDescriptor &RdxDesc) { - return EnableStrictReductions && RdxDesc.isOrdered(); -} - void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR, VPTransformState &State) { PHINode *OrigPhi = cast(PhiR->getUnderlyingValue()); @@ -4340,7 +4339,7 @@ BasicBlock *VectorLoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch(); bool IsOrdered = State.VF.isVector() && IsInLoopReductionPhi && - useOrderedReductions(RdxDesc); + Cost->useOrderedReductions(RdxDesc); for (unsigned Part = 0; Part < UF; ++Part) { if (IsOrdered && Part > 0) @@ -4646,6 +4645,10 @@ } } +bool InnerLoopVectorizer::useOrderedReductions(RecurrenceDescriptor &RdxDesc) { + return Cost->useOrderedReductions(RdxDesc); +} + void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Operands, unsigned UF, ElementCount VF, bool IsPtrLoopInvariant, @@ -4785,7 +4788,7 @@ bool IsOrdered = State.VF.isVector() && Cost->isInLoopReduction(cast(PN)) && - useOrderedReductions(*RdxDesc); + Cost->useOrderedReductions(*RdxDesc); for (unsigned Part = 0; Part < State.UF; ++Part) { // This is phase one of vectorizing PHIs. @@ -9478,7 +9481,7 @@ Value *PrevInChain = State.get(getChainOp(), 0); for (unsigned Part = 0; Part < State.UF; ++Part) { RecurKind Kind = RdxDesc->getRecurrenceKind(); - bool IsOrdered = useOrderedReductions(*RdxDesc); + bool IsOrdered = State.ILV->useOrderedReductions(*RdxDesc); Value *NewVecOp = State.get(getVecOp(), Part); if (VPValue *Cond = getCondOp()) { Value *NewCond = State.get(Cond, Part); Index: llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -1,6 +1,7 @@ ; RUN: opt < %s -loop-vectorize -scalable-vectorization=on -mtriple aarch64-unknown-linux-gnu -mattr=+sve -enable-strict-reductions=false -hints-allow-reordering=false -S 2>%t | FileCheck %s --check-prefix=CHECK-NOT-VECTORIZED ; RUN: opt < %s -loop-vectorize -scalable-vectorization=on -mtriple aarch64-unknown-linux-gnu -mattr=+sve -enable-strict-reductions=false -hints-allow-reordering=true -S 2>%t | FileCheck %s --check-prefix=CHECK-UNORDERED ; RUN: opt < %s -loop-vectorize -scalable-vectorization=on -mtriple aarch64-unknown-linux-gnu -mattr=+sve -enable-strict-reductions=true -hints-allow-reordering=false -S 2>%t | FileCheck %s --check-prefix=CHECK-ORDERED +; RUN: opt < %s -loop-vectorize -scalable-vectorization=on -mtriple aarch64-unknown-linux-gnu -mattr=+sve -enable-strict-reductions=true -hints-allow-reordering=true -S 2>%t | FileCheck %s --check-prefix=CHECK-UNORDERED define float @fadd_strict(float* noalias nocapture readonly %a, i64 %n) { ; CHECK-ORDERED-LABEL: @fadd_strict Index: llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll @@ -1,6 +1,7 @@ ; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -enable-strict-reductions=false -hints-allow-reordering=false -S 2>%t | FileCheck %s --check-prefix=CHECK-NOT-VECTORIZED ; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -enable-strict-reductions=false -hints-allow-reordering=true -S 2>%t | FileCheck %s --check-prefix=CHECK-UNORDERED ; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -enable-strict-reductions=true -hints-allow-reordering=false -S 2>%t | FileCheck %s --check-prefix=CHECK-ORDERED +; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -enable-strict-reductions=true -hints-allow-reordering=true -S 2>%t | FileCheck %s --check-prefix=CHECK-UNORDERED define float @fadd_strict(float* noalias nocapture readonly %a, i64 %n) { ; CHECK-ORDERED-LABEL: @fadd_strict @@ -551,10 +552,10 @@ ; return sum; ;} ; -; Note: These tests do not use metadata hints, and as such we should not expect the CHECK-UNORDERED case to vectorize, even -; with the -hints-allow-reordering flag set to true. ; Strict reduction could be performed in-loop, but ordered FP induction variables are not supported +; Note: This test does not use metadata hints, and as such we should not expect the CHECK-UNORDERED case to vectorize, even +; with the -hints-allow-reordering flag set to true. define float @induction_and_reduction(float* nocapture readonly %values, float %init, float* noalias nocapture %A, i64 %N) { ; CHECK-ORDERED-LABEL: @induction_and_reduction ; CHECK-ORDERED-NOT: vector.body @@ -612,7 +613,31 @@ ; CHECK-ORDERED: ret float %[[RES_PHI]] ; CHECK-UNORDERED-LABEL: @fast_induction_and_reduction -; CHECK-UNORDERED-NOT: vector.body +; CHECK-UNORDERED: vector.ph +; CHECK-UNORDERED: %[[INDUCTION:.*]] = fadd fast <4 x float> {{.*}}, +; CHECK-UNORDERED: vector.body +; CHECK-UNORDERED: %[[RDX_PHI1:.*]] = phi <4 x float> [ , %vector.ph ], [ %[[VEC_FADD1:.*]], %vector.body ] +; CHECK-UNORDERED: %[[RDX_PHI2:.*]] = phi <4 x float> [ , %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ] +; CHECK-UNORDERED: %[[IND_PHI:.*]] = phi <4 x float> [ %[[INDUCTION]], %vector.ph ], [ %[[VEC_IND_NEXT:.*]], %vector.body ] +; CHECK-UNORDERED: %[[STEP_ADD:.*]] = fadd fast <4 x float> %[[IND_PHI]], +; CHECK-UNORDERED: %[[LOAD1:.*]] = load <4 x float>, <4 x float>* +; CHECK-UNORDERED: %[[LOAD2:.*]] = load <4 x float>, <4 x float>* +; CHECK-UNORDERED: %[[VEC_FADD1]] = fadd <4 x float> %[[RDX_PHI1]], %[[LOAD1]] +; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <4 x float> %[[RDX_PHI2]], %[[LOAD2]] +; CHECK-UNORDERED: %[[VEC_IND_NEXT]] = fadd fast <4 x float> %[[STEP_ADD]], +; CHECK-UNORDERED: middle.block: +; CHECK-UNORDERED: %[[BIN_RDX:.*]] = fadd <4 x float> %[[VEC_FADD2]], %[[VEC_FADD1]] +; CHECK-UNORDERED: %[[VEC_RDX:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[BIN_RDX]]) +; CHECK-UNORDERED: for.body: +; CHECK-UNORDERED: %[[RDX_SUM_PHI:.*]] = phi float [ {{.*}}, %scalar.ph ], [ %[[FADD:.*]], %for.body ] +; CHECK-UNORDERED: %[[IND_SUM_PHI:.*]] = phi fast float [ {{.*}}, %scalar.ph ], [ %[[ADD_IND:.*]], %for.body ] +; CHECK-UNORDERED: store float %[[IND_SUM_PHI]], float* +; CHECK-UNORDERED: %[[ADD_IND]] = fadd fast float %[[IND_SUM_PHI]], 2.000000e+00 +; CHECK-UNORDERED: %[[LOAD3:.*]] = load float, float* +; CHECK-UNORDERED: %[[FADD]] = fadd float %[[RDX_SUM_PHI]], %[[LOAD3]] +; CHECK-UNORDERED: for.end +; CHECK-UNORDERED: %[[RES_PHI:.*]] = phi float [ %[[FADD]], %for.body ], [ %[[VEC_RDX]], %middle.block ] +; CHECK-UNORDERED: ret float %[[RES_PHI]] ; CHECK-NOT-VECTORIZED-LABEL: @fast_induction_and_reduction ; CHECK-NOT-VECTORIZED-NOT: vector.body @@ -632,13 +657,15 @@ %add3 = fadd float %sum.015, %0 %iv.next = add nuw nsw i64 %iv, 1 %exitcond.not = icmp eq i64 %iv.next, %N - br i1 %exitcond.not, label %for.end, label %for.body + br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !4 for.end: ret float %add3 } ; The FP induction is fast, but here we can't vectorize as only one of the reductions is an FAdd that can be performed in-loop +; Note: This test does not use metadata hints, and as such we should not expect the CHECK-UNORDERED case to vectorize, even +; with the -hints-allow-reordering flag set to true. define float @fast_induction_unordered_reduction(float* nocapture readonly %values, float %init, float* noalias nocapture %A, float* noalias nocapture %B, i64 %N) { ; CHECK-ORDERED-LABEL: @fast_induction_unordered_reduction @@ -674,14 +701,16 @@ ret float %add6 } -!0 = distinct !{!0, !4, !7, !9} -!1 = distinct !{!1, !4, !8, !9} -!2 = distinct !{!2, !5, !7, !9} -!3 = distinct !{!3, !6, !7, !9, !10} -!4 = !{!"llvm.loop.vectorize.width", i32 8} -!5 = !{!"llvm.loop.vectorize.width", i32 4} -!6 = !{!"llvm.loop.vectorize.width", i32 2} -!7 = !{!"llvm.loop.interleave.count", i32 1} -!8 = !{!"llvm.loop.interleave.count", i32 4} -!9 = !{!"llvm.loop.vectorize.enable", i1 true} -!10 = !{!"llvm.loop.vectorize.predicate.enable", i1 true} +!0 = distinct !{!0, !5, !8, !11} +!1 = distinct !{!1, !5, !10, !11} +!2 = distinct !{!2, !6, !8, !11} +!3 = distinct !{!3, !7, !8, !11, !12} +!4 = distinct !{!4, !6, !9, !11} +!5 = !{!"llvm.loop.vectorize.width", i32 8} +!6 = !{!"llvm.loop.vectorize.width", i32 4} +!7 = !{!"llvm.loop.vectorize.width", i32 2} +!8 = !{!"llvm.loop.interleave.count", i32 1} +!9 = !{!"llvm.loop.interleave.count", i32 2} +!10 = !{!"llvm.loop.interleave.count", i32 4} +!11 = !{!"llvm.loop.vectorize.enable", i1 true} +!12 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}