[LoopVectorize] Add a flag to prevent reordering of FP operations with hints

[kmclaughlin]

Summary

When loop hints are passed via metadata, the allowReordering() function
in LoopVectorizationLegality will allow the order of floating point
operations to be changed:

bool allowReordering() const {
  // When enabling loop hints are provided we allow the vectorizer to change
  // the order of operations that is given by the scalar loop. This is not
  // enabled by default because can be unsafe or inefficient.

This patch adds a flag to disable this behaviour for testing purposes,
particularly for testing vectorization with in-order reductions, which
requires further changes (D101836) to be enabled without the use of hints.

[sdesmalen]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

342

Can you add to the description that this has always been the default behaviour for Clang, so it is enabled by default to avoid causing unexpected regressions for code that uses these loop hints.

8008

This looks a bit odd. How about you make this an option to LoopVectorizationLegality, so that you can just call Hints.allowReordering() here without passing the option?

[kmclaughlin]

Abandoning as these changes have been combined with D101836

llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h

public:
   }
 
   bool isScalable() const { return Scalable.Value; }
 
   /// If hints are provided that force vectorization, use the AlwaysPrint
   /// pass name to force the frontend to print the diagnostic.
   const char *vectorizeAnalysisPassName() const;
 
-  bool allowReordering() const {
+  bool allowReordering(bool HintsAllowReordering) const {
     // When enabling loop hints are provided we allow the vectorizer to change
     // the order of operations that is given by the scalar loop. This is not
     // enabled by default because can be unsafe or inefficient. For example,
     // reordering floating-point operations will change the way round-off
     // error accumulates in the loop.
     ElementCount EC = getWidth();
-    return getForce() == LoopVectorizeHints::FK_Enabled ||
-           EC.getKnownMinValue() > 1;
+    return HintsAllowReordering &&
+           (getForce() == LoopVectorizeHints::FK_Enabled ||
+            EC.getKnownMinValue() > 1);
   }
 
   bool isPotentiallyUnsafe() const {
     // Avoid FP vectorization if the target is unsure about proper support.
     // This may be related to the SIMD unit in the target not handling
     // IEEE 754 FP ops properly, or bad single-to-double promotions.
     // Otherwise, a sequence of vectorized loops, even without reduction,
     // could lead to different end results on the destination vectors.
void addExactFPMathInst(Instruction *I) {
     if (I && !ExactFPMathInst)
       ExactFPMathInst = I;
   }
 
   void addRuntimePointerChecks(unsigned Num) { NumRuntimePointerChecks = Num; }
 
 
   Instruction *getExactFPInst() { return ExactFPMathInst; }
-  bool canVectorizeFPMath(const LoopVectorizeHints &Hints) const {
-    return !ExactFPMathInst || Hints.allowReordering();
+  bool canVectorizeFPMath(const LoopVectorizeHints &Hints,
+                          bool HintsAllowReordering) const {
+    return !ExactFPMathInst || Hints.allowReordering(HintsAllowReordering);
   }
 
   unsigned getNumRuntimePointerChecks() const {
     return NumRuntimePointerChecks;
   }
 
 private:
   unsigned NumRuntimePointerChecks = 0;

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
                            cl::desc("Prefer in-loop vector reductions, "
                                     "overriding the targets preference."));
 
 cl::opt<bool> EnableStrictReductions(
     "enable-strict-reductions", cl::init(false), cl::Hidden,
     cl::desc("Enable the vectorisation of loops with in-order (strict) "
              "FP reductions"));
 
+cl::opt<bool> HintsAllowReordering(
+    "hints-allow-reordering", cl::init(true), cl::Hidden,
+    cl::desc("Allow enabling loop hints to reorder FP operations "
+             "during vectorization."));
+
 static cl::opt<bool> PreferPredicatedReductionSelect(
     "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
     cl::desc(
         "Prefer predicating a reduction operation over an after loop select."));
 
 cl::opt<bool> EnableVPlanNativePath(
     "enable-vplan-native-path", cl::init(false), cl::Hidden,
     cl::desc("Enable VPlan-native vectorization path with "
LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
 
   // Check if it is profitable to vectorize with runtime checks.
   unsigned NumRuntimePointerChecks = Requirements.getNumRuntimePointerChecks();
   if (SelectedVF.Width.getKnownMinValue() > 1 && NumRuntimePointerChecks) {
     bool PragmaThresholdReached =
         NumRuntimePointerChecks > PragmaVectorizeMemoryCheckThreshold;
     bool ThresholdReached =
         NumRuntimePointerChecks > VectorizerParams::RuntimeMemoryCheckThreshold;
-    if ((ThresholdReached && !Hints.allowReordering()) ||
+    if ((ThresholdReached && !Hints.allowReordering(HintsAllowReordering)) ||
         PragmaThresholdReached) {
       ORE->emit([&]() {
         return OptimizationRemarkAnalysisAliasing(
                    DEBUG_TYPE, "CantReorderMemOps", OrigLoop->getStartLoc(),
                    OrigLoop->getHeader())
                << "loop not vectorized: cannot prove it is safe to reorder "
                   "memory operations";
       });
if (Hints.isPotentiallyUnsafe() &&
     reportVectorizationFailure(
         "Potentially unsafe FP op prevents vectorization",
         "loop not vectorized due to unsafe FP support.",
         "UnsafeFP", ORE, L);
     Hints.emitRemarkWithHints();
     return false;
   }
 
-  if (!Requirements.canVectorizeFPMath(Hints)) {
+  if (!Requirements.canVectorizeFPMath(Hints, HintsAllowReordering)) {
     ORE->emit([&]() {
       auto *ExactFPMathInst = Requirements.getExactFPInst();
       return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
                                                  ExactFPMathInst->getDebugLoc(),
                                                  ExactFPMathInst->getParent())
              << "loop not vectorized: cannot prove it is safe to reorder "
                 "floating-point operations";
     });

llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll

 ; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -enable-strict-reductions -S | FileCheck %s -check-prefix=CHECK
+; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -enable-strict-reductions -hints-allow-reordering=false -S | FileCheck %s -check-prefix=CHECK-NO-REORDER
 
 define float @fadd_strict(float* noalias nocapture readonly %a, i64 %n) {
 ; CHECK-LABEL: @fadd_strict
 ; CHECK: vector.body:
 ; CHECK: %[[VEC_PHI:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %vector.body ]
 ; CHECK: %[[LOAD:.*]] = load <vscale x 8 x float>, <vscale x 8 x float>*
 ; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.nxv8f32(float %[[VEC_PHI]], <vscale x 8 x float> %[[LOAD]])
 ; CHECK: for.end
 ; CHECK: %[[PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX]], %middle.block ]
 ; CHECK: ret float %[[PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_strict
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: %[[LOAD4:.*]] = load <vscale x 8 x float>, <vscale x 8 x float>*
 ; CHECK: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float %[[VEC_PHI1]], <vscale x 8 x float> %[[LOAD1]])
 ; CHECK: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float %[[RDX1]], <vscale x 8 x float> %[[LOAD2]])
 ; CHECK: %[[RDX3:.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float %[[RDX2]], <vscale x 8 x float> %[[LOAD3]])
 ; CHECK: %[[RDX4]] = call float @llvm.vector.reduce.fadd.nxv8f32(float %[[RDX3]], <vscale x 8 x float> %[[LOAD4]])
 ; CHECK: for.end
 ; CHECK: %[[PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX4]], %middle.block ]
 ; CHECK: ret float %[[PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_strict_unroll
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: %[[MGATHER1:.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0f32(<vscale x 4 x float*> %[[GEP1]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> undef, i1 true, i32 0), <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> undef)
 ; CHECK: %[[RDX1]] = call float @llvm.vector.reduce.fadd.nxv4f32(float %[[VEC_PHI1]], <vscale x 4 x float> %[[MGATHER1]])
 ; CHECK: %[[OR:.*]] = or <vscale x 4 x i64> %[[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK: %[[GEP2:.*]] = getelementptr inbounds float, float* %b, <vscale x 4 x i64> %[[OR]]
 ; CHECK: %[[MGATHER2:.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0f32(<vscale x 4 x float*> %[[GEP2]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> undef, i1 true, i32 0), <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> undef)
 ; CHECK: %[[RDX2]] = call float @llvm.vector.reduce.fadd.nxv4f32(float %[[VEC_PHI2]], <vscale x 4 x float> %[[MGATHER2]])
 ; CHECK: for.end
 ; CHECK ret void
+
+; CHECK-NO-REORDER-LABEL: @fadd_strict_interleave
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   %arrayidxa = getelementptr inbounds float, float* %a, i64 1
   %a1 = load float, float* %a, align 4
   %a2 = load float, float* %arrayidxa, align 4
   br label %for.body
 
 for.body:
   %add.phi1 = phi float [ %a2, %entry ], [ %add2, %for.body ]

 ; CHECK: %[[LOAD2:.*]] = load <vscale x 4 x float>, <vscale x 4 x float>*
 ; CHECK: %[[ADD:.*]] = fadd <vscale x 4 x float> %[[LOAD1]], %[[LOAD2]]
 ; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.nxv4f32(float %[[VEC_PHI1]], <vscale x 4 x float> %[[ADD]])
 ; CHECK: for.end.loopexit
 ; CHECK: %[[EXIT_PHI:.*]] = phi float [ {{.*}}, %for.body ], [ %[[RDX]], %middle.block ]
 ; CHECK: for.end
 ; CHECK: %[[PHI:.*]] = phi float [ 0.000000e+00, %entry ], [ %[[EXIT_PHI]], %for.end.loopexit ]
 ; CHECK: ret float %[[PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_invariant
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   %arrayidx = getelementptr inbounds float, float* %a, i64 1
   %0 = load float, float* %arrayidx, align 4
   %cmp1 = fcmp ogt float %0, 5.000000e-01
   br i1 %cmp1, label %for.body, label %for.end
 
 for.body:                                      ; preds = %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]

 ; CHECK: if.then
 ; CHECK: %[[LOAD2:.*]] = load float, float*
 ; CHECK: for.inc
 ; CHECK: %[[PHI:.*]] = phi float [ %[[LOAD2]], %if.then ], [ 3.000000e+00, %for.body ]
 ; CHECK: %[[FADD]] = fadd float %[[RES]], %[[PHI]]
 ; CHECK: for.end
 ; CHECK: %[[RDX_PHI:.*]] = phi float [ %[[FADD]], %for.inc ], [ %[[RDX]], %middle.block ]
 ; CHECK: ret float %[[RDX_PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_conditional
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:                                      ; preds = %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
   %res = phi float [ 1.000000e+00, %entry ], [ %fadd, %for.inc ]
   %arrayidx = getelementptr inbounds float, float* %b, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: %[[SUM:.*]] = phi float [ %bc.merge.rdx, %scalar.ph ], [ %[[FADD2:.*]], %for.body ]
 ; CHECK: %[[LOAD1:.*]] = load float, float*
 ; CHECK: %[[FADD1:.*]] = fadd float %[[SUM]], %[[LOAD1]]
 ; CHECK: %[[LOAD2:.*]] = load float, float*
 ; CHECK: %[[FADD2]] = fadd float %[[FADD1]], %[[LOAD2]]
 ; CHECK: for.end
 ; CHECK: %[[RET:.*]] = phi float [ %[[FADD2]], %for.body ], [ %[[RDX]], %middle.block ]
 ; CHECK: ret float %[[RET]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_multiple
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum = phi float [ -0.000000e+00, %entry ], [ %add3, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4

llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll

 ; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -enable-strict-reductions -S | FileCheck %s -check-prefix=CHECK
+; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -enable-strict-reductions -hints-allow-reordering=false -S | FileCheck %s -check-prefix=CHECK-NO-REORDER
 
 define float @fadd_strict(float* noalias nocapture readonly %a, i64 %n) {
 ; CHECK-LABEL: @fadd_strict
 ; CHECK: vector.body:
 ; CHECK: %[[VEC_PHI:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %vector.body ]
 ; CHECK: %[[LOAD:.*]] = load <8 x float>, <8 x float>*
 ; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI]], <8 x float> %[[LOAD]])
 ; CHECK: for.end
 ; CHECK: %[[PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX]], %middle.block ]
 ; CHECK: ret float %[[PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_strict
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: %[[LOAD4:.*]] = load <8 x float>, <8 x float>*
 ; CHECK: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI1]], <8 x float> %[[LOAD1]])
 ; CHECK: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX1]], <8 x float> %[[LOAD2]])
 ; CHECK: %[[RDX3:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX2]], <8 x float> %[[LOAD3]])
 ; CHECK: %[[RDX4]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX3]], <8 x float> %[[LOAD4]])
 ; CHECK: for.end
 ; CHECK: %[[PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX4]], %middle.block ]
 ; CHECK: ret float %[[PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_strict_unroll
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: %[[FADD]] =  fadd float %[[SUM_PHI]], %[[LOAD5]]
 ; CHECK: for.cond.cleanup
 ; CHECK: %[[FADD_LCSSA:.*]] = phi float [ %[[FADD]], %for.body ], [ %[[RDX4]], %middle.block ]
 ; CHECK: %[[FADD_42:.*]] = fadd float %[[FADD_LCSSA]], 4.200000e+01
 ; CHECK: store float %[[FADD_42]], float* %b
 ; CHECK: for.end
 ; CHECK: %[[SUM_LCSSA:.*]] = phi float [ %[[FADD_LCSSA]], %for.cond.cleanup ], [ 0.000000e+00, %entry ]
 ; CHECK: ret float %[[SUM_LCSSA]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_strict_unroll_last_val
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   %cmp = icmp sgt i64 %n, 0
   br i1 %cmp, label %for.body, label %for.end
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum = phi float [ 0.000000e+00, %entry ], [ %fadd, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv

 ; CHECK: %[[VEC_PHI2:.*]] = phi float [ %[[LOAD1]], %vector.ph ], [ %[[RDX1:.*]], %vector.body ]
 ; CHECK: %[[WIDE_LOAD:.*]] = load <8 x float>, <8 x float>*
 ; CHECK: %[[STRIDED1:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK: %[[STRIDED2:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK: %[[RDX1]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI2]], <4 x float> %[[STRIDED1]])
 ; CHECK: %[[RDX2]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI1]], <4 x float> %[[STRIDED2]])
 ; CHECK: for.end
 ; CHECK ret void
+
+; CHECK-NO-REORDER-LABEL: @fadd_strict_interleave
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   %arrayidxa = getelementptr inbounds float, float* %a, i64 1
   %a1 = load float, float* %a, align 4
   %a2 = load float, float* %arrayidxa, align 4
   br label %for.body
 
 for.body:
   %add.phi1 = phi float [ %a2, %entry ], [ %add2, %for.body ]

 ; CHECK: %[[LOAD2:.*]] = load <4 x float>, <4 x float>*
 ; CHECK: %[[ADD:.*]] = fadd <4 x float> %[[LOAD1]], %[[LOAD2]]
 ; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI1]], <4 x float> %[[ADD]])
 ; CHECK: for.end.loopexit
 ; CHECK: %[[EXIT_PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX]], %middle.block ]
 ; CHECK: for.end
 ; CHECK: %[[PHI:.*]] = phi float [ 0.000000e+00, %entry ], [ %[[EXIT_PHI]], %for.end.loopexit ]
 ; CHECK: ret float %[[PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_invariant
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   %arrayidx = getelementptr inbounds float, float* %a, i64 1
   %0 = load float, float* %arrayidx, align 4
   %cmp1 = fcmp ogt float %0, 5.000000e-01
   br i1 %cmp1, label %for.body, label %for.end
 
 for.body:                                      ; preds = %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]

 ; CHECK: %[[LOAD3:.*]] = load float, float*
 ; CHECK: br label %for.inc
 ; CHECK: for.inc
 ; CHECK: %[[PHI2:.*]] = phi float [ %[[LOAD3]], %if.then ], [ 3.000000e+00, %for.body ]
 ; CHECK: %[[FADD]] = fadd float %[[RES_PHI]], %[[PHI2]]
 ; CHECK: for.end
 ; CHECK: %[[RDX_PHI:.*]] = phi float [ %[[FADD]], %for.inc ], [ %[[RDX]], %middle.block ]
 ; CHECK: ret float %[[RDX_PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_conditional
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:                                      ; preds = %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
   %res = phi float [ 1.000000e+00, %entry ], [ %fadd, %for.inc ]
   %arrayidx = getelementptr inbounds float, float* %b, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: %[[RDX_PHI:.*]] =  phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %pred.load.continue2 ]
 ; CHECK: pred.load.continue2
 ; CHECK: %[[PHI:.*]] = phi <2 x float> [ %[[PHI0:.*]], %pred.load.continue ], [ %[[INS_ELT:.*]], %pred.load.if1 ]
 ; CHECK: %[[MASK:.*]] = select <2 x i1> %0, <2 x float> %[[PHI]], <2 x float> <float -0.000000e+00, float -0.000000e+00>
 ; CHECK: %[[RDX]] = call float @llvm.vector.reduce.fadd.v2f32(float %[[RDX_PHI]], <2 x float> %[[MASK]])
 ; CHECK: for.end:
 ; CHECK: %[[RES_PHI:.*]] = phi float [ %[[FADD:.*]], %for.body ], [ %[[RDX]], %middle.block ]
 ; CHECK: ret float %[[RES_PHI]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_predicated
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:                                           ; preds = %entry, %for.body
   %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
   %sum.02 = phi float [ %l7, %for.body ], [ 0.000000e+00, %entry ]
   %l2 = getelementptr inbounds float, float* %a, i64 %iv
   %l3 = load float, float* %l2, align 4

 ; CHECK: %[[SUM:.*]] = phi float [ %bc.merge.rdx, %scalar.ph ], [ %[[FADD2:.*]], %for.body ]
 ; CHECK: %[[LOAD1:.*]] = load float, float*
 ; CHECK: %[[FADD1:.*]] = fadd float %sum, %[[LOAD1]]
 ; CHECK: %[[LOAD2:.*]] = load float, float*
 ; CHECK: %[[FADD2]] = fadd float %[[FADD1]], %[[LOAD2]]
 ; CHECK: for.end
 ; CHECK: %[[RET:.*]] = phi float [ %[[FADD2]], %for.body ], [ %[[RDX]], %middle.block ]
 ; CHECK: ret float %[[RET]]
+
+; CHECK-NO-REORDER-LABEL: @fadd_multiple
+; CHECK-NO-REORDER-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum = phi float [ -0.000000e+00, %entry ], [ %add3, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4