Index: include/llvm/Transforms/Utils/LoopUtils.h =================================================================== --- include/llvm/Transforms/Utils/LoopUtils.h +++ include/llvm/Transforms/Utils/LoopUtils.h @@ -74,7 +74,8 @@ RK_IntegerMinMax, ///< Min/max implemented in terms of select(cmp()). RK_FloatAdd, ///< Sum of floats. RK_FloatMult, ///< Product of floats. - RK_FloatMinMax ///< Min/max implemented in terms of select(cmp()). + RK_FloatMinMax, ///< Min/max implemented in terms of select(cmp()). + RK_SelectFAdd ///< Sum of selected floats. }; // This enum represents the kind of minmax recurrence. @@ -151,6 +152,10 @@ /// or max(X, Y). static InstDesc isMinMaxSelectCmpPattern(Instruction *I, InstDesc &Prev); + /// Returns a struct describing if the instruction is a + /// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern. + static InstDesc isSelectPattern(Instruction *I, RecurrenceKind K); + /// Returns identity corresponding to the RecurrenceKind. static Constant *getRecurrenceIdentity(RecurrenceKind K, Type *Tp); Index: lib/Transforms/Utils/LoopUtils.cpp =================================================================== --- lib/Transforms/Utils/LoopUtils.cpp +++ lib/Transforms/Utils/LoopUtils.cpp @@ -301,7 +301,7 @@ // A reduction operation must only have one use of the reduction value. if (!IsAPhi && Kind != RK_IntegerMinMax && Kind != RK_FloatMinMax && - hasMultipleUsesOf(Cur, VisitedInsts)) + Kind != RK_SelectFAdd && hasMultipleUsesOf(Cur, VisitedInsts)) return false; // All inputs to a PHI node must be a reduction value. @@ -362,7 +362,8 @@ } else if (!isa(UI) && ((!isa(UI) && !isa(UI) && !isa(UI)) || - !isMinMaxSelectCmpPattern(UI, IgnoredVal).isRecurrence())) + (!isSelectPattern(UI, Kind).isRecurrence() && + !isMinMaxSelectCmpPattern(UI, IgnoredVal).isRecurrence()))) return false; // Remember that we completed the cycle. @@ -491,6 +492,43 @@ return InstDesc(false, I); } +/// Returns true if the instruction has a following chain of instructions. +/// %sum.1 = phi [...], [ %sum.2, ...] +/// ... +/// %cmp = fcmp pred %0, %1 +/// %add = fadd %0, %sum.1 +/// %sum.2 = select %cmp, %add, %sum.1 +RecurrenceDescriptor::InstDesc +RecurrenceDescriptor::isSelectPattern(Instruction *I, RecurrenceKind Kind) { + SelectInst *SI = dyn_cast(I); + // Handle only select instruction. + if (!SI) + return InstDesc(false, I); + + CmpInst *CI = dyn_cast(SI->getCondition()); + // Handle only single use cases for now. + if (!CI || !CI->hasOneUse()) + return InstDesc(false, I); + + Value *TrueVal = SI->getTrueValue(); + Value *FalseVal = SI->getFalseValue(); + // Handle only when either of SelectInst operands is a PHI node for now. + if ((isa(*TrueVal) && isa(*FalseVal)) || + (!isa(*TrueVal) && !isa(*FalseVal))) + return InstDesc(false, I); + + Instruction *Inst = isa(*TrueVal) + ? dyn_cast(FalseVal) : dyn_cast(TrueVal); + if (!Inst) + return InstDesc(false, I); + + Value *Op1, *Op2; + // Handle only the case of fadd for now. + if (m_FAdd(m_Value(Op1), m_Value(Op2)).match(Inst)) + return InstDesc(Kind == RK_SelectFAdd, SI); + return InstDesc(false, I); +} + RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, InstDesc &Prev, bool HasFunNoNaNAttr) { @@ -518,11 +556,15 @@ case Instruction::FMul: return InstDesc(Kind == RK_FloatMult, I, UAI); case Instruction::FSub: - case Instruction::FAdd: return InstDesc(Kind == RK_FloatAdd, I, UAI); + case Instruction::FAdd: + return InstDesc(Kind == RK_FloatAdd || Kind == RK_SelectFAdd, I, UAI); + case Instruction::Select: + if (Kind == RK_SelectFAdd) + return isSelectPattern(I, Kind); + LLVM_FALLTHROUGH; case Instruction::FCmp: case Instruction::ICmp: - case Instruction::Select: if (Kind != RK_IntegerMinMax && (!HasFunNoNaNAttr || Kind != RK_FloatMinMax)) return InstDesc(false, I); @@ -599,6 +641,12 @@ << "\n"); return true; } + if (AddReductionVar(Phi, RK_SelectFAdd, TheLoop, HasFunNoNaNAttr, RedDes, DB, + AC, DT)) { + LLVM_DEBUG(dbgs() << "Found an float SelectFAdd reduction PHI." << *Phi + << "\n"); + return true; + } // Not a reduction of known type. return false; } @@ -675,6 +723,7 @@ // Multiplying a number by 1 does not change it. return ConstantFP::get(Tp, 1.0L); case RK_FloatAdd: + case RK_SelectFAdd: // Adding zero to a number does not change it. return ConstantFP::get(Tp, 0.0L); default: @@ -698,6 +747,7 @@ case RK_FloatMult: return Instruction::FMul; case RK_FloatAdd: + case RK_SelectFAdd: return Instruction::FAdd; case RK_IntegerMinMax: return Instruction::ICmp; @@ -1693,6 +1743,7 @@ Flags.NoNaN = NoNaN; switch (RecKind) { case RD::RK_FloatAdd: + case RD::RK_SelectFAdd: return createSimpleTargetReduction(B, TTI, Instruction::FAdd, Src, Flags); case RD::RK_FloatMult: return createSimpleTargetReduction(B, TTI, Instruction::FMul, Src, Flags); Index: test/Transforms/LoopVectorize/if-reduction.ll =================================================================== --- test/Transforms/LoopVectorize/if-reduction.ll +++ test/Transforms/LoopVectorize/if-reduction.ll @@ -0,0 +1,429 @@ +; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + +; Float pattern: +; Check vectorization of reduction code which has an fadd instruction after +; an fcmp instruction which compares an array element and 0. +; +; float fcmp_0_fadd_select1(float * restrict x, const int N) { +; float sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > (float)0.) +; sum += x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_0_fadd_select1( +; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x float> %[[V0:.*]], zeroinitializer +; CHECK-NEXT: %[[V3:.*]] = fadd fast <4 x float> %[[V0]], %[[V2:.*]] +; CHECK-NEXT: select <4 x i1> %[[V1]], <4 x float> %[[V3]], <4 x float> %[[V2]] +define float @fcmp_0_fadd_select1(float* noalias %x, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %header, %for.body + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv + %0 = load float, float* %arrayidx, align 4 + %cmp.2 = fcmp fast ogt float %0, 0.000000e+00 + %add = fadd fast float %0, %sum.1 + %sum.2 = select i1 %cmp.2, float %add, float %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret float %1 +} + +; Double pattern: +; Check vectorization of reduction code which has an fadd instruction after +; an fcmp instruction which compares an array element and 0. +; +; double fcmp_0_fadd_select2(double * restrict x, const int N) { +; double sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > 0.) +; sum += x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_0_fadd_select2( +; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x double> %[[V0:.*]], zeroinitializer +; CHECK-NEXT: %[[V3:.*]] = fadd fast <4 x double> %[[V0]], %[[V2:.*]] +; CHECK-NEXT: select <4 x i1> %[[V1]], <4 x double> %[[V3]], <4 x double> %[[V2]] +define double @fcmp_0_fadd_select2(double* noalias %x, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %header, %for.body + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv + %0 = load double, double* %arrayidx, align 4 + %cmp.2 = fcmp fast ogt double %0, 0.000000e+00 + %add = fadd fast double %0, %sum.1 + %sum.2 = select i1 %cmp.2, double %add, double %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret double %1 +} + +; Float pattern: +; Check vectorization of reduction code which has an fadd instruction after +; an fcmp instruction which compares an array element and a floating-point +; value. +; +; float fcmp_val_fadd_select1(float * restrict x, float y, const int N) { +; float sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > y) +; sum += x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_val_fadd_select1( +; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x float> %[[V0:.*]], %broadcast.splat2 +; CHECK-NEXT: %[[V3:.*]] = fadd fast <4 x float> %[[V0]], %[[V2:.*]] +; CHECK-NEXT: select <4 x i1> %[[V1]], <4 x float> %[[V3]], <4 x float> %[[V2]] +define float @fcmp_val_fadd_select1(float* noalias %x, float %y, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %header, %for.body + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv + %0 = load float, float* %arrayidx, align 4 + %cmp.2 = fcmp fast ogt float %0, %y + %add = fadd fast float %0, %sum.1 + %sum.2 = select i1 %cmp.2, float %add, float %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret float %1 +} + +; Double pattern: +; Check vectorization of reduction code which has an fadd instruction after +; an fcmp instruction which compares an array element and a floating-point +; value. +; +; double fcmp_val_fadd_select2(double * restrict x, double y, const int N) { +; double sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > y) +; sum += x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_val_fadd_select2( +; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x double> %[[V0:.*]], %broadcast.splat2 +; CHECK-NEXT: %[[V3:.*]] = fadd fast <4 x double> %[[V0]], %[[V2:.*]] +; CHECK-NEXT: select <4 x i1> %[[V1]], <4 x double> %[[V3]], <4 x double> %[[V2]] +define double @fcmp_val_fadd_select2(double* noalias %x, double %y, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %header, %for.body + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv + %0 = load double, double* %arrayidx, align 4 + %cmp.2 = fcmp fast ogt double %0, %y + %add = fadd fast double %0, %sum.1 + %sum.2 = select i1 %cmp.2, double %add, double %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret double %1 +} + +; Float pattern: +; Check vectorization of reduction code which has an fadd instruction after +; an fcmp instruction which compares an array element and another array +; element. +; +; float fcmp_array_elm_fadd_select1(float * restrict x, float * restrict y, +; const int N) { +; float sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > y[i]) +; sum += x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_array_elm_fadd_select1( +; CHECK: %[[V2:.*]] = fcmp fast ogt <4 x float> %[[V0:.*]], %[[V1:.*]] +; CHECK-NEXT: %[[V4:.*]] = fadd fast <4 x float> %[[V0]], %[[V3:.*]] +; CHECK-NEXT: select <4 x i1> %[[V2]], <4 x float> %[[V4]], <4 x float> %[[V3]] +define float @fcmp_array_elm_fadd_select1(float* noalias %x, float* noalias %y, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %for.body, %for.header + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx.1 = getelementptr inbounds float, float* %x, i64 %indvars.iv + %0 = load float, float* %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds float, float* %y, i64 %indvars.iv + %1 = load float, float* %arrayidx.2, align 4 + %cmp.2 = fcmp fast ogt float %0, %1 + %add = fadd fast float %0, %sum.1 + %sum.2 = select i1 %cmp.2, float %add, float %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %2 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret float %2 +} + +; Double pattern: +; Check vectorization of reduction code which has an fadd instruction after +; an fcmp instruction which compares an array element and another array +; element. +; +; double fcmp_array_elm_fadd_select2(double * restrict x, double * restrict y, +; const int N) { +; double sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > y[i]) +; sum += x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_array_elm_fadd_select2( +; CHECK: %[[V2:.*]] = fcmp fast ogt <4 x double> %[[V0:.*]], %[[V1:.*]] +; CHECK-NEXT: %[[V4:.*]] = fadd fast <4 x double> %[[V0]], %[[V3:.*]] +; CHECK-NEXT: select <4 x i1> %[[V2]], <4 x double> %[[V4]], <4 x double> %[[V3]] +define double @fcmp_array_elm_fadd_select2(double* noalias %x, double* noalias %y, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %for.body, %for.header + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx.1 = getelementptr inbounds double, double* %x, i64 %indvars.iv + %0 = load double, double* %arrayidx.1, align 4 + %arrayidx.2 = getelementptr inbounds double, double* %y, i64 %indvars.iv + %1 = load double, double* %arrayidx.2, align 4 + %cmp.2 = fcmp fast ogt double %0, %1 + %add = fadd fast double %0, %sum.1 + %sum.2 = select i1 %cmp.2, double %add, double %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %2 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret double %2 +} + +; Float pattern: +; Check fsub not vectorized. +; +; float fcmp_0_fsub_select1(float * restrict x, const int N) { +; float sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > (float)0.) +; sum -= x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_0_fsub_select1( +; CHECK: %[[V1:.*]] = fcmp ogt float %[[V0:.*]], 0.000000e+00 +; CHECK-NEXT: %[[V3:.*]] = fsub float %[[V2:.*]], %[[V0]] +; CHECK-NEXT: select i1 %[[V1]], float %[[V3]], float %[[V2:.*]] +define float @fcmp_0_fsub_select1(float* noalias %x, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %for.body, %for.header + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv + %0 = load float, float* %arrayidx, align 4 + %cmp.2 = fcmp ogt float %0, 0.000000e+00 + %sub = fsub float %sum.1, %0 + %sum.2 = select i1 %cmp.2, float %sub, float %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret float %1 +} + +; Double pattern: +; Check fsub not vectorized. +; +; double fcmp_0_fsub_select2(double * restrict x, const int N) { +; double sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > 0.) +; sum -= x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_0_fsub_select2( +; CHECK: %[[V1:.*]] = fcmp ogt double %[[V0:.*]], 0.000000e+00 +; CHECK-NEXT: %[[V3:.*]] = fsub double %[[V2:.*]], %[[V0]] +; CHECK-NEXT: select i1 %[[V1]], double %[[V3]], double %[[V2:.*]] +define double @fcmp_0_fsub_select2(double* noalias %x, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %for.body, %for.header + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv + %0 = load double, double* %arrayidx, align 4 + %cmp.2 = fcmp ogt double %0, 0.000000e+00 + %sub = fsub double %sum.1, %0 + %sum.2 = select i1 %cmp.2, double %sub, double %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret double %1 +} + +; Float pattern: +; Check fmul not vectorized. +; +; float fcmp_0_fmult_select1(float * restrict x, const int N) { +; float sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > (float)0.) +; sum *= x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_0_fmult_select1( +; CHECK: %[[V1:.*]] = fcmp ogt float %[[V0:.*]], 0.000000e+00 +; CHECK-NEXT: %[[V3:.*]] = fmul float %[[V2:.*]], %[[V0]] +; CHECK-NEXT: select i1 %[[V1]], float %[[V3]], float %[[V2]] +define float @fcmp_0_fmult_select1(float* noalias %x, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %for.body, %for.header + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv + %0 = load float, float* %arrayidx, align 4 + %cmp.2 = fcmp ogt float %0, 0.000000e+00 + %mult = fmul float %sum.1, %0 + %sum.2 = select i1 %cmp.2, float %mult, float %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret float %1 +} + +; Double pattern: +; Check fmul not vectorized. +; +; double fcmp_0_fmult_select2(double * restrict x, const int N) { +; double sum = 0. +; for (int i = 0; i < N; ++i) +; if (x[i] > 0.) +; sum *= x[i]; +; return sum; +; } + +; CHECK-LABEL: @fcmp_0_fmult_select2( +; CHECK: %[[V1:.*]] = fcmp ogt double %[[V0:.*]], 0.000000e+00 +; CHECK-NEXT: %[[V3:.*]] = fmul double %[[V2:.*]], %[[V0]] +; CHECK-NEXT: select i1 %[[V1]], double %[[V3]], double %[[V2]] +define double @fcmp_0_fmult_select2(double* noalias %x, i32 %N) nounwind readonly { +entry: + %cmp.1 = icmp sgt i32 %N, 0 + br i1 %cmp.1, label %for.header, label %for.end + +for.header: ; preds = %entry + %zext = zext i32 %N to i64 + br label %for.body + +for.body: ; preds = %for.body, %for.header + %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ] + %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ] + %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv + %0 = load double, double* %arrayidx, align 4 + %cmp.2 = fcmp ogt double %0, 0.000000e+00 + %mult = fmul double %sum.1, %0 + %sum.2 = select i1 %cmp.2, double %mult, double %sum.1 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %zext + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ] + ret double %1 +}