Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7273,8 +7273,16 @@ return TTI.getShuffleCost(TargetTransformInfo::SK_ExtractSubvector, VectorTy, VF - 1, VectorTy); - // TODO: IF-converted IFs become selects. - return 0; + // Phi nodes in non-header blocks (not inductions, reductions, etc.) are + // converted into select instructions. We require N - 1 selects per phi + // node, where N is the number of incoming values. + if (VF > 1 && Phi->getParent() != TheLoop->getHeader()) + return (Phi->getNumIncomingValues() - 1) * + TTI.getCmpSelInstrCost( + Instruction::Select, ToVectorTy(Phi->getType(), VF), + ToVectorTy(Type::getInt1Ty(Phi->getContext()), VF)); + + return TTI.getCFInstrCost(Instruction::PHI); } case Instruction::UDiv: case Instruction::SDiv: Index: test/Transforms/LoopVectorize/phi-cost.ll =================================================================== --- /dev/null +++ test/Transforms/LoopVectorize/phi-cost.ll @@ -0,0 +1,80 @@ +; REQUIRES: asserts +; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s + +target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" + +; CHECK-LABEL: phi_two_incoming_values +; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction: %tmp5 = phi i32 [ %tmp1, %for.body ], [ %tmp4, %if.then ] +; CHECK: vector.body: +; CHECK: [[WIDE_LOAD:%.*]] = load <2 x i32>, <2 x i32>* {{.*}} +; CHECK: [[TMP5:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[WIDE_LOAD]], +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP5]], <2 x i32> [[TMP6]], <2 x i32> [[WIDE_LOAD]] +; CHECK: store <2 x i32> [[PREDPHI]], <2 x i32>* {{.*}} +; +define void @phi_two_incoming_values(i32* %a, i32* %b, i64 %n) { +entry: + br label %for.body + +for.body: + %i = phi i64 [ %i.next, %if.end ], [ 0, %entry ] + %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i + %tmp1 = load i32, i32* %tmp0, align 4 + %tmp2 = getelementptr inbounds i32, i32* %b, i64 %i + %tmp3 = icmp sgt i32 %tmp1, 0 + br i1 %tmp3, label %if.then, label %if.end + +if.then: + %tmp4 = add i32 %tmp1, 1 + br label %if.end + +if.end: + %tmp5 = phi i32 [ %tmp1, %for.body ], [ %tmp4, %if.then ] + store i32 %tmp5, i32* %tmp2, align 4 + %i.next = add i64 %i, 1 + %cond = icmp eq i64 %i, %n + br i1 %cond, label %for.end, label %for.body + +for.end: + ret void +} + +; CHECK-LABEL: phi_three_incoming_values +; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction: %tmp8 = phi i32 [ 9, %for.body ], [ 3, %if.then ], [ %tmp7, %if.else ] +; CHECK: vector.body: +; CHECK: [[PREDPHI:%.*]] = select <2 x i1> {{.*}}, <2 x i32> , <2 x i32> +; CHECK: [[PREDPHI7:%.*]] = select <2 x i1> {{.*}}, <2 x i32> {{.*}}, <2 x i32> [[PREDPHI]] +; CHECK: store <2 x i32> [[PREDPHI7]], <2 x i32>* {{.*}} +; +define void @phi_three_incoming_values(i32* %a, i32* %b, i64 %n) { +entry: + br label %for.body + +for.body: + %i = phi i64 [ %i.next, %if.end ], [ 0, %entry ] + %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i + %tmp1 = load i32, i32* %tmp0, align 4 + %tmp2 = getelementptr inbounds i32, i32* %b, i64 %i + %tmp3 = load i32, i32* %tmp2, align 4 + %tmp4 = icmp sgt i32 %tmp1, %tmp3 + br i1 %tmp4, label %if.then, label %if.end + +if.then: + %tmp5 = icmp sgt i32 %tmp1, 19 + br i1 %tmp5, label %if.end, label %if.else + +if.else: + %tmp6 = icmp slt i32 %tmp3, 4 + %tmp7 = select i1 %tmp6, i32 4, i32 5 + br label %if.end + +if.end: + %tmp8 = phi i32 [ 9, %for.body ], [ 3, %if.then ], [ %tmp7, %if.else ] + store i32 %tmp8, i32* %tmp0, align 4 + %i.next = add i64 %i, 1 + %cond = icmp eq i64 %i, %n + br i1 %cond, label %for.end, label %for.body + +for.end: + ret void +}