LoopVectorize: extend SelectCmp pattern for non-invariants

[artagnon]

Summary

Extend LoopVectorize's SelectCmp pattern for non-invariant statements,
and get LoopUtils' createCmpSelectTargetReduction to CodeGen for this
case. Consider the following example:

int src[n] = {4, 5, 2};
int r = 331;
for (int i = 0; i < n; i++) {
  if (src[i] > 3)
    r = i;
}
return r;

Here, r = i is non-invariant, and CodeGen'ing for this case involves the
following:

1. Create a Splat with the int_min/int_max values, depending on the loop direction.
2. The Src is filled with values: {0, 1, 2, 3, 4, ...}.
3. The Right is filled with values: {0, 1, 331, 331, 331, ...}.
4. The CmpVector is filled with values: {1, 1, 0, 0, 0, ...}.
5. Select using this CmpVector between Src and the Splat with int_min/int_max values.
6. Use a max-reduce/min-reduce on the result of the Select.
7. Use the exising Cmp which determines whether or not any assignment took place in the loop to select between the result of the max-reduce/min-reduce, and the initial value (InitVal).

Hence, the above case is vectorized.

[artagnon]

More comments in LoopUtils.cpp.

[shiva0217]

llvm/lib/Analysis/IVDescriptors.cpp

678

For the following case, the final value might not be able to pick by the min/max reduction.

int test(int a[32000]) {
   int k = -1;
   for (int i = 0; i < 32000; i++)
     if (a[i] < 0)
        k = a[i];
 
     return k;
 }

Extra checking of NonPhi might be needed. So the reduction can generate min/max.

auto *AR = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(NonPhi));
if (AR) {
  const SCEV *Step = AR->getStepRecurrence(*SE);
  const SCEVConstant *ConstStep = dyn_cast<SCEVConstant>(Step);
  if (!ConstStep && !SE->isLoopInvariant(Step, Loop))
    return InstDesc(false, I);
} else
  return InstDesc(false, I);

Would it need to check AR->hasNoUnsignedWrap?

[artagnon]

Address @shiva0217's comment, and fix miscompile for the following case:

int a[n] = {0, 0, 4, 3};
int k = a[0];
for (int i = 0; i < n; i++)
  if (a[i] > 2)
    k = a[i];
return k;

Here, instead of i being maximized, a[i] is maximized. To forbid this
case, let us check that the SCEV AddRec expression has a constant step,
and not vectorize this case at all.

[artagnon]

Will redo this patch and re-open a new review request.

llvm/include/llvm/Analysis/IVDescriptors.h

public:
   /// Returns a struct describing if the instruction 'I' can be a recurrence
   /// variable of type 'Kind' for a Loop \p L and reduction PHI \p Phi.
   /// If the recurrence is a min/max pattern of select(icmp()) this function
   /// advances the instruction pointer 'I' from the compare instruction to the
   /// select instruction and stores this pointer in 'PatternLastInst' member of
   /// the returned struct.
   static InstDesc isRecurrenceInstr(Loop *L, PHINode *Phi, Instruction *I,
                                     RecurKind Kind, InstDesc &Prev,
-                                    FastMathFlags FuncFMF);
+                                    FastMathFlags FuncFMF, ScalarEvolution *SE);
 
   /// Returns true if instruction I has multiple uses in Insts
   static bool hasMultipleUsesOf(Instruction *I,
                                 SmallPtrSetImpl<Instruction *> &Insts,
                                 unsigned MaxNumUses);
 
   /// Returns true if all uses of the instruction I is within the Set.
   static bool areAllUsesIn(Instruction *I, SmallPtrSetImpl<Instruction *> &Set);
 
   /// Returns a struct describing if the instruction is a llvm.(s/u)(min/max),
   /// llvm.minnum/maxnum or a Select(ICmp(X, Y), X, Y) pair of instructions
   /// corresponding to a min(X, Y) or max(X, Y), matching the recurrence kind \p
   /// Kind. \p Prev specifies the description of an already processed select
   /// instruction, so its corresponding cmp can be matched to it.
   static InstDesc isMinMaxPattern(Instruction *I, RecurKind Kind,
                                   const InstDesc &Prev);
 
   /// Returns a struct describing whether the instruction is either a
   ///   Select(ICmp(A, B), X, Y), or
   ///   Select(FCmp(A, B), X, Y)
   /// where one of (X, Y) is a loop invariant integer and the other is a PHI
   /// value. \p Prev specifies the description of an already processed select
   /// instruction, so its corresponding cmp can be matched to it.
   static InstDesc isSelectCmpPattern(Loop *Loop, PHINode *OrigPhi,
-                                     Instruction *I, InstDesc &Prev);
+                                     Instruction *I, InstDesc &Prev,
+                                     ScalarEvolution *SE);
 
   /// Returns a struct describing if the instruction is a
   /// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern.
   static InstDesc isConditionalRdxPattern(RecurKind Kind, Instruction *I);
 
   /// Returns identity corresponding to the RecurrenceKind.
   Value *getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const;
 

llvm/include/llvm/Transforms/Utils/LoopUtils.h

 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
 #define LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
 
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 
 namespace llvm {
 
 template <typename T> class DomTreeNodeBase;
 using DomTreeNode = DomTreeNodeBase<BasicBlock>;
 class AssumptionCache;
 class StringRef;

 
 /// Create a target reduction of the given vector \p Src for a reduction of the
 /// kind RecurKind::SelectICmp or RecurKind::SelectFCmp. The reduction operation
 /// is described by \p Desc.
 Value *createSelectCmpTargetReduction(IRBuilderBase &B,
                                       const TargetTransformInfo *TTI,
                                       Value *Src,
                                       const RecurrenceDescriptor &Desc,
-                                      PHINode *OrigPhi);
+                                      PHINode *OrigPhi, Loop *OrigLoop,
+                                      Loop::LoopBounds::Direction Direction);
 
 /// Create a generic target reduction using a recurrence descriptor \p Desc
 /// The target is queried to determine if intrinsics or shuffle sequences are
 /// required to implement the reduction.
 /// Fast-math-flags are propagated using the RecurrenceDescriptor.
 Value *createTargetReduction(IRBuilderBase &B, const TargetTransformInfo *TTI,
                              const RecurrenceDescriptor &Desc, Value *Src,
-                             PHINode *OrigPhi = nullptr);
+                             PHINode *OrigPhi = nullptr,
+                             Loop *OrigLoop = nullptr,
+                             Loop::LoopBounds::Direction Direction =
+                                 Loop::LoopBounds::Direction::Unknown);
 
 /// Create an ordered reduction intrinsic using the given recurrence
 /// descriptor \p Desc.
 Value *createOrderedReduction(IRBuilderBase &B,
                               const RecurrenceDescriptor &Desc, Value *Src,
                               Value *Start);
 
 /// Get the intersection (logical and) of all of the potential IR flags

llvm/lib/Analysis/IVDescriptors.cpp

if (!Cur->isCommutative() && !IsAPhi && !isa<SelectInst>(Cur) &&
         !VisitedInsts.count(dyn_cast<Instruction>(Cur->getOperand(0))))
       return false;
 
     // Any reduction instruction must be of one of the allowed kinds. We ignore
     // the starting value (the Phi or an AND instruction if the Phi has been
     // type-promoted).
     if (Cur != Start) {
       ReduxDesc =
-          isRecurrenceInstr(TheLoop, Phi, Cur, Kind, ReduxDesc, FuncFMF);
+          isRecurrenceInstr(TheLoop, Phi, Cur, Kind, ReduxDesc, FuncFMF, SE);
       ExactFPMathInst = ExactFPMathInst == nullptr
                             ? ReduxDesc.getExactFPMathInst()
                             : ExactFPMathInst;
       if (!ReduxDesc.isRecurrence())
         return false;
       // FIXME: FMF is allowed on phi, but propagation is not handled correctly.
       if (isa<FPMathOperator>(ReduxDesc.getPatternInst()) && !IsAPhi) {
         FastMathFlags CurFMF = ReduxDesc.getPatternInst()->getFastMathFlags();
for (User *U : Cur->users()) {
             return false;
           }
           NonPHIs.push_back(UI);
         }
       } else if (!isa<PHINode>(UI) &&
                  ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) &&
                    !isa<SelectInst>(UI)) ||
                   (!isConditionalRdxPattern(Kind, UI).isRecurrence() &&
-                   !isSelectCmpPattern(TheLoop, Phi, UI, IgnoredVal)
+                   !isSelectCmpPattern(TheLoop, Phi, UI, IgnoredVal, SE)
                         .isRecurrence() &&
                    !isMinMaxPattern(UI, Kind, IgnoredVal).isRecurrence())))
         return false;
 
       // Remember that we completed the cycle.
       if (UI == Phi)
         FoundStartPHI = true;
     }
bool RecurrenceDescriptor::AddReductionVar(
   RecurrenceDescriptor RD(RdxStart, ExitInstruction, IntermediateStore, Kind,
                           FMF, ExactFPMathInst, RecurrenceType, IsSigned,
                           IsOrdered, CastInsts, MinWidthCastToRecurrenceType);
   RedDes = RD;
 
   return true;
 }
 
-// We are looking for loops that do something like this:
+// We are looking for two patterns.
+//
+// The first is loops that do something like this:
 //   int r = 0;
 //   for (int i = 0; i < n; i++) {
 //     if (src[i] > 3)
 //       r = 3;
 //   }
 // where the reduction value (r) only has two states, in this example 0 or 3.
 // The generated LLVM IR for this type of loop will be like this:
 //   for.body:
 //     %r = phi i32 [ %spec.select, %for.body ], [ 0, %entry ]
 //     ...
 //     %cmp = icmp sgt i32 %5, 3
 //     %spec.select = select i1 %cmp, i32 3, i32 %r
 //     ...
 // In general we can support vectorization of loops where 'r' flips between
 // any two non-constants, provided they are loop invariant. The only thing
 // we actually care about at the end of the loop is whether or not any lane
 // in the selected vector is different from the start value. The final
 // across-vector reduction after the loop simply involves choosing the start
 // value if nothing changed (0 in the example above) or the other selected
 // value (3 in the example above).
+//
+// The second is loops that do something like this:
+//   int r = 0;
+//   for (int i = 0; i < n; i++) {
+//     if (src[i] > 3)
+//       r = i;
+//   }
+// where the reduction value (r) has many states, in this example 0 or the
+// maximal value of the loop induction variable, with the mask src[i] > 3.
+//
+// In general we can support vectorization of loops where 'r' is a variable. The
+// only thing we actually care about at the end of the loop is whether or not
+// any lane in the selected vector is different from the start value. The final
+// across-vector min/max reduction after the loop involves setting the value
+// from a mask, depening on whether i is increasing or decreasing in the loop.
 RecurrenceDescriptor::InstDesc
 RecurrenceDescriptor::isSelectCmpPattern(Loop *Loop, PHINode *OrigPhi,
-                                         Instruction *I, InstDesc &Prev) {
+                                         Instruction *I, InstDesc &Prev,
+                                         ScalarEvolution *SE) {
   // We must handle the select(cmp(),x,y) as a single instruction. Advance to
   // the select.
   CmpInst::Predicate Pred;
   if (match(I, m_OneUse(m_Cmp(Pred, m_Value(), m_Value())))) {
     if (auto *Select = dyn_cast<SelectInst>(*I->user_begin()))
       return InstDesc(Select, Prev.getRecKind());
   }
 
   // Only match select with single use cmp condition.
   if (!match(I, m_Select(m_OneUse(m_Cmp(Pred, m_Value(), m_Value())), m_Value(),
                          m_Value())))
     return InstDesc(false, I);
 
   SelectInst *SI = cast<SelectInst>(I);
   Value *NonPhi = nullptr;
 
   if (OrigPhi == dyn_cast<PHINode>(SI->getTrueValue()))
     NonPhi = SI->getFalseValue();
   else if (OrigPhi == dyn_cast<PHINode>(SI->getFalseValue()))
     NonPhi = SI->getTrueValue();
   else
     return InstDesc(false, I);
 
-  // We are looking for selects of the form:
+  // In the loop-invariant case, we are looking for selects of the form:
   //   select(cmp(), phi, loop_invariant) or
   //   select(cmp(), loop_invariant, phi)
-  if (!Loop->isLoopInvariant(NonPhi))
+  // If the loop is not invariant, we need to know the direction of the loop.
+  auto Bounds = SE ? Loop->getBounds(*SE) : std::nullopt;
+  if (!Loop->isLoopInvariant(NonPhi) &&
+      (!Bounds ||
+       Bounds->getDirection() == Loop::LoopBounds::Direction::Unknown)) {
     return InstDesc(false, I);
+  }
 
   return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::SelectICmp
                                                      : RecurKind::SelectFCmp);
 }
 
 RecurrenceDescriptor::InstDesc
 RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
                                       const InstDesc &Prev) {
RecurrenceDescriptor::isConditionalRdxPattern(RecurKind Kind, Instruction *I) {
   Instruction *IPhi = isa<PHINode>(*Op1) ? dyn_cast<Instruction>(Op1)
                                          : dyn_cast<Instruction>(Op2);
   if (!IPhi || IPhi != FalseVal)
     return InstDesc(false, I);
 
   return InstDesc(true, SI);
 }
 
-RecurrenceDescriptor::InstDesc
-RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi,
-                                        Instruction *I, RecurKind Kind,
-                                        InstDesc &Prev, FastMathFlags FuncFMF) {
+RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
+    Loop *L, PHINode *OrigPhi, Instruction *I, RecurKind Kind, InstDesc &Prev,
+    FastMathFlags FuncFMF, ScalarEvolution *SE) {
   assert(Prev.getRecKind() == RecurKind::None || Prev.getRecKind() == Kind);
   switch (I->getOpcode()) {
   default:
     return InstDesc(false, I);
   case Instruction::PHI:
     return InstDesc(I, Prev.getRecKind(), Prev.getExactFPMathInst());
   case Instruction::Sub:
   case Instruction::Add:
case Instruction::Select:
     if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
         Kind == RecurKind::Add || Kind == RecurKind::Mul)
       return isConditionalRdxPattern(Kind, I);
     [[fallthrough]];
   case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Call:
     if (isSelectCmpRecurrenceKind(Kind))
-      return isSelectCmpPattern(L, OrigPhi, I, Prev);
+      return isSelectCmpPattern(L, OrigPhi, I, Prev, SE);
     if (isIntMinMaxRecurrenceKind(Kind) ||
         (((FuncFMF.noNaNs() && FuncFMF.noSignedZeros()) ||
           (isa<FPMathOperator>(I) && I->hasNoNaNs() &&
            I->hasNoSignedZeros())) &&
          isFPMinMaxRecurrenceKind(Kind)))
       return isMinMaxPattern(I, Kind, Prev);
     else if (isFMulAddIntrinsic(I))
       return InstDesc(Kind == RecurKind::FMulAdd, I,

llvm/lib/Transforms/Utils/LoopUtils.cpp

if (Op != Instruction::ICmp && Op != Instruction::FCmp) {
              "Invalid min/max");
       TmpVec = createMinMaxOp(Builder, RdxKind, TmpVec, Shuf);
     }
   }
   // The result is in the first element of the vector.
   return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
 }
 
-Value *llvm::createSelectCmpTargetReduction(IRBuilderBase &Builder,
-                                            const TargetTransformInfo *TTI,
-                                            Value *Src,
-                                            const RecurrenceDescriptor &Desc,
-                                            PHINode *OrigPhi) {
+Value *llvm::createSelectCmpTargetReduction(
+    IRBuilderBase &Builder, const TargetTransformInfo *TTI, Value *Src,
+    const RecurrenceDescriptor &Desc, PHINode *OrigPhi, Loop *OrigLoop,
+    Loop::LoopBounds::Direction Direction) {
   assert(RecurrenceDescriptor::isSelectCmpRecurrenceKind(
              Desc.getRecurrenceKind()) &&
          "Unexpected reduction kind");
   Value *InitVal = Desc.getRecurrenceStartValue();
   Value *NewVal = nullptr;
 
   // First use the original phi to determine the new value we're trying to
   // select from in the loop.
assert(SI->getFalseValue() == OrigPhi &&
            "At least one input to the select should be the original Phi");
     NewVal = SI->getTrueValue();
   }
 
   // Create a splat vector with the new value and compare this to the vector
   // we want to reduce.
   ElementCount EC = cast<VectorType>(Src->getType())->getElementCount();
   Value *Right = Builder.CreateVectorSplat(EC, InitVal);
-  Value *Cmp =
+  Value *CmpVector =
       Builder.CreateCmp(CmpInst::ICMP_NE, Src, Right, "rdx.select.cmp");
+  Value *Cmp = Builder.CreateOrReduce(CmpVector);
+
+  if (!OrigLoop->isLoopInvariant(NewVal)) {
+    // If NewVal isn't loop invariant, consider the following example:
+    //   int src[n] = {4, 5, 2};
+    //   int r = 331;
+    //   for (int i = 0; i < n; i++) {
+    //     if (src[i] > 3)
+    //       r = i;
+    //   }
+    //   return r;
+    // We vectorize this case as follows:
+    // 1. Create a Splat with the int_min/int_max values, depending on the loop
+    // direction.
+    // 2. The Src is filled with values: {0, 1, 2, 3, 4, ...}.
+    // 3. The Right is filled with values: {0, 1, 331, 331, 331, ...}.
+    // 4. The CmpVector is filled with values: {1, 1, 0, 0, 0, ...}.
+    // 5. Select using this CmpVector between Src and the Splat with
+    // int_min/int_max values.
+    // 6. Use a max-reduce/min-reduce on the result of the Select.
+    // 7. Use the exising Cmp which determines whether or not any assignment
+    // took place in the loop to select between the result of the
+    // max-reduce/min-reduce, and the initial value (InitVal).
+    assert(Direction != Loop::LoopBounds::Direction::Unknown &&
+           "Unknown loop direction");
+    auto IdxTy = cast<IntegerType>(InitVal->getType());
+    if (Direction == Loop::LoopBounds::Direction::Increasing) {
+      auto MinValue = Desc.isSigned()
+                          ? APInt::getSignedMinValue(IdxTy->getBitWidth())
+                          : APInt::getMinValue(IdxTy->getBitWidth());
+      Value *MinSplat =
+          Builder.CreateVectorSplat(EC, ConstantInt::get(IdxTy, MinValue));
+      Value *Mask =
+          Builder.CreateSelect(CmpVector, Src, MinSplat, "rdx.select.mask");
+      NewVal = Builder.CreateIntMaxReduce(Mask, Desc.isSigned());
+    } else {
+      auto MaxValue = Desc.isSigned()
+                          ? APInt::getSignedMaxValue(IdxTy->getBitWidth())
+                          : APInt::getMaxValue(IdxTy->getBitWidth());
+      Value *MaxSplat =
+          Builder.CreateVectorSplat(EC, ConstantInt::get(IdxTy, MaxValue));
+      Value *Mask =
+          Builder.CreateSelect(CmpVector, Src, MaxSplat, "rdx.select.mask");
+      NewVal = Builder.CreateIntMinReduce(Mask, Desc.isSigned());
+    }
+  }
 
-  // If any predicate is true it means that we want to select the new value.
-  Cmp = Builder.CreateOrReduce(Cmp);
   return Builder.CreateSelect(Cmp, NewVal, InitVal, "rdx.select");
 }
 
 Value *llvm::createSimpleTargetReduction(IRBuilderBase &Builder,
                                          const TargetTransformInfo *TTI,
                                          Value *Src, RecurKind RdxKind) {
   auto *SrcVecEltTy = cast<VectorType>(Src->getType())->getElementType();
   switch (RdxKind) {
Value *llvm::createSimpleTargetReduction(IRBuilderBase &Builder,
   default:
     llvm_unreachable("Unhandled opcode");
   }
 }
 
 Value *llvm::createTargetReduction(IRBuilderBase &B,
                                    const TargetTransformInfo *TTI,
                                    const RecurrenceDescriptor &Desc, Value *Src,
-                                   PHINode *OrigPhi) {
+                                   PHINode *OrigPhi, Loop *OrigLoop,
+                                   Loop::LoopBounds::Direction Direction) {
   // TODO: Support in-order reductions based on the recurrence descriptor.
   // All ops in the reduction inherit fast-math-flags from the recurrence
   // descriptor.
   IRBuilderBase::FastMathFlagGuard FMFGuard(B);
   B.setFastMathFlags(Desc.getFastMathFlags());
 
   RecurKind RK = Desc.getRecurrenceKind();
   if (RecurrenceDescriptor::isSelectCmpRecurrenceKind(RK))
-    return createSelectCmpTargetReduction(B, TTI, Src, Desc, OrigPhi);
+    return createSelectCmpTargetReduction(B, TTI, Src, Desc, OrigPhi, OrigLoop,
+                                          Direction);
 
   return createSimpleTargetReduction(B, TTI, Src, RK);
 }
 
 Value *llvm::createOrderedReduction(IRBuilderBase &B,
                                     const RecurrenceDescriptor &Desc,
                                     Value *Src, Value *Start) {
   assert((Desc.getRecurrenceKind() == RecurKind::FAdd ||

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
     // of the original loop header may change as the transformation happens.
     OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
         OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
 
     if (MinProfitableTripCount.isZero())
       this->MinProfitableTripCount = VecWidth;
     else
       this->MinProfitableTripCount = MinProfitableTripCount;
+
+    auto Bounds = OrigLoop->getBounds(*PSE.getSE());
+    if (!Bounds ||
+        Bounds->getDirection() == Loop::LoopBounds::Direction::Unknown)
+      Direction = Loop::LoopBounds::Direction::Unknown;
+    else
+      Direction = Bounds->getDirection();
   }
 
   virtual ~InnerLoopVectorizer() = default;
 
   /// Create a new empty loop that will contain vectorized instructions later
   /// on, while the old loop will be used as the scalar remainder. Control flow
   /// is generated around the vectorized (and scalar epilogue) loops consisting
   /// of various checks and bypasses. Return the pre-header block of the new
protected:
   /// Allow subclasses to override and print debug traces before/after vplan
   /// execution, when trace information is requested.
   virtual void printDebugTracesAtStart(){};
   virtual void printDebugTracesAtEnd(){};
 
   /// The original loop.
   Loop *OrigLoop;
 
+  /// The direction of iteration of the original loop.
+  Loop::LoopBounds::Direction Direction;
+
   /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
   /// dynamic knowledge to simplify SCEV expressions and converts them to a
   /// more usable form.
   PredicatedScalarEvolution &PSE;
 
   /// Loop Info.
   LoopInfo *LI;
 
for (unsigned Part = 1; Part < UF; ++Part) {
         ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
     }
   }
 
   // Create the reduction after the loop. Note that inloop reductions create the
   // target reduction in the loop using a Reduction recipe.
   if (VF.isVector() && !PhiR->isInLoop()) {
     ReducedPartRdx =
-        createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx, OrigPhi);
+        createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx, OrigPhi,
+                              OrigLoop, State.ILV->Direction);
     // If the reduction can be performed in a smaller type, we need to extend
     // the reduction to the wider type before we branch to the original loop.
     if (PhiTy != RdxDesc.getRecurrenceType())
       ReducedPartRdx = RdxDesc.isSigned()
                            ? Builder.CreateSExt(ReducedPartRdx, PhiTy)
                            : Builder.CreateZExt(ReducedPartRdx, PhiTy);
   }
 

llvm/test/Transforms/LoopVectorize/if-reduction.ll

 
 for.end:                                          ; preds = %for.body, %entry
   %1 = phi i32 [ 0, %entry ], [ %sum.2, %for.body ]
   ret i32 %1
 }
 
 @table = constant [13 x i16] [i16 10, i16 35, i16 69, i16 147, i16 280, i16 472, i16 682, i16 1013, i16 1559, i16 2544, i16 4553, i16 6494, i16 10000], align 1 
 
-; CHECK-LABEL: @non_reduction_index(
-; CHECK-NOT:     <4 x i16>
-define i16 @non_reduction_index(i16 noundef %val) {
+; CHECK-LABEL: @reduction_index(
+; CHECK: %[[V1:.*]] = insertelement <4 x i16> poison, i16 %[[ARG:.*]], i64 0
+; CHECK: shufflevector <4 x i16> %[[V1]], <4 x i16> poison, <4 x i32> zeroinitializer
+define i16 @reduction_index(i16 noundef %val) {
 entry:
   br label %for.body
 
 for.cond.cleanup:                                 ; preds = %for.body
   %spec.select.lcssa = phi i16 [ %spec.select, %for.body ]
   ret i16 %spec.select.lcssa
 
 for.body:                                         ; preds = %entry, %for.body
   %i.05 = phi i16 [ 12, %entry ], [ %sub, %for.body ]
   %k.04 = phi i16 [ 0, %entry ], [ %spec.select, %for.body ]
   %arrayidx = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %i.05
   %0 = load i16, ptr %arrayidx, align 1
   %cmp1 = icmp ugt i16 %0, %val
   %sub = add nsw i16 %i.05, -1
   %spec.select = select i1 %cmp1, i16 %sub, i16 %k.04
   %cmp.not = icmp eq i16 %sub, 0
   br i1 %cmp.not, label %for.cond.cleanup, label %for.body
 }
 
 @tablef = constant [13 x half] [half 10.0, half 35.0, half 69.0, half 147.0, half 280.0, half 472.0, half 682.0, half 1013.0, half 1559.0, half 2544.0, half 4556.0, half 6496.0, half 10000.0], align 1 
 
-; CHECK-LABEL: @non_reduction_index_half(
-; CHECK-NOT:     <4 x half>
-define i16 @non_reduction_index_half(half noundef %val) {
+; CHECK-LABEL: @reduction_index_half(
+; CHECK: %[[V1:.*]] = insertelement <4 x half> poison, half %[[ARG:.*]], i64 0
+; CHECK: shufflevector <4 x half> %[[V1]], <4 x half> poison, <4 x i32> zeroinitializer
+define i16 @reduction_index_half(half noundef %val) {
 entry:
   br label %for.body
 
 for.cond.cleanup:                                 ; preds = %for.body
   %spec.select.lcssa = phi i16 [ %spec.select, %for.body ]
   ret i16 %spec.select.lcssa
 
 for.body:                                         ; preds = %entry, %for.body

llvm/test/Transforms/LoopVectorize/select-cmp.ll

for.body:                                      ; preds = %entry, %for.body
   %4 = add nuw nsw i64 %0, 1
   %5 = icmp eq i64 %4, %n
   br i1 %5, label %exit, label %for.body
 
 exit:                                     ; preds = %for.body
   ret i32 %3
 }
 
+define i32 @select_variant_i32_from_icmp(ptr nocapture readonly %v1, ptr nocapture readonly %v2, i64 %n) {
+; CHECK-LABEL: @select_variant_i32_from_icmp(
+; CHECK-VF4IC1-NEXT:  entry:
+; CHECK-VF4IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
+; CHECK-VF4IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-VF4IC1:       vector.ph:
+; CHECK-VF4IC1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VF4IC1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-VF4IC1:       vector.body:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 3, i32 3, i32 3, i32 3>, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[V1:%.*]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[V2:%.*]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK-VF4IC1-NEXT:    [[TMP6]] = select <4 x i1> [[TMP5]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD1]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-VF4IC1:       middle.block:
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne <4 x i32> [[TMP6]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[RDX_SELECT_CMP]])
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT_MASK:%.*]] = select <4 x i1> [[RDX_SELECT_CMP]], <4 x i32> [[TMP6]], <4 x i32> zeroinitializer
+; CHECK-VF4IC1-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[RDX_SELECT_MASK]])
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP8]], i32 [[TMP9]], i32 3
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-VF4IC1:       scalar.ph:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 3, [[ENTRY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-VF4IC1:       for.body:
+; CHECK-VF4IC1-NEXT:    [[TMP10:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[TMP18:%.*]], [[FOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP11:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP17:%.*]], [[FOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[V1]], i64 [[TMP10]]
+; CHECK-VF4IC1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[V2]], i64 [[TMP10]]
+; CHECK-VF4IC1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
+; CHECK-VF4IC1-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP13]], 3
+; CHECK-VF4IC1-NEXT:    [[TMP17]] = select i1 [[TMP16]], i32 [[TMP11]], i32 [[TMP15]]
+; CHECK-VF4IC1-NEXT:    [[TMP18]] = add nuw nsw i64 [[TMP10]], 1
+; CHECK-VF4IC1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[TMP18]], [[N]]
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP19]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-VF4IC1:       exit:
+; CHECK-VF4IC1-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[DOTLCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                      ; preds = %entry, %for.body
+  %0 = phi i64 [ 0, %entry ], [ %8, %for.body ]
+  %1 = phi i32 [ 3, %entry ], [ %7, %for.body ]
+  %2 = getelementptr inbounds i32, ptr %v1, i64 %0
+  %3 = load i32, ptr %2, align 4
+  %4 = getelementptr inbounds i32, ptr %v2, i64 %0
+  %5 = load i32, ptr %4, align 4
+  %6 = icmp eq i32 %3, 3
+  %7 = select i1 %6, i32 %1, i32 %5
+  %8 = add nuw nsw i64 %0, 1
+  %9 = icmp eq i64 %8, %n
+  br i1 %9, label %exit, label %for.body
+
+exit:                                     ; preds = %for.body
+  ret i32 %7
+}
+
+define i32 @test_select_variant_i32_from_fcmp(ptr %a) {
+; CHECK-LABEL: @test_select_variant_i32_from_fcmp(
+; CHECK-VF4IC1-NEXT:  entry:
+; CHECK-VF4IC1-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK-VF4IC1:       for.cond1.preheader:
+; CHECK-VF4IC1-NEXT:    [[NL_015:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC7:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
+; CHECK-VF4IC1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-VF4IC1:       vector.ph:
+; CHECK-VF4IC1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-VF4IC1:       vector.body:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP2]], align 8
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = fcmp ogt <4 x double> [[WIDE_LOAD]], <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>
+; CHECK-VF4IC1-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-VF4IC1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-VF4IC1:       middle.block:
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne <4 x i32> [[TMP4]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-VF4IC1-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[RDX_SELECT_CMP]])
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT_MASK:%.*]] = select <4 x i1> [[RDX_SELECT_CMP]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[RDX_SELECT_MASK]])
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP6]], i32 [[TMP7]], i32 -1
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 32000, 32000
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]]
+; CHECK-VF4IC1:       scalar.ph:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 32000, [[MIDDLE_BLOCK]] ], [ 0, [[FOR_COND1_PREHEADER]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ -1, [[FOR_COND1_PREHEADER]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    br label [[FOR_BODY4:%.*]]
+; CHECK-VF4IC1:       for.cond.cleanup:
+; CHECK-VF4IC1-NEXT:    [[J_2_LCSSA_LCSSA:%.*]] = phi i32 [ [[J_2_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[J_2_LCSSA_LCSSA]]
+; CHECK-VF4IC1:       for.cond.cleanup3:
+; CHECK-VF4IC1-NEXT:    [[J_2_LCSSA]] = phi i32 [ [[J_2:%.*]], [[FOR_BODY4]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    [[INC7]] = add nuw nsw i32 [[NL_015]], 1
+; CHECK-VF4IC1-NEXT:    [[EXITCOND16_NOT:%.*]] = icmp eq i32 [[INC7]], 200000
+; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND16_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
+; CHECK-VF4IC1:       for.body4:
+; CHECK-VF4IC1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY4]] ]
+; CHECK-VF4IC1-NEXT:    [[J_113:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[J_2]], [[FOR_BODY4]] ]
+; CHECK-VF4IC1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-VF4IC1-NEXT:    [[CMP5:%.*]] = fcmp ogt double [[TMP8]], 3.000000e+00
+; CHECK-VF4IC1-NEXT:    [[TMP9:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-VF4IC1-NEXT:    [[J_2]] = select i1 [[CMP5]], i32 [[TMP9]], i32 [[J_113]]
+; CHECK-VF4IC1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 32000
+; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_BODY4]], !llvm.loop [[LOOP3:![0-9]+]]
+;
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %nl.015 = phi i32 [ 0, %entry ], [ %inc7, %for.cond.cleanup3 ]
+  br label %for.body4
+
+for.cond.cleanup:
+  ret i32 %j.2
+
+for.cond.cleanup3:
+  %inc7 = add nuw nsw i32 %nl.015, 1
+  %exitcond16.not = icmp eq i32 %inc7, 200000
+  br i1 %exitcond16.not, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.body4:
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body4 ]
+  %j.113 = phi i32 [ -1, %for.cond1.preheader ], [ %j.2, %for.body4 ]
+  %arrayidx = getelementptr inbounds double, ptr %a, i64 %indvars.iv
+  %0 = load double, ptr %arrayidx
+  %cmp5 = fcmp ogt double %0, 3.000000e+00
+  %1 = trunc i64 %indvars.iv to i32
+  %j.2 = select i1 %cmp5, i32 %1, i32 %j.113
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, 32000
+  br i1 %exitcond.not, label %for.cond.cleanup3, label %for.body4
+}
+
+define i32 @test_select_variant_i32_from_fcmp_reverse(ptr %a) {
+; CHECK-LABEL: @test_select_variant_i32_from_fcmp_reverse(
+; CHECK-VF4IC1-NEXT:  entry:
+; CHECK-VF4IC1-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK-VF4IC1:       for.cond1.preheader:
+; CHECK-VF4IC1-NEXT:    [[NL_014:%.*]] = phi i32 [ 199999, [[ENTRY:%.*]] ], [ [[DEC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
+; CHECK-VF4IC1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-VF4IC1:       vector.ph:
+; CHECK-VF4IC1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-VF4IC1:       vector.body:
+; CHECK-VF4IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-VF4IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[TMP0]]
+; CHECK-VF4IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 0
+; CHECK-VF4IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP2]], align 8
+; CHECK-VF4IC1-NEXT:    [[TMP3:%.*]] = fcmp ogt <4 x double> [[WIDE_LOAD]], <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>
+; CHECK-VF4IC1-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
+; CHECK-VF4IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-VF4IC1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
+; CHECK-VF4IC1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VF4IC1:       middle.block:
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne <4 x i32> [[TMP4]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-VF4IC1-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[RDX_SELECT_CMP]])
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT_MASK:%.*]] = select <4 x i1> [[RDX_SELECT_CMP]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer
+; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[RDX_SELECT_MASK]])
+; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP6]], i32 [[TMP7]], i32 -1
+; CHECK-VF4IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 32000, 32000
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]]
+; CHECK-VF4IC1:       scalar.ph:
+; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 32000, [[MIDDLE_BLOCK]] ], [ 0, [[FOR_COND1_PREHEADER]] ]
+; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ -1, [[FOR_COND1_PREHEADER]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    br label [[FOR_BODY4:%.*]]
+; CHECK-VF4IC1:       for.cond.cleanup:
+; CHECK-VF4IC1-NEXT:    [[J_2_LCSSA_LCSSA:%.*]] = phi i32 [ [[J_2_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ]
+; CHECK-VF4IC1-NEXT:    ret i32 [[J_2_LCSSA_LCSSA]]
+; CHECK-VF4IC1:       for.cond.cleanup3:
+; CHECK-VF4IC1-NEXT:    [[J_2_LCSSA]] = phi i32 [ [[J_2:%.*]], [[FOR_BODY4]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT:    [[DEC]] = add nsw i32 [[NL_014]], -1
+; CHECK-VF4IC1-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[NL_014]], 0
+; CHECK-VF4IC1-NEXT:    br i1 [[CMP_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
+; CHECK-VF4IC1:       for.body4:
+; CHECK-VF4IC1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY4]] ]
+; CHECK-VF4IC1-NEXT:    [[J_112:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[J_2]], [[FOR_BODY4]] ]
+; CHECK-VF4IC1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-VF4IC1-NEXT:    [[TMP8:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-VF4IC1-NEXT:    [[CMP5:%.*]] = fcmp ogt double [[TMP8]], 3.000000e+00
+; CHECK-VF4IC1-NEXT:    [[TMP9:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-VF4IC1-NEXT:    [[J_2]] = select i1 [[CMP5]], i32 [[TMP9]], i32 [[J_112]]
+; CHECK-VF4IC1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-VF4IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 32000
+; CHECK-VF4IC1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_BODY4]], !llvm.loop [[LOOP5:![0-9]+]]
+;
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %nl.014 = phi i32 [ 199999, %entry ], [ %dec, %for.cond.cleanup3 ]
+  br label %for.body4
+
+for.cond.cleanup:
+  ret i32 %j.2
+
+for.cond.cleanup3:
+  %dec = add nsw i32 %nl.014, -1
+  %cmp.not = icmp eq i32 %nl.014, 0
+  br i1 %cmp.not, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.body4:
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body4 ]
+  %j.112 = phi i32 [ -1, %for.cond1.preheader ], [ %j.2, %for.body4 ]
+  %arrayidx = getelementptr inbounds double, ptr %a, i64 %indvars.iv
+  %0 = load double, ptr %arrayidx
+  %cmp5 = fcmp ogt double %0, 3.000000e+00
+  %1 = trunc i64 %indvars.iv to i32
+  %j.2 = select i1 %cmp5, i32 %1, i32 %j.112
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, 32000
+  br i1 %exitcond.not, label %for.cond.cleanup3, label %for.body4
+}
 
 ; Negative tests
 
 ; We don't support FP reduction variables at the moment.
 define float @select_const_f32_from_icmp(ptr nocapture readonly %v, i64 %n) {
 ; CHECK: @select_const_f32_from_icmp
 ; CHECK-NOT: vector.body
 entry:
for.body:                                      ; preds = %entry, %for.body
   %9 = icmp eq i64 %8, %n
   br i1 %9, label %exit, label %for.body
 
 exit:                                     ; preds = %for.body
   store i32 %7, ptr %v2, align 4
   ret i32 %6
 }
 
-
-; We don't support selecting loop-variant values.
-define i32 @select_variant_i32_from_icmp(ptr nocapture readonly %v1, ptr nocapture readonly %v2, i64 %n) {
-; CHECK-LABEL: @select_variant_i32_from_icmp
-; CHECK-NOT: vector.body
-entry:
-  br label %for.body
-
-for.body:                                      ; preds = %entry, %for.body
-  %0 = phi i64 [ 0, %entry ], [ %8, %for.body ]
-  %1 = phi i32 [ 3, %entry ], [ %7, %for.body ]
-  %2 = getelementptr inbounds i32, ptr %v1, i64 %0
-  %3 = load i32, ptr %2, align 4
-  %4 = getelementptr inbounds i32, ptr %v2, i64 %0
-  %5 = load i32, ptr %4, align 4
-  %6 = icmp eq i32 %3, 3
-  %7 = select i1 %6, i32 %1, i32 %5
-  %8 = add nuw nsw i64 %0, 1
-  %9 = icmp eq i64 %8, %n
-  br i1 %9, label %exit, label %for.body
-
-exit:                                     ; preds = %for.body
-  ret i32 %7
-}
-
-
 ; We only support selects where the input comes from the same PHI as the
 ; reduction PHI. In the example below, the select uses the induction
 ; variable input and the icmp uses the reduction PHI.
 define i32 @select_i32_from_icmp_non_redux_phi(i32 %a, i32 %b, i32 %n) {
 ; CHECK-LABEL: @select_i32_from_icmp_non_redux_phi
 ; CHECK-NOT: vector.body
 entry:
   br label %for.body

llvm/test/Transforms/LoopVectorize/select-min-index.ll

 ; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
-; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s | FileCheck %s
 
 ; Test cases for selecting the index with the minimum value.
 
 define i64 @test_vectorize_select_umin_idx(ptr %src) {
 ; CHECK-LABEL: @test_vectorize_select_umin_idx(
 ; CHECK-NOT:   vector.body:
 ;
 entry:
loop:
   %exitcond.not = icmp eq i64 %iv.next, 0
   br i1 %exitcond.not, label %exit, label %loop
 
 exit:
   %res = phi i64 [ %min.idx.next, %loop ]
   ret i64 %res
 }
 
-define i64 @test_not_vectorize_select_no_min_reduction(ptr %src) {
-; CHECK-LABEL: @test_not_vectorize_select_no_min_reduction(
-; CHECK-NOT:   vector.body:
+define i64 @test_vectorize_select_no_min_reduction(ptr %src) {
+; CHECK-LABEL: @test_vectorize_select_no_min_reduction(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 true, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[SRC:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3]] = add <4 x i64> [[WIDE_LOAD]], <i64 1, i64 1, i64 1, i64 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ugt <4 x i64> [[TMP4]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP6]] = select <4 x i1> [[TMP5]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne <4 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[RDX_SELECT_CMP]])
+; CHECK-NEXT:    [[RDX_SELECT_MASK:%.*]] = select <4 x i1> [[RDX_SELECT_CMP]], <4 x i64> [[TMP6]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[RDX_SELECT_MASK]])
+; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP8]], i64 [[TMP9]], i64 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 0, 0
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[SCALAR_RECUR]], [[L]]
+; CHECK-NEXT:    [[MIN_VAL_NEXT]] = add i64 [[L]], 1
+; CHECK-NEXT:    [[FOO:%.*]] = call i64 @llvm.umin.i64(i64 [[SCALAR_RECUR]], i64 [[L]])
+; CHECK-NEXT:    [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], [[LOOP]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
   br label %loop
 
 loop:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
   %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
   %min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
   %gep = getelementptr i64, ptr %src, i64 %iv
   %l = load i64, ptr %gep
   %cmp = icmp ugt i64 %min.val, %l
   %min.val.next = add i64 %l, 1
   %foo = call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
   %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, 0
   br i1 %exitcond.not, label %exit, label %loop
 
 exit:
   %res = phi i64 [ %min.idx.next, %loop ]
   ret i64 %res
 }
 
-
 define i64 @test_not_vectorize_cmp_value(i64 %x) {
 ; CHECK-LABEL: @test_not_vectorize_cmp_value(
 ; CHECK-NOT:   vector.body:
 ;
 entry:
   br label %loop
 
 loop: