[LV] Introduce TTI::getMinimumVF

[kparzysz]

Summary

TargetTransformInfo::getMinimumVF(ElementType) will return the target's minimum VF for a given element type. By default the minimum VF is 2. The VF=1 (i.e. not vectorizing) is still considered a valid VF, and if vectorization is not profitable, the minimum VF will not force it to happen.

This is a followup to D44523.

[kparzysz]

lib/Transforms/Vectorize/LoopVectorize.cpp

6088

This early exit introduced some changes in behavior. Originally, SmallestType and WidestType would be -1u and 8 respectively in case where no types were found. The rest of the calculations would then operate on those values.

With this change, in a few testcases, the loops were not vectorized, so the vectorization width was set via metadata. These testcases were not testing whether the vectorization actually occurs, but some other aspects of its functionality, so such changes did not seem unreasonable.

7447

This is somewhat suboptimal because the loop blocks are traversed twice, but I didn't want to make too many changes before the first review.

[hsaito]

This is a larger change than I was thinking. Making MaximizeBandwidth TTI based not good enough?

[kparzysz]

In D44574#1044055, @hsaito wrote:

This is a larger change than I was thinking. Making MaximizeBandwidth TTI based not good enough?

It sounds like a good idea to allow targets to enable it. I published D44735 to do it.

The advantage of the approach in this patch is that it gives a guarantee of the minimum VF. I'm not sure if the TTI callback for maximizing bandwidth will ensure that, since it still considers all powers of 2 between 1 and the MaxVF.

[kparzysz]

Replaced by D45271.

include/llvm/Analysis/TargetTransformInfo.h

unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
                                VectorType *VecTy) const;
 
   /// \returns The new vector factor value if the target doesn't support \p
   /// SizeInBytes stores or has a better vector factor.
   unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
                                 unsigned ChainSizeInBytes,
                                 VectorType *VecTy) const;
 
+  /// \returns The minimum vector factor for vectors of elements of type
+  /// \p ElementTy.
+  unsigned getMinimumVF(Type *ElementTy) const;
+
   /// Flags describing the kind of vector reduction.
   struct ReductionFlags {
     ReductionFlags() : IsMaxOp(false), IsSigned(false), NoNaN(false) {}
     bool IsMaxOp;  ///< If the op a min/max kind, true if it's a max operation.
     bool IsSigned; ///< Whether the operation is a signed int reduction.
     bool NoNaN;    ///< If op is an fp min/max, whether NaNs may be present.
   };
 
virtual bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
                                             unsigned Alignment,
                                             unsigned AddrSpace) const = 0;
   virtual unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
                                        unsigned ChainSizeInBytes,
                                        VectorType *VecTy) const = 0;
   virtual unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
                                         unsigned ChainSizeInBytes,
                                         VectorType *VecTy) const = 0;
+  virtual unsigned getMinimumVF(Type *ElementTy) const = 0;
   virtual bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                                      ReductionFlags) const = 0;
   virtual bool shouldExpandReduction(const IntrinsicInst *II) const = 0;
   virtual int getInstructionLatency(const Instruction *I) = 0;
 };
 
 template <typename T>
 class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
                                VectorType *VecTy) const override {
     return Impl.getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
   }
   unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
                                 unsigned ChainSizeInBytes,
                                 VectorType *VecTy) const override {
     return Impl.getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
   }
+  unsigned getMinimumVF(Type *ElementTy) const override {
+    return Impl.getMinimumVF(ElementTy);
+  }
   bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                              ReductionFlags Flags) const override {
     return Impl.useReductionIntrinsic(Opcode, Ty, Flags);
   }
   bool shouldExpandReduction(const IntrinsicInst *II) const override {
     return Impl.shouldExpandReduction(II);
   }
   int getInstructionLatency(const Instruction *I) override {

include/llvm/Analysis/TargetTransformInfoImpl.h

public:
   }
 
   unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
                                 unsigned ChainSizeInBytes,
                                 VectorType *VecTy) const {
     return VF;
   }
 
+  unsigned getMinimumVF(Type *ElementTy) const {
+    return 2;
+  }
+
   bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                              TTI::ReductionFlags Flags) const {
     return false;
   }
 
   bool shouldExpandReduction(const IntrinsicInst *II) const {
     return true;
   }

lib/Analysis/TargetTransformInfo.cpp

 
 unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
                                                    unsigned StoreSize,
                                                    unsigned ChainSizeInBytes,
                                                    VectorType *VecTy) const {
   return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
 }
 
+unsigned TargetTransformInfo::getMinimumVF(Type *ElementTy) const {
+  return TTIImpl->getMinimumVF(ElementTy);
+}
+
 bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode,
                                                 Type *Ty, ReductionFlags Flags) const {
   return TTIImpl->useReductionIntrinsic(Opcode, Ty, Flags);
 }
 
 bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
   return TTIImpl->shouldExpandReduction(II);
 }

lib/Transforms/Vectorize/LoopVectorize.cpp

LoopVectorizationCostModel(Loop *L, PredicatedScalarEvolution &PSE,
                              const TargetTransformInfo &TTI,
                              const TargetLibraryInfo *TLI, DemandedBits *DB,
                              AssumptionCache *AC,
                              OptimizationRemarkEmitter *ORE, const Function *F,
                              const LoopVectorizeHints *Hints)
       : TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), DB(DB),
         AC(AC), ORE(ORE), TheFunction(F), Hints(Hints) {}
 
-  /// \return An upper bound for the vectorization factor, or None if
-  /// vectorization should be avoided up front.
-  Optional<unsigned> computeMaxVF(bool OptForSize);
+  /// \return A lower and an upper bounds for the vectorization factor,
+  /// or None if vectorization should be avoided up front.
+  Optional<std::pair<unsigned,unsigned>> computeMaxVF(bool OptForSize);
 
   /// \return The most profitable vectorization factor and the cost of that VF.
-  /// This method checks every power of two up to MaxVF. If UserVF is not ZERO
-  /// then this vectorization factor will be selected if vectorization is
-  /// possible.
-  VectorizationFactor selectVectorizationFactor(unsigned MaxVF);
+  /// This method checks every power of two from MinVF up to MaxVF. If UserVF
+  /// is not ZERO then this vectorization factor will be selected if
+  /// vectorization is possible.
+  VectorizationFactor selectVectorizationFactor(unsigned MinVF, unsigned MaxVF);
 
   /// Setup cost-based decisions for user vectorization factor.
   void selectUserVectorizationFactor(unsigned UserVF) {
     collectUniformsAndScalars(UserVF);
     collectInstsToScalarize(UserVF);
   }
 
   /// \return The size (in bits) of the smallest and widest types in the code
   /// that needs to be vectorized. We ignore values that remain scalar such as
   /// 64 bit loop indices.
-  std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
+  SmallVector<Type*,4> getSmallestAndWidestTypes();
 
   /// \return The desired interleave count.
   /// If interleave count has been specified by metadata it will be returned.
   /// Otherwise, the interleave count is computed and returned. VF and LoopCost
   /// are the selected vectorization factor and the cost of the selected VF.
   unsigned selectInterleaveCount(bool OptForSize, unsigned VF,
                                  unsigned LoopCost);
 
public:
 
   /// Returns true if \p I is a memory instruction with consecutive memory
   /// access that can be widened.
   bool memoryInstructionCanBeWidened(Instruction *I, unsigned VF = 1);
 
 private:
   unsigned NumPredStores = 0;
 
-  /// \return An upper bound for the vectorization factor, larger than zero.
-  /// One is returned if vectorization should best be avoided due to cost.
-  unsigned computeFeasibleMaxVF(bool OptForSize, unsigned ConstTripCount);
+  /// \return A lower and an upper bound for the vectorization factor,
+  /// larger than zero. One is returned if vectorization should best be
+  /// avoided due to cost.
+  std::pair<unsigned,unsigned> computeFeasibleMaxVF(bool OptForSize,
+                                                    unsigned ConstTripCount);
 
   /// The vectorization cost is a combination of the cost itself and a boolean
   /// indicating whether any of the contributing operations will actually
   /// operate on
   /// vector values after type legalization in the backend. If this latter value
   /// is
   /// false, then all operations will be scalarized (i.e. no vectorization has
   /// actually taken place).
if (LastMember) {
         continue;
       }
       DEBUG(dbgs() << "LV: Interleaved group requires epilogue iteration.\n");
       RequiresScalarEpilogue = true;
     }
   }
 }
 
-Optional<unsigned> LoopVectorizationCostModel::computeMaxVF(bool OptForSize) {
+Optional<std::pair<unsigned,unsigned>>
+LoopVectorizationCostModel::computeMaxVF(bool OptForSize) {
   if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
     // TODO: It may by useful to do since it's still likely to be dynamically
     // uniform if the target can skip.
     DEBUG(dbgs() << "LV: Not inserting runtime ptr check for divergent target");
 
     ORE->emit(
       createMissedAnalysis("CantVersionLoopWithDivergentTarget")
       << "runtime pointer checks needed. Not enabled for divergent target");
LoopVectorizationCostModel::computeMaxVF(bool OptForSize) {
   if (TC < 2) {
     ORE->emit(
         createMissedAnalysis("UnknownLoopCountComplexCFG")
         << "unable to calculate the loop count due to complex control flow");
     DEBUG(dbgs() << "LV: Aborting. A tail loop is required with -Os/-Oz.\n");
     return None;
   }
 
-  unsigned MaxVF = computeFeasibleMaxVF(OptForSize, TC);
+  std::pair<unsigned,unsigned> Range = computeFeasibleMaxVF(OptForSize, TC);
 
-  if (TC % MaxVF != 0) {
+  if (TC % Range.first != 0 || TC % Range.second != 0) {
     // If the trip count that we found modulo the vectorization factor is not
     // zero then we require a tail.
     // FIXME: look for a smaller MaxVF that does divide TC rather than give up.
     // FIXME: return None if loop requiresScalarEpilog(<MaxVF>), or look for a
     //        smaller MaxVF that does not require a scalar epilog.
 
     ORE->emit(createMissedAnalysis("NoTailLoopWithOptForSize")
               << "cannot optimize for size and vectorize at the "
                  "same time. Enable vectorization of this loop "
                  "with '#pragma clang loop vectorize(enable)' "
                  "when compiling with -Os/-Oz");
     DEBUG(dbgs() << "LV: Aborting. A tail loop is required with -Os/-Oz.\n");
     return None;
   }
 
-  return MaxVF;
+  return Range;
 }
 
-unsigned
+std::pair<unsigned,unsigned>
 LoopVectorizationCostModel::computeFeasibleMaxVF(bool OptForSize,
                                                  unsigned ConstTripCount) {
   MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
-  unsigned SmallestType, WidestType;
-  std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
+  SmallVector<Type*,4> ElemTypes = getSmallestAndWidestTypes();
+  if (ElemTypes.empty())
+    return { 1, 1 };
+  const DataLayout &DL = TheFunction->getParent()->getDataLayout();
+  unsigned SmallestType = DL.getTypeSizeInBits(ElemTypes.front());
+  unsigned WidestType = std::max(8ul, DL.getTypeSizeInBits(ElemTypes.back()));
   unsigned WidestRegister = TTI.getRegisterBitWidth(true);
 
   // Get the maximum safe dependence distance in bits computed by LAA.
   // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
   // the memory accesses that is most restrictive (involved in the smallest
   // dependence distance).
   unsigned MaxSafeRegisterWidth = Legal->getMaxSafeRegisterWidth();
 
   WidestRegister = std::min(WidestRegister, MaxSafeRegisterWidth);
 
-  unsigned MaxVectorSize = WidestRegister / WidestType;
+  unsigned MinVF = 1;
+  for (Type *Ty : ElemTypes)
+    MinVF = std::max(MinVF, TTI.getMinimumVF(Ty));
+
+  unsigned MaxVectorSize = std::max(MinVF, WidestRegister / WidestType);
 
   DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType << " / "
                << WidestType << " bits.\n");
   DEBUG(dbgs() << "LV: The Widest register safe to use is: " << WidestRegister
                << " bits.\n");
 
   assert(MaxVectorSize <= 64 && "Did not expect to pack so many elements"
                                 " into one vector!");
   if (MaxVectorSize == 0) {
     DEBUG(dbgs() << "LV: The target has no vector registers.\n");
     MaxVectorSize = 1;
-    return MaxVectorSize;
+    return { MaxVectorSize, MaxVectorSize };
   } else if (ConstTripCount && ConstTripCount < MaxVectorSize &&
              isPowerOf2_32(ConstTripCount)) {
     // We need to clamp the VF to be the ConstTripCount. There is no point in
     // choosing a higher viable VF as done in the loop below.
     DEBUG(dbgs() << "LV: Clamping the MaxVF to the constant trip count: "
                  << ConstTripCount << "\n");
     MaxVectorSize = ConstTripCount;
-    return MaxVectorSize;
+    return { MaxVectorSize, MaxVectorSize };
   }
 
   unsigned MaxVF = MaxVectorSize;
   if (MaximizeBandwidth && !OptForSize) {
     // Collect all viable vectorization factors larger than the default MaxVF
     // (i.e. MaxVectorSize).
     SmallVector<unsigned, 8> VFs;
     unsigned NewMaxVectorSize = WidestRegister / SmallestType;
     for (unsigned VS = MaxVectorSize * 2; VS <= NewMaxVectorSize; VS *= 2)
       VFs.push_back(VS);
 
     // For each VF calculate its register usage.
     auto RUs = calculateRegisterUsage(VFs);
 
     // Select the largest VF which doesn't require more registers than existing
     // ones.
     unsigned TargetNumRegisters = TTI.getNumberOfRegisters(true);
     for (int i = RUs.size() - 1; i >= 0; --i) {
       if (RUs[i].MaxLocalUsers <= TargetNumRegisters) {
         MaxVF = VFs[i];
         break;
       }
     }
   }
-  return MaxVF;
+  return { MinVF, MaxVF };
 }
 
 VectorizationFactor
-LoopVectorizationCostModel::selectVectorizationFactor(unsigned MaxVF) {
+LoopVectorizationCostModel::selectVectorizationFactor(unsigned MinVF,
+                                                      unsigned MaxVF) {
   float Cost = expectedCost(1).first;
   const float ScalarCost = Cost;
   unsigned Width = 1;
   DEBUG(dbgs() << "LV: Scalar loop costs: " << (int)ScalarCost << ".\n");
 
   bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;
   // Ignore scalar width, because the user explicitly wants vectorization.
   if (ForceVectorization && MaxVF > 1) {
-    Width = 2;
+    Width = MinVF;
     Cost = expectedCost(Width).first / (float)Width;
   }
 
-  for (unsigned i = 2; i <= MaxVF; i *= 2) {
+  for (unsigned i = MinVF; i <= MaxVF; i *= 2) {
     // Notice that the vector loop needs to be executed less times, so
     // we need to divide the cost of the vector loops by the width of
     // the vector elements.
     VectorizationCostTy C = expectedCost(i);
     float VectorCost = C.first / (float)i;
     DEBUG(dbgs() << "LV: Vector loop of width " << i
                  << " costs: " << (int)VectorCost << ".\n");
     if (!C.second && !ForceVectorization) {
LoopVectorizationCostModel::selectVectorizationFactor(unsigned MinVF,
   DEBUG(if (ForceVectorization && Width > 1 && Cost >= ScalarCost) dbgs()
         << "LV: Vectorization seems to be not beneficial, "
         << "but was forced by a user.\n");
   DEBUG(dbgs() << "LV: Selecting VF: " << Width << ".\n");
   VectorizationFactor Factor = {Width, (unsigned)(Width * Cost)};
   return Factor;
 }
 
-std::pair<unsigned, unsigned>
+SmallVector<Type*,4>
 LoopVectorizationCostModel::getSmallestAndWidestTypes() {
-  unsigned MinWidth = -1U;
-  unsigned MaxWidth = 8;
+  SmallVector<Type*,4> SortedTypes;
+  SetVector<Type*> ElemTypes;
   const DataLayout &DL = TheFunction->getParent()->getDataLayout();
 
   // For each block.
   for (BasicBlock *BB : TheLoop->blocks()) {
     // For each instruction in the loop.
     for (Instruction &I : *BB) {
       Type *T = I.getType();
 
for (Instruction &I : *BB) {
       //        been selected. Here, we assume that if an access can be
       //        vectorized, it will be. We should also look at extending this
       //        optimization to non-pointer types.
       //
       if (T->isPointerTy() && !isConsecutiveLoadOrStore(&I) &&
           !Legal->isAccessInterleaved(&I) && !isLegalGatherOrScatter(&I))
         continue;
 
-      MinWidth = std::min(MinWidth,
-                          (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
-      MaxWidth = std::max(MaxWidth,
-                          (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
+      ElemTypes.insert(T->getScalarType());
     }
   }
 
-  return {MinWidth, MaxWidth};
+  SortedTypes.assign(ElemTypes.begin(), ElemTypes.end());
+  llvm::sort(SortedTypes.begin(), SortedTypes.end(),
+             [&DL] (Type *T0, Type *T1) {
+               return DL.getTypeSizeInBits(T0) < DL.getTypeSizeInBits(T1);
+             });
+  return SortedTypes;
 }
 
 unsigned LoopVectorizationCostModel::selectInterleaveCount(bool OptForSize,
                                                            unsigned VF,
                                                            unsigned LoopCost) {
   // -- The interleave heuristics --
   // We interleave the loop in order to expose ILP and reduce the loop overhead.
   // There are many micro-architectural considerations that we can't predict
for (auto &Induction : *Legal->getInductionVars()) {
     VecValuesToIgnore.insert(Casts.begin(), Casts.end());
   }
 }
 
 VectorizationFactor
 LoopVectorizationPlanner::plan(bool OptForSize, unsigned UserVF) {
   // Width 1 means no vectorize, cost 0 means uncomputed cost.
   const VectorizationFactor NoVectorization = {1U, 0U};
-  Optional<unsigned> MaybeMaxVF = CM.computeMaxVF(OptForSize);
+  Optional<std::pair<unsigned,unsigned>> MaybeMaxVF =
+     CM.computeMaxVF(OptForSize);
   if (!MaybeMaxVF.hasValue()) // Cases considered too costly to vectorize.
     return NoVectorization;
 
   if (UserVF) {
     DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
     assert(isPowerOf2_32(UserVF) && "VF needs to be a power of two");
     // Collect the instructions (and their associated costs) that will be more
     // profitable to scalarize.
     CM.selectUserVectorizationFactor(UserVF);
     buildVPlans(UserVF, UserVF);
     DEBUG(printPlans(dbgs()));
     return {UserVF, 0};
   }
 
-  unsigned MaxVF = MaybeMaxVF.getValue();
-  assert(MaxVF != 0 && "MaxVF is zero.");
+  unsigned MinVF, MaxVF;
+  std::tie(MinVF, MaxVF) = MaybeMaxVF.getValue();
+  assert(MinVF != 0 && MaxVF != 0 && "MinVF or MaxVF is zero.");
+
+  // Collect Uniform and Scalar instructions.
+  if (MinVF > 1)
+    CM.collectUniformsAndScalars(1);
 
-  for (unsigned VF = 1; VF <= MaxVF; VF *= 2) {
+  for (unsigned VF = MinVF; VF <= MaxVF; VF *= 2) {
     // Collect Uniform and Scalar instructions after vectorization with VF.
     CM.collectUniformsAndScalars(VF);
 
     // Collect the instructions (and their associated costs) that will be more
     // profitable to scalarize.
-    if (VF > 1)
-      CM.collectInstsToScalarize(VF);
   }
 
-  buildVPlans(1, MaxVF);
+  if (MinVF > 1)
+    buildVPlans(1, 1);
+  buildVPlans(MinVF, MaxVF);
   DEBUG(printPlans(dbgs()));
   if (MaxVF == 1)
     return NoVectorization;
 
   // Select the optimal vectorization factor.
-  return CM.selectVectorizationFactor(MaxVF);
+  return CM.selectVectorizationFactor(MinVF, MaxVF);
 }
 
 void LoopVectorizationPlanner::setBestPlan(unsigned VF, unsigned UF) {
   DEBUG(dbgs() << "Setting best plan to VF=" << VF << ", UF=" << UF << '\n');
   BestVF = VF;
   BestUF = UF;
 
   erase_if(VPlans, [VF](const VPlanPtr &Plan) {

test/Transforms/LoopUnroll/runtime-loop4.ll

loop2.header:
   br i1 %e, label %loop2, label %exit2
 
 loop2:
   %iv2 = phi i32 [ 0, %loop2.header ], [ %inc2, %loop2 ]
   %offset2 = getelementptr i32, i32* %addr2, i32 %iv2
   store i32 %iv2, i32* %offset2, align 4
   %inc2 = add i32 %iv2, 1
   %exitcnd2 = icmp uge i32 %inc2, %iter
-  br i1 %exitcnd2, label %exit2, label %loop2
+  br i1 %exitcnd2, label %exit2, label %loop2, !llvm.loop !0
 
 exit2:
   br label %loop1.latch
 
 loop1.latch:
   %inc1 = add i32 %iv1, 1
   %exitcnd1 = icmp uge i32 %inc1, 1024
-  br i1 %exitcnd1, label %exit, label %loop1
+  br i1 %exitcnd1, label %exit, label %loop1, !llvm.loop !0
 
 exit:
   ret void
 }
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 0}

test/Transforms/LoopVectorize/X86/funclet.ll

 for.cond.cleanup:                                 ; preds = %for.body
   catchret from %1 to label %try.cont
 
 for.body:                                         ; preds = %for.body, %catch
   %i.07 = phi i32 [ 0, %catch ], [ %inc, %for.body ]
   %call = call double @floor(double 1.0) #1 [ "funclet"(token %1) ]
   %inc = add nuw nsw i32 %i.07, 1
   %exitcond = icmp eq i32 %inc, 1024
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !0
 
 try.cont:                                         ; preds = %for.cond.cleanup
   ret void
 
 unreachable:                                      ; preds = %entry
   unreachable
 }
 
 ; CHECK-LABEL: define void @test1(
 ; CHECK: %[[cpad:.*]] = catchpad within {{.*}} [i8* null, i32 64, i8* null]
 ; CHECK: call <16 x double> @llvm.floor.v16f64(<16 x double> {{.*}}) [ "funclet"(token %[[cpad]]) ]
 
 declare x86_stdcallcc void @_CxxThrowException(i8*, i8*)
 
 declare i32 @__CxxFrameHandler3(...)
 
 declare double @floor(double) #1
 
 attributes #0 = { "target-features"="+sse2" }
 attributes #1 = { nounwind readnone }
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 16}

test/Transforms/LoopVectorize/pr32859.ll

 for.cond1.preheader.i:                            ; preds = %if.end.2.i, %entry
   %c.06.i = phi i32 [ 0, %entry ], [ %inc5.i, %if.end.2.i ]
   %tobool.i = icmp ne i32 undef, 0
   br label %if.end.2.i
 
 if.end.2.i:                                       ; preds = %for.cond1.preheader.i
   %inc5.i = add nsw i32 %c.06.i, 1
   %cmp.i = icmp slt i32 %inc5.i, 16
-  br i1 %cmp.i, label %for.cond1.preheader.i, label %for.cond.preheader
+  br i1 %cmp.i, label %for.cond1.preheader.i, label %for.cond.preheader, !llvm.loop !0
 
 for.cond.preheader:                               ; preds = %if.end.2.i
   %e.0.ph = phi i32 [ 0, %if.end.2.i ]
   unreachable
 }
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 4}

test/Transforms/LoopVectorize/unroll.ll

 
 for.body:                                         ; preds = %for.body.lr.ph, %for.body
   %i.06 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
   %mul = mul nuw nsw i64 %i.06, 7
   %arrayidx = getelementptr inbounds [1000 x i32], [1000 x i32]* @a, i64 0, i64 %mul
   store i32 3, i32* %arrayidx, align 4
   %inc = add nuw nsw i64 %i.06, 1
   %cmp = icmp slt i64 %inc, %conv
-  br i1 %cmp, label %for.body, label %for.end.loopexit
+  br i1 %cmp, label %for.body, label %for.end.loopexit, !llvm.loop !0
 
 for.end.loopexit:                                 ; preds = %for.body
   br label %for.end
 
 for.end:                                          ; preds = %for.end.loopexit, %entry
   ret void
 }
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 1}