[LV][Metadata] Add loop.interleave.enable for loop vectorizer

[eopXD]

Summary

Adding this metadata allows {loop.vectorize.enable, false} to be used
without disabling the whole pass.

[eopXD]

Please feel free to add appropriate reviewer to this patch.

[mcberg2021]

clang/lib/CodeGen/CGLoopInfo.cpp

673

Can you update the comments on lines: 665 and 671 as they both need update.

[eopXD]

Update comment.

[eopXD]

After this patch, we can further remove setVectorizeWidth(1) and setInterleaveCount(1) when vectorize(disable) and interleave(disable) is specified. For now it is not removed to keep the patch limited and simple enough.

[eopXD]

Fix grammar error in comment.

[eopXD]

Gentle ping, thank you ;)

[fhahn]

Adding this metadata allows {loop.vectorize.enable, false} to be used without disabling the whole pass.

Could you please describe the behavior in more detail here? The new metadata should also be documented in LangRef, the new pragma in https://clang.llvm.org/docs/LanguageExtensions.htm.

Will this change the existing behavior if only llvm.loop.vectorize.enable == false? To preserve backwards compatibility, I think the behavior shouldn;t change unless llvm.loop.interleave.enable is provided.

Also, shouldn't #pragma clang loop vectorize(disable) interleave(enable) be equivalent to #pragma clang loop vectorize(enable) vectorize_width(1)?

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

2142

Will getInterleaveForce() return ENABLED if an interleave count > 1 is set through metadata?

3013–3014

Should this check interleaving or vectorization forced?

[eopXD]

@fhahn Thank you very much for the review, backward compatibility is a good point. I will update the revision.

[eopXD]

Hi @fhahn,

After a second thought, I think posting a clear RFC [0] to the community that defines the
problem and goal this patch tries to achieve so the community is notified of such change
is a better approach than directly updating this patch. Please consider to drop by and
give a review.

Thank you very much for your time.

[0] [[RFC] Enabling LoopVectorizer for vectorization width of 1](https://discourse.llvm.org/t/rfc-enabling-loopvectorizer-for-vectorization-width-of-1/65769)

Regards,

eop Chen

clang/lib/CodeGen/CGLoopInfo.h

struct LoopAttributes {
   LVEnableState VectorizePredicateEnable;
 
   /// Value for llvm.loop.vectorize.width metadata.
   unsigned VectorizeWidth;
 
   // Value for llvm.loop.vectorize.scalable.enable
   LVEnableState VectorizeScalable;
 
+  /// Value for llvm.loop.interleave.enable metadata.
+  LVEnableState InterleaveEnable;
+
   /// Value for llvm.loop.interleave.count metadata.
   unsigned InterleaveCount;
 
   /// llvm.unroll.
   unsigned UnrollCount;
 
   /// llvm.unroll.
   unsigned UnrollAndJamCount;
public:
 
   /// Set the vectorize width for the next loop pushed.
   void setVectorizeWidth(unsigned W) { StagedAttrs.VectorizeWidth = W; }
 
   void setVectorizeScalable(const LoopAttributes::LVEnableState &State) {
     StagedAttrs.VectorizeScalable = State;
   }
 
+  void setInterleaveEnable(bool Enable = true) {
+    StagedAttrs.InterleaveEnable =
+        Enable ? LoopAttributes::Enable : LoopAttributes::Disable;
+  }
+
   /// Set the interleave count for the next loop pushed.
   void setInterleaveCount(unsigned C) { StagedAttrs.InterleaveCount = C; }
 
   /// Set the unroll count for the next loop pushed.
   void setUnrollCount(unsigned C) { StagedAttrs.UnrollCount = C; }
 
   /// \brief Set the unroll count for the next loop pushed.
   void setUnrollAndJamCount(unsigned C) { StagedAttrs.UnrollAndJamCount = C; }

clang/lib/CodeGen/CGLoopInfo.cpp

 
 MDNode *
 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
                                       ArrayRef<Metadata *> LoopProperties,
                                       bool &HasUserTransforms) {
   LLVMContext &Ctx = Header->getContext();
 
   Optional<bool> Enabled;
-  if (Attrs.VectorizeEnable == LoopAttributes::Disable)
+  if (Attrs.VectorizeEnable == LoopAttributes::Disable &&
+      Attrs.InterleaveEnable == LoopAttributes::Disable)
     Enabled = false;
   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
+           Attrs.InterleaveEnable != LoopAttributes::Unspecified ||
            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
            Attrs.VectorizeScalable != LoopAttributes::Unspecified)
     Enabled = true;
 
   if (Enabled != true) {
     SmallVector<Metadata *, 4> NewLoopProperties;
     if (Enabled == false) {
       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
       NewLoopProperties.push_back(
+          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.interleave.enable"),
+                            ConstantAsMetadata::get(ConstantInt::get(
+                                llvm::Type::getInt1Ty(Ctx), 0))}));
+      NewLoopProperties.push_back(
           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
                             ConstantAsMetadata::get(ConstantInt::get(
                                 llvm::Type::getInt1Ty(Ctx), 0))}));
       LoopProperties = NewLoopProperties;
     }
     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
   }
 
LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
   if (Attrs.InterleaveCount > 0) {
     Metadata *Vals[] = {
         MDString::get(Ctx, "llvm.loop.interleave.count"),
         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
                                                  Attrs.InterleaveCount))};
     Args.push_back(MDNode::get(Ctx, Vals));
   }
 
+  // Setting interleave.enable
+  if (Attrs.InterleaveEnable != LoopAttributes::Unspecified ||
+      Attrs.InterleaveCount > 0) {
+    bool IsInterleaveEnabled =
+        (Attrs.InterleaveEnable == LoopAttributes::Enable) ||
+        Attrs.InterleaveCount > 1;
+    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.enable"),
+                        ConstantAsMetadata::get(ConstantInt::get(
+                            llvm::Type::getInt1Ty(Ctx), IsInterleaveEnabled))};
+    Args.push_back(MDNode::get(Ctx, Vals));
+  }
+
   // vectorize.enable is set if:
   // 1) loop hint vectorize.enable is set, or
   // 2) it is implied when vectorize.predicate is set, or
   // 3) it is implied when vectorize.width is set to a value > 1
   // 4) it is implied when vectorize.scalable.enable is true
   // 5) it is implied when vectorize.width is unset (0) and the user
   //    explicitly requested fixed-width vectorization, i.e.
   //    vectorize.scalable.enable is false.
MDNode *LoopInfo::createMetadata(
   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
 }
 
 LoopAttributes::LoopAttributes(bool IsParallel)
     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
       UnrollEnable(LoopAttributes::Unspecified),
       UnrollAndJamEnable(LoopAttributes::Unspecified),
       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
-      VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
+      VectorizeScalable(LoopAttributes::Unspecified),
+      InterleaveEnable(LoopAttributes::Unspecified), InterleaveCount(0),
       UnrollCount(0), UnrollAndJamCount(0),
       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
       PipelineInitiationInterval(0), MustProgress(false) {}
 
 void LoopAttributes::clear() {
   IsParallel = false;
   VectorizeWidth = 0;
   VectorizeScalable = LoopAttributes::Unspecified;
   InterleaveCount = 0;
   UnrollCount = 0;
   UnrollAndJamCount = 0;
   VectorizeEnable = LoopAttributes::Unspecified;
+  InterleaveEnable = LoopAttributes::Unspecified;
   UnrollEnable = LoopAttributes::Unspecified;
   UnrollAndJamEnable = LoopAttributes::Unspecified;
   VectorizePredicateEnable = LoopAttributes::Unspecified;
   DistributeEnable = LoopAttributes::Unspecified;
   PipelineDisabled = false;
   PipelineInitiationInterval = 0;
   MustProgress = false;
 }
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
 
   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
       Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
       Attrs.PipelineInitiationInterval == 0 &&
       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
+      Attrs.InterleaveEnable == LoopAttributes::Unspecified &&
       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
       !EndLoc && !Attrs.MustProgress)
     return;
 
   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
 }
if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
     LoopAttributes BeforeJam, AfterJam;
 
     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
 
     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
     BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
+    BeforeJam.InterleaveEnable = Attrs.InterleaveEnable;
     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
 
     switch (Attrs.UnrollEnable) {
     case LoopAttributes::Unspecified:
     case LoopAttributes::Disable:
       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
       AfterJam.UnrollEnable = Attrs.UnrollEnable;
if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
     // consider the first inner loop.
     if (!Parent->UnrollAndJamInnerFollowup) {
       // Splitting the attributes into a BeforeJam and an AfterJam part will
       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
       // to be forwarded to the AfterJam part. We detect the situation here and
       // add it manually.
       SmallVector<Metadata *, 1> BeforeLoopProperties;
       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
+          BeforeJam.InterleaveEnable != LoopAttributes::Unspecified ||
           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
           BeforeJam.VectorizeScalable == LoopAttributes::Enable)
         BeforeLoopProperties.push_back(
             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
 
       bool InnerFollowupHasTransform = false;
       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
if (OpenCLHint) {
       Option = LH->getOption();
       State = LH->getState();
     }
     switch (State) {
     case LoopHintAttr::Disable:
       switch (Option) {
       case LoopHintAttr::Vectorize:
         // Disable vectorization by specifying a width of 1.
+        setVectorizeEnable(false);
         setVectorizeWidth(1);
         setVectorizeScalable(LoopAttributes::Unspecified);
         break;
       case LoopHintAttr::Interleave:
         // Disable interleaving by speciyfing a count of 1.
+        setInterleaveEnable(false);
         setInterleaveCount(1);
         break;
       case LoopHintAttr::Unroll:
         setUnrollState(LoopAttributes::Disable);
         break;
       case LoopHintAttr::UnrollAndJam:
         setUnrollAndJamState(LoopAttributes::Disable);
         break;
       case LoopHintAttr::VectorizePredicate:
case LoopHintAttr::Disable:
       case LoopHintAttr::PipelineInitiationInterval:
         llvm_unreachable("Options cannot be disabled.");
         break;
       }
       break;
     case LoopHintAttr::Enable:
       switch (Option) {
       case LoopHintAttr::Vectorize:
-      case LoopHintAttr::Interleave:
         setVectorizeEnable(true);
         break;
+      case LoopHintAttr::Interleave:
+        setInterleaveEnable(true);
+        break;
       case LoopHintAttr::Unroll:
         setUnrollState(LoopAttributes::Enable);
         break;
       case LoopHintAttr::UnrollAndJam:
         setUnrollAndJamState(LoopAttributes::Enable);
         break;
       case LoopHintAttr::VectorizePredicate:
         setVectorizePredicateState(LoopAttributes::Enable);
case LoopHintAttr::Enable:
       case LoopHintAttr::PipelineInitiationInterval:
         llvm_unreachable("Options cannot enabled.");
         break;
       }
       break;
     case LoopHintAttr::AssumeSafety:
       switch (Option) {
       case LoopHintAttr::Vectorize:
-      case LoopHintAttr::Interleave:
         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
         setParallel(true);
         setVectorizeEnable(true);
         break;
+      case LoopHintAttr::Interleave:
+        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
+        setParallel(true);
+        setInterleaveEnable(true);
+        break;
       case LoopHintAttr::Unroll:
       case LoopHintAttr::UnrollAndJam:
       case LoopHintAttr::VectorizePredicate:
       case LoopHintAttr::UnrollCount:
       case LoopHintAttr::UnrollAndJamCount:
       case LoopHintAttr::VectorizeWidth:
       case LoopHintAttr::InterleaveCount:
       case LoopHintAttr::Distribute:

clang/test/CodeGenCXX/pragma-loop-predicate.cpp

 // CHECK-NEXT: ![[LOOP1]] = distinct !{![[LOOP1]], [[MP]], [[GEN6:![0-9]+]], [[GEN3]]}
 // CHECK-NEXT: [[GEN6]] = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
 
 // CHECK-NEXT: ![[LOOP2]] = distinct !{![[LOOP2]], [[MP]], [[GEN8:![0-9]+]], [[GEN3]]}
 // CHECK-NEXT: [[GEN8]] = !{!"llvm.loop.vectorize.predicate.enable", i1 false}
 
 // CHECK-NEXT: ![[LOOP3]] = distinct !{![[LOOP3]], [[MP]], [[GEN6]], [[GEN3]]}
 
-// CHECK-NEXT: ![[LOOP4]] = distinct !{![[LOOP4]], [[MP]], [[GEN10:![0-9]+]]}
+// CHECK-NEXT: ![[LOOP4]] = distinct !{![[LOOP4]], [[MP]], [[GEN10:![0-9]+]], [[VECTORIZE_DISABLE:.*]]}
 // CHECK-NEXT: [[GEN10]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK-NEXT: [[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false}
 
-// CHECK-NEXT: ![[LOOP5]] = distinct !{![[LOOP5]], [[MP]], [[GEN6]], [[GEN10]]}
+// CHECK-NEXT: ![[LOOP5]] = distinct !{![[LOOP5]], [[MP]], [[GEN6]], [[GEN10]], [[VECTORIZE_DISABLE]]}
 
 // CHECK-NEXT: ![[LOOP6]] = distinct !{![[LOOP6]], [[MP]], [[GEN8]], [[GEN10]], [[GEN11:![0-9]+]]}
 // CHECK-NEXT: [[GEN11]] = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
 
 // CHECK-NEXT: ![[LOOP7]] = distinct !{![[LOOP7]], [[MP]], [[GEN8]], [[GEN12:![0-9]+]], [[GEN11]], [[GEN3]]}
 // CHECK-NEXT: [[GEN12]] = !{!"llvm.loop.vectorize.width", i32 4}
 
 // CHECK-NEXT: ![[LOOP8]] = distinct !{![[LOOP8]], [[MP]], [[GEN6]], [[GEN10]], [[GEN11]]}
 
 // CHECK-NEXT: ![[LOOP9]] = distinct !{![[LOOP9]], [[MP]], [[GEN6]], [[GEN12]], [[GEN11]], [[GEN3]]}

clang/test/CodeGenCXX/pragma-loop-safety.cpp

for (int i = 0; i < Length; i++) {
     // CHECK-NEXT: br label [[LOOP2_INC:[^,]+]]
     List[i] = i * 2;
 
     // CHECK: br label [[LOOP2_COND:[^,]+]], !llvm.loop ![[LOOP2_HINTS:[0-9]+]]
   }
 }
 
 // CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
-// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], [[MP:![0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]]}
+// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], [[MP:![0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTERLEAVE_DISABLE:.*]], ![[VECTORIZE_ENABLE:[0-9]+]]}
 // CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
 // CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
 // CHECK: ![[INTERLEAVE_1]] = !{!"llvm.loop.interleave.count", i32 1}
-// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.enable", i1 false}
+// CHECK: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
 // CHECK: ![[ACCESS_GROUP_8]] = distinct !{}
-// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], [[MP]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]]}
+// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], [[MP]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_DISABLE:[0-9]+]]}
 // CHECK: ![[PARALLEL_ACCESSES_11]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_8]]}
 // CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.interleave.enable", i1 true}
+// CHECK: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false}

clang/test/CodeGenCXX/pragma-loop.cpp

#pragma clang loop vectorize_width(2) interleave_count(2)
   for (int i : List) {
     // CHECK: br label {{.*}}, !llvm.loop ![[LOOP_4:.*]]
     List[i] = i;
   }
 }
 
 // Verify disable pragma clang loop directive generates correct metadata
 void disable_test(int *List, int Length) {
-#pragma clang loop vectorize(disable) unroll(disable) distribute(disable)
+#pragma clang loop vectorize(disable) interleave(disable) unroll(disable) distribute(disable)
   for (int i = 0; i < Length; i++) {
     // CHECK: br label {{.*}}, !llvm.loop ![[LOOP_5:.*]]
     List[i] = i * 2;
   }
 }
 
 #define VECWIDTH 2
 #define INTCOUNT 2
for (int i = 0; i < Length; i++) {
     // CHECK: br label {{.*}}, !llvm.loop ![[LOOP_19:.*]]
     List[i] = i * 2;
   }
 }
 
 // CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], [[MP:![0-9]+]], ![[UNROLL_FULL:.*]]}
 // CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"}
 
-// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], [[MP]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[FIXED_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], [[MP]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[FIXED_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_ENABLE:.*]]}
 // CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
 // CHECK: ![[DISTRIBUTE_DISABLE]] = !{!"llvm.loop.distribute.enable", i1 false}
 // CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
 // CHECK: ![[FIXED_VEC]] = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
 // CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
+// CHECK: ![[INTERLEAVE_ENABLE]] = !{!"llvm.loop.interleave.enable", i1 true}
 // CHECK: ![[VECTORIZE_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
 
-// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], [[MP]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_3:.*]]}
+// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], [[MP]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[FOLLOWUP_VECTOR_3:.*]]}
 // CHECK: ![[FOLLOWUP_VECTOR_3]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_3:.*]]}
 // CHECK: ![[AFTER_VECTOR_3]] = distinct !{![[AFTER_VECTOR_3]], [[MP]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
 // CHECK: ![[ISVECTORIZED]] = !{!"llvm.loop.isvectorized"}
 // CHECK: ![[UNROLL_8]] = !{!"llvm.loop.unroll.count", i32 8}
 
-// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[VECTORIZE_ENABLE]]}
+// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[INTERLEAVE_ENABLE:.*]], ![[VECTORIZE_ENABLE]]}
 // CHECK: ![[WIDTH_2]] = !{!"llvm.loop.vectorize.width", i32 2}
 // CHECK: ![[INTERLEAVE_2]] = !{!"llvm.loop.interleave.count", i32 2}
 
-// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]]}
-// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[INTERLEAVE_DISABLE:.*]], ![[VECTORIZE_DISABLE:.*]]}
+// CHECK: ![[INTERLEAVE_DISABLE]] = !{!"llvm.loop.interleave.enable", i1 false}
+// CHECK: ![[VECTORIZE_DISABLE]] = !{!"llvm.loop.vectorize.enable", i1 false}
 
 // CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], [[MP]], ![[WIDTH_2:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_6:.*]]}
 // CHECK: ![[FOLLOWUP_VECTOR_6]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_6:.*]]}
 // CHECK: ![[AFTER_VECTOR_6]] = distinct !{![[AFTER_VECTOR_6]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
 
 // CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], [[MP]], ![[WIDTH_5:.*]], ![[FIXED_VEC]], ![[VECTORIZE_ENABLE]]}
 // CHECK: ![[WIDTH_5]] = !{!"llvm.loop.vectorize.width", i32 5}
 

 // CHECK: ![[FOLLOWUP_VECTOR_11]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_11:.*]]}
 // CHECK: ![[AFTER_VECTOR_11]] = distinct !{![[AFTER_VECTOR_11]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
 
 // CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_10:.*]], ![[FOLLOWUP_VECTOR_12:.*]]}
 // CHECK: ![[FOLLOWUP_VECTOR_12]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_12:.*]]}
 // CHECK: ![[AFTER_VECTOR_12]] = distinct !{![[AFTER_VECTOR_12]], ![[ISVECTORIZED:.*]], ![[UNROLL_24:.*]]}
 // CHECK: ![[UNROLL_24]] = !{!"llvm.loop.unroll.count", i32 24}
 
-// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_13:.*]]}
+// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE]], ![[FOLLOWUP_VECTOR_13:.*]]}
 // CHECK: ![[INTERLEAVE_16]] = !{!"llvm.loop.interleave.count", i32 16}
 // CHECK: ![[FOLLOWUP_VECTOR_13]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_13:.*]]}
 // CHECK: ![[AFTER_VECTOR_13]] = distinct !{![[AFTER_VECTOR_13]], ![[ISVECTORIZED:.*]], ![[UNROLL_32:.*]]}
 // CHECK: ![[UNROLL_32]] = !{!"llvm.loop.unroll.count", i32 32}
 
 // CHECK: ![[LOOP_14]] = distinct !{![[LOOP_14]], [[MP]], ![[WIDTH_10:.*]], ![[FIXED_VEC]], ![[VECTORIZE_ENABLE]]}
 // CHECK: ![[WIDTH_10]] = !{!"llvm.loop.vectorize.width", i32 10}
 
-// CHECK: ![[LOOP_15]] = distinct !{![[LOOP_15]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_15]] = distinct !{![[LOOP_15]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
 // CHECK: ![[WIDTH_16]] = !{!"llvm.loop.vectorize.width", i32 16}
 
-// CHECK: ![[LOOP_16]] = distinct !{![[LOOP_16]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16]], ![[SCALABLE_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_16]] = distinct !{![[LOOP_16]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_16]], ![[SCALABLE_VEC:.*]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
 // CHECK: ![[SCALABLE_VEC]] = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
 
-// CHECK: ![[LOOP_17]] = distinct !{![[LOOP_17]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
-// CHECK: ![[LOOP_18]] = distinct !{![[LOOP_18]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
-// CHECK: ![[LOOP_19]] = distinct !{![[LOOP_19]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_17]] = distinct !{![[LOOP_17]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[FIXED_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_18]] = distinct !{![[LOOP_18]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[LOOP_19]] = distinct !{![[LOOP_19]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]], ![[SCALABLE_VEC]], ![[INTERLEAVE_4:.*]], ![[INTERLEAVE_ENABLE]], ![[VECTORIZE_ENABLE:.*]]}
+// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}

llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h

class LoopVectorizeHints {
 
   /// Vectorization width.
   Hint Width;
 
   /// Vectorization interleave factor.
   Hint Interleave;
 
   /// Vectorization forced
-  Hint Force;
+  Hint VectorizationForce;
+
+  /// Interleave forced
+  Hint InterleaveForce;
 
   /// Already Vectorized
   Hint IsVectorized;
 
   /// Vector Predicate
   Hint Predicate;
 
   /// Says whether we should use fixed width or scalable vectorization.
unsigned getInterleave() const {
     // If interleaving is not explicitly set, assume that if we do not want
     // unrolling, we also don't want any interleaving.
     if (llvm::hasUnrollTransformation(TheLoop) & TM_Disable)
       return 1;
     return 0;
   }
   unsigned getIsVectorized() const { return IsVectorized.Value; }
   unsigned getPredicate() const { return Predicate.Value; }
-  enum ForceKind getForce() const {
-    if ((ForceKind)Force.Value == FK_Undefined &&
+  enum ForceKind getVectorizationForce() const {
+    if ((ForceKind)VectorizationForce.Value == FK_Undefined &&
+        hasDisableAllTransformsHint(TheLoop))
+      return FK_Disabled;
+    return (ForceKind)VectorizationForce.Value;
+  }
+  enum ForceKind getInterleaveForce() const {
+    if ((ForceKind)InterleaveForce.Value == FK_Undefined &&
         hasDisableAllTransformsHint(TheLoop))
       return FK_Disabled;
-    return (ForceKind)Force.Value;
+    return (ForceKind)InterleaveForce.Value;
   }
 
   /// \return true if scalable vectorization has been explicitly disabled.
   bool isScalableVectorizationDisabled() const {
     return (ScalableForceKind)Scalable.Value == SK_FixedWidthOnly;
   }
 
   /// If hints are provided that force vectorization, use the AlwaysPrint
   /// pass name to force the frontend to print the diagnostic.
   const char *vectorizeAnalysisPassName() const;
 
   /// When enabling loop hints are provided we allow the vectorizer to change
   /// the order of operations that is given by the scalar loop. This is not
   /// enabled by default because can be unsafe or inefficient. For example,
   /// reordering floating-point operations will change the way round-off
   /// error accumulates in the loop.
   bool allowReordering() const;
 
   bool isPotentiallyUnsafe() const {
     // Avoid FP vectorization if the target is unsure about proper support.
     // This may be related to the SIMD unit in the target not handling
     // IEEE 754 FP ops properly, or bad single-to-double promotions.
     // Otherwise, a sequence of vectorized loops, even without reduction,
     // could lead to different end results on the destination vectors.
-    return getForce() != LoopVectorizeHints::FK_Enabled && PotentiallyUnsafe;
+    return getVectorizationForce() != LoopVectorizeHints::FK_Enabled &&
+           PotentiallyUnsafe;
   }
 
   void setPotentiallyUnsafe() { PotentiallyUnsafe = true; }
 
 private:
   /// Find hints specified in the loop metadata and update local values.
   void getHintsFromMetadata();
 

llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

 }
 
 LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
                                        bool InterleaveOnlyWhenForced,
                                        OptimizationRemarkEmitter &ORE,
                                        const TargetTransformInfo *TTI)
     : Width("vectorize.width", VectorizerParams::VectorizationFactor, HK_WIDTH),
       Interleave("interleave.count", InterleaveOnlyWhenForced, HK_INTERLEAVE),
-      Force("vectorize.enable", FK_Undefined, HK_FORCE),
+      VectorizationForce("vectorize.enable", FK_Undefined, HK_FORCE),
+      InterleaveForce("interleave.enable", FK_Undefined, HK_FORCE),
       IsVectorized("isvectorized", 0, HK_ISVECTORIZED),
       Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE),
       Scalable("vectorize.scalable.enable", SK_Unspecified, HK_SCALABLE),
       TheLoop(L), ORE(ORE) {
   // Populate values with existing loop metadata.
   getHintsFromMetadata();
 
   // force-vector-interleave overrides DisableInterleaving.
void LoopVectorizeHints::setAlreadyVectorized() {
   TheLoop->setLoopID(NewLoopID);
 
   // Update internal cache.
   IsVectorized.Value = 1;
 }
 
 bool LoopVectorizeHints::allowVectorization(
     Function *F, Loop *L, bool VectorizeOnlyWhenForced) const {
-  if (getForce() == LoopVectorizeHints::FK_Disabled) {
-    LLVM_DEBUG(dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n");
+  if (getVectorizationForce() == LoopVectorizeHints::FK_Disabled &&
+      getInterleaveForce() == LoopVectorizeHints::FK_Disabled) {
+    LLVM_DEBUG(
+        dbgs()
+        << "LV: Not vectorizing: #pragma vectorize and interleave disable.\n");
     emitRemarkWithHints();
     return false;
   }
 
-  if (VectorizeOnlyWhenForced && getForce() != LoopVectorizeHints::FK_Enabled) {
-    LLVM_DEBUG(dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n");
+  if (VectorizeOnlyWhenForced &&
+      getVectorizationForce() != LoopVectorizeHints::FK_Enabled &&
+      getInterleaveForce() != LoopVectorizeHints::FK_Enabled) {
+    LLVM_DEBUG(
+        dbgs()
+        << "LV: Not vectorizing: No #pragma vectorize or interleave enable.\n");
     emitRemarkWithHints();
     return false;
   }
 
   if (getIsVectorized() == 1) {
     LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n");
-    // FIXME: Add interleave.disable metadata. This will allow
-    // vectorize.disable to be used without disabling the pass and errors
-    // to differentiate between disabled vectorization and a width of 1.
     ORE.emit([&]() {
       return OptimizationRemarkAnalysis(vectorizeAnalysisPassName(),
                                         "AllDisabled", L->getStartLoc(),
                                         L->getHeader())
              << "loop not vectorized: vectorization and interleaving are "
                 "explicitly disabled, or the loop has already been "
                 "vectorized";
     });
     return false;
   }
 
   return true;
 }
 
 void LoopVectorizeHints::emitRemarkWithHints() const {
   using namespace ore;
 
   ORE.emit([&]() {
-    if (Force.Value == LoopVectorizeHints::FK_Disabled)
+    if (VectorizationForce.Value == LoopVectorizeHints::FK_Disabled &&
+        InterleaveForce.Value == LoopVectorizeHints::FK_Disabled)
       return OptimizationRemarkMissed(LV_NAME, "MissedExplicitlyDisabled",
                                       TheLoop->getStartLoc(),
                                       TheLoop->getHeader())
-             << "loop not vectorized: vectorization is explicitly disabled";
+             << "loop not vectorized: vectorization and interleave is "
+                "explicitly disabled";
     else {
       OptimizationRemarkMissed R(LV_NAME, "MissedDetails",
                                  TheLoop->getStartLoc(), TheLoop->getHeader());
       R << "loop not vectorized";
-      if (Force.Value == LoopVectorizeHints::FK_Enabled) {
-        R << " (Force=" << NV("Force", true);
+      if (VectorizationForce.Value == LoopVectorizeHints::FK_Enabled) {
+        R << " (VectorizationForce=" << NV("VectorizationForce", true);
         if (Width.Value != 0)
           R << ", Vector Width=" << NV("VectorWidth", getWidth());
+        R << ")";
+      }
+      if (InterleaveForce.Value == LoopVectorizeHints::FK_Enabled) {
+        R << " (InterleaveForce=" << NV("InterleaveForce", true);
         if (getInterleave() != 0)
           R << ", Interleave Count=" << NV("InterleaveCount", getInterleave());
         R << ")";
       }
       return R;
     }
   });
 }
 
 const char *LoopVectorizeHints::vectorizeAnalysisPassName() const {
   if (getWidth() == ElementCount::getFixed(1))
     return LV_NAME;
-  if (getForce() == LoopVectorizeHints::FK_Disabled)
+  if (getVectorizationForce() == LoopVectorizeHints::FK_Disabled)
     return LV_NAME;
-  if (getForce() == LoopVectorizeHints::FK_Undefined && getWidth().isZero())
+  if (getVectorizationForce() == LoopVectorizeHints::FK_Undefined &&
+      getWidth().isZero())
     return LV_NAME;
   return OptimizationRemarkAnalysis::AlwaysPrint;
 }
 
 bool LoopVectorizeHints::allowReordering() const {
   // Allow the vectorizer to change the order of operations if enabling
   // loop hints are provided
   ElementCount EC = getWidth();
   return HintsAllowReordering &&
-         (getForce() == LoopVectorizeHints::FK_Enabled ||
+         (getVectorizationForce() == LoopVectorizeHints::FK_Enabled ||
           EC.getKnownMinValue() > 1);
 }
 
 void LoopVectorizeHints::getHintsFromMetadata() {
   MDNode *LoopID = TheLoop->getLoopID();
   if (!LoopID)
     return;
 
if (!Name.startswith(Prefix()))
     return;
   Name = Name.substr(Prefix().size(), StringRef::npos);
 
   const ConstantInt *C = mdconst::dyn_extract<ConstantInt>(Arg);
   if (!C)
     return;
   unsigned Val = C->getZExtValue();
 
-  Hint *Hints[] = {&Width,        &Interleave, &Force,
-                   &IsVectorized, &Predicate,  &Scalable};
+  Hint *Hints[] = {&Width,           &Interleave,   &VectorizationForce,
+                   &InterleaveForce, &IsVectorized, &Predicate,
+                   &Scalable};
   for (auto *H : Hints) {
     if (Name == H->Name) {
       if (H->validate(Val))
         H->Value = Val;
       else
         LLVM_DEBUG(dbgs() << "LV: ignoring invalid hint '" << Name << "'\n");
       break;
     }
bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
 
   LLVM_DEBUG(dbgs() << "LV: We can vectorize this loop"
                     << (LAI->getRuntimePointerChecking()->Need
                             ? " (with a runtime bound check)"
                             : "")
                     << "!\n");
 
   unsigned SCEVThreshold = VectorizeSCEVCheckThreshold;
-  if (Hints->getForce() == LoopVectorizeHints::FK_Enabled)
+  if (Hints->getVectorizationForce() == LoopVectorizeHints::FK_Enabled)
     SCEVThreshold = PragmaVectorizeSCEVCheckThreshold;
 
   if (PSE.getPredicate().getComplexity() > SCEVThreshold) {
     reportVectorizationFailure("Too many SCEV checks needed",
         "Too many SCEV assumptions need to be made and checked at runtime",
         "TooManySCEVRunTimeChecks", ORE, TheLoop);
     if (DoExtraAnalysis)
       Result = false;

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

 // representation for pragma 'omp simd' is introduced.
 static bool isExplicitVecOuterLoop(Loop *OuterLp,
                                    OptimizationRemarkEmitter *ORE) {
   assert(!OuterLp->isInnermost() && "This is not an outer loop");
   LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
 
   // Only outer loops with an explicit vectorization hint are supported.
   // Unannotated outer loops are ignored.
-  if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
+  if (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Undefined)
     return false;
 
   Function *Fn = OuterLp->getHeader()->getParent();
   if (!Hints.allowVectorization(Fn, OuterLp,
                                 true /*VectorizeOnlyWhenForced*/)) {
     LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
     return false;
   }
 
-  if (Hints.getInterleave() > 1) {
+  if (Hints.getInterleaveForce()) {
     // TODO: Interleave support is future work.
     LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
                          "outer loops.\n");
     Hints.emitRemarkWithHints();
     return false;
   }
 
   return true;

 
 BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
   BasicBlock *const SCEVCheckBlock =
       RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader, LoopExitBlock);
   if (!SCEVCheckBlock)
     return nullptr;
 
   assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||
-           (OptForSizeBasedOnProfile &&
-            Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&
+           (OptForSizeBasedOnProfile && Cost->Hints->getVectorizationForce() !=
+                                            LoopVectorizeHints::FK_Enabled)) &&
          "Cannot SCEV check stride or overflow when optimizing for size");
 
-
   // Update dominator only if this is first RT check.
   if (LoopBypassBlocks.empty()) {
     DT->changeImmediateDominator(Bypass, SCEVCheckBlock);
     if (!Cost->requiresScalarEpilogue(VF))
       // If there is an epilogue which must run, there's no edge from the
       // middle block to exit blocks  and thus no need to update the immediate
       // dominator of the exit blocks.
       DT->changeImmediateDominator(LoopExitBlock, SCEVCheckBlock);
BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
 
   // Check if we generated code that checks in runtime if arrays overlap. We put
   // the checks into a separate block to make the more common case of few
   // elements faster.
   if (!MemCheckBlock)
     return nullptr;
 
   if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
-    assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
+    assert(Cost->Hints->getVectorizationForce() ==
+               LoopVectorizeHints::FK_Enabled &&
            "Cannot emit memory checks when optimizing for size, unless forced "
            "to vectorize.");
     ORE->emit([&]() {
       return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
                                         OrigLoop->getStartLoc(),
                                         OrigLoop->getHeader())
              << "Code-size may be reduced by not forcing "
                 "vectorization, or by source-code modifications "
VectorizationFactor LoopVectorizationCostModel::selectVectorizationFactor(
   assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop");
   assert(VFCandidates.count(ElementCount::getFixed(1)) &&
          "Expected Scalar VF to be a candidate");
 
   const VectorizationFactor ScalarCost(ElementCount::getFixed(1), ExpectedCost,
                                        ExpectedCost);
   VectorizationFactor ChosenFactor = ScalarCost;
 
-  bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;
+  bool ForceVectorization =
+      Hints->getVectorizationForce() == LoopVectorizeHints::FK_Enabled;
   if (ForceVectorization && VFCandidates.size() > 1) {
     // Ignore scalar width, because the user explicitly wants vectorization.
     // Initialize cost to max so that VF = 2 is, at least, chosen during cost
     // evaluation.
     ChosenFactor.Cost = InstructionCost::getMax();
   }
 
   SmallVector<InstructionVFPair> InvalidCosts;
static ScalarEpilogueLowering getScalarEpilogueLowering(
   // 1) OptSize takes precedence over all other options, i.e. if this is set,
   // don't look at hints or options, and don't request a scalar epilogue.
   // (For PGSO, as shouldOptimizeForSize isn't currently accessible from
   // LoopAccessInfo (due to code dependency and not being able to reliably get
   // PSI/BFI from a loop analysis under NPM), we cannot suppress the collection
   // of strides in LoopAccessInfo::analyzeLoop() and vectorize without
   // versioning when the vectorization is forced, unlike hasOptSize. So revert
   // back to the old way and vectorize with versioning when forced. See D81345.)
-  if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
-                                                      PGSOQueryType::IRPass) &&
-                          Hints.getForce() != LoopVectorizeHints::FK_Enabled))
+  if (F->hasOptSize() ||
+      (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
+       Hints.getInterleaveForce() != LoopVectorizeHints::FK_Enabled))
     return CM_ScalarEpilogueNotAllowedOptSize;
 
   // 2) If set, obey the directives
   if (PreferPredicateOverEpilogue.getNumOccurrences()) {
     switch (PreferPredicateOverEpilogue) {
     case PreferPredicateTy::ScalarEpilogue:
       return CM_ScalarEpilogueAllowed;
     case PreferPredicateTy::PredicateElseScalarEpilogue:
#endif /* NDEBUG */
 
   LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
                     << L->getHeader()->getParent()->getName() << "' from "
                     << DebugLocStr << "\n");
 
   LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
 
   LLVM_DEBUG(
-      dbgs() << "LV: Loop hints:"
-             << " force="
-             << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
-                     ? "disabled"
-                     : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
-                            ? "enabled"
-                            : "?"))
-             << " width=" << Hints.getWidth()
-             << " interleave=" << Hints.getInterleave() << "\n");
+      dbgs()
+      << "LV: Loop hints:"
+      << " force_vectorization="
+      << (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Disabled
+              ? "disabled"
+              : (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled
+                     ? "enabled"
+                     : "?"))
+      << " width=" << Hints.getWidth() << " force_interleave="
+      << (Hints.getInterleaveForce() == LoopVectorizeHints::FK_Disabled
+              : (Hints.getInterleaveForce() == LoopVectorizeHints::FK_Enabled
 
   // Function containing loop
   Function *F = L->getHeader()->getParent();
 
   // Looking at the diagnostic output is the only way to determine if a loop
   // was vectorized (other than looking at the IR or machine code), so it
   // is important to generate an optimization remark for each loop. Most of
   // these messages are generated as OptimizationRemarkAnalysis. Remarks
#endif /* NDEBUG */
 
   // Check the loop for a trip count threshold: vectorize loops with a tiny trip
   // count by optimizing for size, to minimize overheads.
   auto ExpectedTC = getSmallBestKnownTC(*SE, L);
   if (ExpectedTC && *ExpectedTC < TinyTripCountVectorThreshold) {
     LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
                       << "This loop is worth vectorizing only if no scalar "
                       << "iteration overheads are incurred.");
-    if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
+    if (Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled)
       LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
     else {
       if (*ExpectedTC > TTI->getMinTripCountTailFoldingThreshold()) {
         LLVM_DEBUG(dbgs() << "\n");
         SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
       } else {
         LLVM_DEBUG(dbgs() << " But the target considers the trip count too "
                              "small to consider vectorizing.\n");
if (MaybeVF) {
     unsigned SelectedIC = std::max(IC, UserIC);
     //  Optimistically generate runtime checks if they are needed. Drop them if
     //  they turn out to not be profitable.
     if (VF.Width.isVector() || SelectedIC > 1)
       Checks.Create(L, *LVL.getLAI(), PSE.getPredicate(), VF.Width, SelectedIC);
 
     // Check if it is profitable to vectorize with runtime checks.
     bool ForceVectorization =
-        Hints.getForce() == LoopVectorizeHints::FK_Enabled;
+        Hints.getVectorizationForce() == LoopVectorizeHints::FK_Enabled;
     if (!ForceVectorization &&
         !areRuntimeChecksProfitable(Checks, VF, CM.getVScaleForTuning(), L,
                                     *PSE.getSE())) {
       ORE->emit([&]() {
         return OptimizationRemarkAnalysisAliasing(
                    DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
                    L->getHeader())
                << "loop not vectorized: cannot prove it is safe to reorder "

llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll

for.inc8:                                         ; preds = %for.body3
   %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
   br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
 
 for.end10:                                        ; preds = %for.inc8
   ret void
 }
 
 
-!1 = distinct !{!1, !2}
+!1 = distinct !{!1, !2, !3}
 !2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll

 ; REQUIRES: asserts
 ; RUN: opt < %s -loop-vectorize -mtriple riscv64 -riscv-v-vector-bits-min=128 -mattr="+v" -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
 
-; CHECK: LV: Loop hints: force=enabled
+; CHECK: LV: Loop hints: force_vectorization=enabled width=vscale x 0 force_interleave=? interleave=0
 ; CHECK: LV: Scalar loop costs: 7.
 ; ChosenFactor.Cost is 8, but the real cost will be divided by the width, which is 4.
 ; CHECK: LV: Vector loop of width 2 costs: 4.
 ; Regardless of force vectorization or not, this loop will eventually be vectorized because of the cost model.
 ; Therefore, the following message does not need to be printed even if vectorization is explicitly forced in the metadata.
 ; CHECK-NOT: LV: Vectorization seems to be not beneficial, but was forced by a user.
 
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"

llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll

 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; This is the loop in c++ being vectorize in this file with
 ;experimental.vector.reverse
 ;  #pragma clang loop vectorize_width(4, scalable)
 ;  for (int i = N-1; i >= 0; --i)
 ;    a[i] = b[i] + 1.0;
 
 ; REQUIRES: asserts
 ; RUN: opt -loop-vectorize -dce -instcombine -mtriple riscv64-linux-gnu \
 ; RUN:   -mattr=+v -debug-only=loop-vectorize -scalable-vectorization=on \
 ; RUN:   -riscv-v-vector-bits-min=128 -disable-output < %s 2>&1 | FileCheck %s
 
 define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
 ; CHECK-LABEL: 'vector_reverse_i64'
-; CHECK-NEXT:  LV: Loop hints: force=enabled width=vscale x 4 interleave=0
+; CHECK-NEXT:  LV: Loop hints: force_vectorization=enabled width=vscale x 4 force_interleave=? interleave=0
 ; CHECK-NEXT:  LV: Found a loop: for.body
 ; CHECK-NEXT:  LV: Found an induction variable.
 ; CHECK-NEXT:  LV: Found an induction variable.
 ; CHECK-NEXT:  LV: Did not find one integer induction var.
 ; CHECK-NEXT:  LV: We can vectorize this loop (with a runtime bound check)!
 ; CHECK-NEXT:  LV: Found trip count: 0
 ; CHECK-NEXT:  LV: Scalable vectorization is available
 ; CHECK-NEXT:  LV: The max safe fixed VF is: 67108864.
for.body:                                         ; preds = %for.body.preheader, %for.body
   store i32 %add9, ptr %arrayidx3, align 4
   %cmp = icmp ugt i64 %indvars.iv, 1
   %indvars.iv.next = add nsw i64 %indvars.iv, -1
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
 }
 
 define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
 ; CHECK-LABEL: 'vector_reverse_f32'
-; CHECK-NEXT:  LV: Loop hints: force=enabled width=vscale x 4 interleave=0
+; CHECK-NEXT:  LV: Loop hints: force_vectorization=enabled width=vscale x 4 force_interleave=? interleave=0
 ; CHECK-NEXT:  LV: Found a loop: for.body
 ; CHECK-NEXT:  LV: Found an induction variable.
 ; CHECK-NEXT:  LV: Found an induction variable.
 ; CHECK-NEXT:  LV: Found FP op with unsafe algebra.
 ; CHECK-NEXT:  LV: Did not find one integer induction var.
 ; CHECK-NEXT:  LV: We can vectorize this loop (with a runtime bound check)!
 ; CHECK-NEXT:  LV: Found trip count: 0
 ; CHECK-NEXT:  LV: Scalable vectorization is available

llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll

for.body:                                         ; preds = %for.body, %entry
   %exitcond = icmp eq i64 %indvars.iv.next, 48
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !2
 
 for.end:                                          ; preds = %for.body
   %1 = load i32, i32* %a, align 4
   ret i32 %1
 }
 
-!0 = !{!0, !1}
+!0 = !{!0, !1, !4}
 !1 = !{!"llvm.loop.vectorize.enable", i1 1}
-!2 = !{!2, !3}
+!2 = !{!2, !3, !4}
 !3 = !{!"llvm.loop.vectorize.enable", i1 0}
+!4 = !{!"llvm.loop.interleave.enable", i1 0}

llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll

for.inc8:                                         ; preds = %for.body3
   %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
   %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
   br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
 
 for.end10:                                        ; preds = %for.inc8
   ret void
 }
 
-!1 = distinct !{!1, !2}
+!1 = distinct !{!1, !2, !3}
 !2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll

 ; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -debug-only=loop-vectorize -stats -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
-; CHECK: LV: Loop hints: force=enabled
-; CHECK: LV: Loop hints: force=?
+; CHECK: LV: Loop hints: force_vectorization=enabled width=0 force_interleave=disabled interleave=0
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0
 ; No more loops in the module
 ; CHECK-NOT: LV: Loop hints: force=
 ; CHECK: 2 loop-vectorize               - Number of loops analyzed for vectorization
 ; CHECK: 1 loop-vectorize               - Number of loops vectorized
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 

 
 for.end.loopexit:
   br label %for.end
 
 for.end:
   ret void
 }
 
-!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}
+!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}, !4}
 !2 = !{!"llvm.loop.vectorize.enable", i1 true}
 !11 = distinct !{}
+!4 = !{!"llvm.loop.interleave.enable", i1 false}
 
 ;
 ; This method will not be vectorized, as scalar cost is lower than any of vector costs.
 ;
 
 define void @not_vectorized(float* noalias nocapture %A, float* noalias nocapture %B) {
 entry:
   br label %for.body

llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll

 ; CHECK: remark: source.cpp:12:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
 
 ; void test_array_bounds(int *A, int *B, int Length) {
 ; #pragma clang loop vectorize(enable)
 ;   for (int i = 0; i < Length; i++)
 ;     A[i] = A[B[i]];
 ; }
 ; CHECK: remark: source.cpp:19:5: loop not vectorized: cannot identify array bounds
-; CHECK: remark: source.cpp:18:8: loop not vectorized (Force=true)
+; CHECK: remark: source.cpp:18:8: loop not vectorized (VectorizationForce=true)
 ; CHECK: warning: source.cpp:18:8: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
 
 ; int foo();
 ; void test_multiple_failures(int *A) {
 ;   int k = 0;
 ; #pragma clang loop vectorize(enable) interleave(enable)
 ;   for (int i = 0; i < 1000; i+=A[i]) {
 ;     if (A[i])

 ; YAML-NEXT: ...
 ; YAML-NEXT: --- !Missed
 ; YAML-NEXT: Pass:            loop-vectorize
 ; YAML-NEXT: Name:            MissedDetails
 ; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 18, Column: 8 }
 ; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
 ; YAML-NEXT: Args:
 ; YAML-NEXT:   - String:          loop not vectorized
-; YAML-NEXT:   - String:          ' (Force='
-; YAML-NEXT:   - Force:           'true'
+; YAML-NEXT:   - String:          ' (VectorizationForce='
+; YAML-NEXT:   - VectorizationForce:           'true'
 ; YAML-NEXT:   - String:          ')'
 ; YAML-NEXT: ...
 ; YAML-NEXT: --- !Failure
 ; YAML-NEXT: Pass:            transform-warning
 ; YAML-NEXT: Name:            FailedRequestedVectorization
 ; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 18, Column: 8 }
 ; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
 ; YAML-NEXT: Args:

llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll

 ;       a[i*M+j] = b[i*M+j] * b[i*M+j];
 ;     }
 ;   }
 ; }
 
 ; Case 1: Annotated outer loop WITH vector width information must be collected.
 
 ; CHECK-LABEL: vector_width
-; CHECK: LV: Loop hints: force=enabled width=4 interleave=0
+; CHECK: LV: Loop hints: force_vectorization=enabled width=4 force_interleave=disabled interleave=0
 ; CHECK: LV: We can vectorize this outer loop!
 ; CHECK: LV: Using user VF 4 to build VPlans.
-; CHECK-NOT: LV: Loop hints: force=?
+; CHECK-NOT: LV: Loop hints:  force_vectorization=? width=0 force_interleave=? interleave=0
 ; CHECK-NOT: LV: Found a loop: inner.body
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @vector_width(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
 entry:
   %cmp32 = icmp sgt i32 %N, 0
   br i1 %cmp32, label %outer.ph, label %for.end15

 
 for.end15:                                        ; preds = %outer.inc, %entry
   ret void
 }
 
 ; Case 2: Annotated outer loop WITHOUT vector width information must be collected.
 
 ; CHECK-LABEL: case2
-; CHECK: LV: Loop hints: force=enabled width=0 interleave=0
+; CHECK: LV: Loop hints: force_vectorization=enabled width=0 force_interleave=disabled interleave=0
 ; CHECK: LV: We can vectorize this outer loop!
 ; CHECK: LV: Using VF 1 to build VPlans.
 
 define void @case2(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
 entry:
   %cmp32 = icmp sgt i32 %N, 0
   br i1 %cmp32, label %outer.ph, label %for.end15
 

 }
 
 ; Case 3: Annotated outer loop WITH vector width and interleave information
 ; doesn't have to be collected.
 
 ; CHECK-LABEL: case3
 ; CHECK-NOT: LV: Loop hints: force=enabled
 ; CHECK-NOT: LV: We can vectorize this outer loop!
-; CHECK: LV: Loop hints: force=?
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0
 ; CHECK: LV: Found a loop: inner.body
 
 define void @case3(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
 entry:
   %cmp32 = icmp sgt i32 %N, 0
   br i1 %cmp32, label %outer.ph, label %for.end15
 
 outer.ph:                                         ; preds = %entry

 }
 
 ; Case 4: Outer loop without any explicit vectorization annotation doesn't have
 ; to be collected.
 
 ; CHECK-LABEL: case4
 ; CHECK-NOT: LV: Loop hints: force=enabled
 ; CHECK-NOT: LV: We can vectorize this outer loop!
-; CHECK: LV: Loop hints: force=?
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=0
 ; CHECK: LV: Found a loop: inner.body
 
 define void @case4(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
 entry:
   %cmp32 = icmp sgt i32 %N, 0
   br i1 %cmp32, label %outer.ph, label %for.end15
 
 outer.ph:                                         ; preds = %entry

 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{!"clang version 6.0.0"}
 !2 = !{!3, !3, i64 0}
 !3 = !{!"int", !4, i64 0}
 !4 = !{!"omnipotent char", !5, i64 0}
 !5 = !{!"Simple C/C++ TBAA"}
 ; Case 1
-!6 = distinct !{!6, !7, !8}
+!6 = distinct !{!6, !7, !8, !12}
 !7 = !{!"llvm.loop.vectorize.width", i32 4}
 !8 = !{!"llvm.loop.vectorize.enable", i1 true}
 ; Case 2
-!9 = distinct !{!9, !8}
+!9 = distinct !{!9, !8, !12}
 ; Case 3
 !10 = !{!"llvm.loop.interleave.count", i32 2}
-!11 = distinct !{!11, !7, !10, !8}
+!11 = distinct !{!11, !7, !10, !8, !13}
+!12 = !{!"llvm.loop.interleave.enable", i1 false}
+!13 = !{!"llvm.loop.interleave.enable", i1 true}

llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll

 !llvm.ident = !{!1}
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{!"clang version 6.0.0"}
 !2 = !{!3, !3, i64 0}
 !3 = !{!"int", !4, i64 0}
 !4 = !{!"omnipotent char", !5, i64 0}
 !5 = !{!"Simple C/C++ TBAA"}
-!6 = distinct !{!6, !7, !8}
+!6 = distinct !{!6, !7, !8, !9}
 !7 = !{!"llvm.loop.vectorize.width", i32 8}
 !8 = !{!"llvm.loop.vectorize.enable", i1 true}
+!9 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll

 !llvm.ident = !{!1}
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{!"clang version 6.0.0"}
 !2 = !{!3, !3, i64 0}
 !3 = !{!"int", !4, i64 0}
 !4 = !{!"omnipotent char", !5, i64 0}
 !5 = !{!"Simple C/C++ TBAA"}
-!6 = distinct !{!6, !7, !8}
+!6 = distinct !{!6, !7, !8, !9}
 !7 = !{!"llvm.loop.vectorize.width", i32 8}
 !8 = !{!"llvm.loop.vectorize.enable", i1 true}
+!9 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/no_switch.ll

 ; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -S 2>&1 | FileCheck %s
 ; RUN: opt < %s -loop-vectorize -force-vector-width=1 -transform-warning -S 2>&1 | FileCheck %s -check-prefix=NOANALYSIS
 ; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -pass-remarks-missed='loop-vectorize' -S 2>&1 | FileCheck %s -check-prefix=MOREINFO
 
 ; CHECK: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
 ; CHECK: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
 
 ; NOANALYSIS-NOT: remark: {{.*}}
 ; NOANALYSIS: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
 
 ; MOREINFO: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
-; MOREINFO: remark: source.cpp:4:5: loop not vectorized (Force=true, Vector Width=4)
+; MOREINFO: remark: source.cpp:4:5: loop not vectorized (VectorizationForce=true, Vector Width=4)
 ; MOREINFO: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
 
 ; CHECK: _Z11test_switchPii
 ; CHECK-NOT: x i32>
 ; CHECK: ret
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 

llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll

 ; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -S 2>&1 | FileCheck %s
 ; RUN: opt < %s -loop-vectorize -force-vector-width=1 -transform-warning -S 2>&1 | FileCheck %s -check-prefix=NOANALYSIS
 ; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -pass-remarks-missed='loop-vectorize' -S 2>&1 | FileCheck %s -check-prefix=MOREINFO
 
 ; This test is a copy of no_switch.ll, with the "llvm.loop.vectorize.enable" metadata set to false.
-; It tests that vectorization is explicitly disabled and no warnings are emitted.
+; It tests that vectorization and interleave is explicitly disabled and no warnings are emitted.
 
 ; CHECK-NOT: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
 ; CHECK-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
 
 ; NOANALYSIS-NOT: remark: {{.*}}
 ; NOANALYSIS-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
 
-; MOREINFO: remark: source.cpp:4:5: loop not vectorized: vectorization is explicitly disabled
+; MOREINFO: remark: source.cpp:4:5: loop not vectorized: vectorization and interleave is explicitly disabled
 ; MOREINFO-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
 
 ; CHECK: _Z11test_switchPii
 ; CHECK-NOT: x i32>
 ; CHECK: ret
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 

 !4 = distinct !DISubprogram(name: "test_switch", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
 !5 = !DIFile(filename: "source.cpp", directory: ".")
 !6 = !DISubroutineType(types: !2)
 !7 = !{i32 2, !"Dwarf Version", i32 2}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 3.5.0"}
 !10 = !DILocation(line: 3, column: 8, scope: !11)
 !11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
-!12 = !{!12, !13, !13}
+!12 = !{!12, !13, !25}
 !13 = !{!"llvm.loop.vectorize.enable", i1 false}
 !14 = !DILocation(line: 4, column: 5, scope: !15)
 !15 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !11)
 !16 = !{!17, !17, i64 0}
 !17 = !{!"int", !18, i64 0}
 !18 = !{!"omnipotent char", !19, i64 0}
 !19 = !{!"Simple C/C++ TBAA"}
 !20 = !DILocation(line: 6, column: 7, scope: !21)
 !21 = distinct !DILexicalBlock(line: 4, column: 18, file: !1, scope: !15)
 !22 = !DILocation(line: 7, column: 5, scope: !21)
 !23 = !DILocation(line: 9, column: 7, scope: !21)
 !24 = !DILocation(line: 14, column: 1, scope: !4)
+!25 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/nounroll.ll

 ; REQUIRES: asserts
 ; RUN: opt < %s -passes='loop-vectorize' -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-n32:64-S128-v256:256:256-v512:512:512"
 
 ; CHECK: LV: Checking a loop in 'f1'
-; CHECK: LV: Loop hints: force=? width=0 interleave=1
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=1
 define dso_local void @f1(i32 signext %n, i32* %A) {
 entry:
   %cmp1 = icmp sgt i32 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
 for.body.preheader:                               ; preds = %entry
   %wide.trip.count = zext i32 %n to i64
   br label %for.body

 for.end.loopexit:                                 ; preds = %for.body
   br label %for.end
 
 for.end:                                          ; preds = %for.end.loopexit, %entry
   ret void
 }
 
 ; CHECK: LV: Checking a loop in 'f2'
-; CHECK: LV: Loop hints: force=? width=0 interleave=4
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=4
 define dso_local void @f2(i32 signext %n, i32* %A) {
 entry:
   %cmp1 = icmp sgt i32 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
 for.body.preheader:                               ; preds = %entry
   %wide.trip.count = zext i32 %n to i64
   br label %for.body

 for.end.loopexit:                                 ; preds = %for.body
   br label %for.end
 
 for.end:                                          ; preds = %for.end.loopexit, %entry
   ret void
 }
 
 ; CHECK: LV: Checking a loop in 'f3'
-; CHECK: LV: Loop hints: force=? width=0 interleave=1
+; CHECK: LV: Loop hints: force_vectorization=? width=0 force_interleave=? interleave=1
 define dso_local void @f3(i32 signext %n, i32* %A) {
 entry:
   %cmp1 = icmp sgt i32 %n, 0
   br i1 %cmp1, label %for.body.preheader, label %for.end
 
 for.body.preheader:                               ; preds = %entry
   %wide.trip.count = zext i32 %n to i64
   br label %for.body

llvm/test/Transforms/LoopVectorize/optsize.ll

 !7 = !{!"NumCounts", i64 3}
 !8 = !{!"NumFunctions", i64 3}
 !9 = !{!"DetailedSummary", !10}
 !10 = !{!11, !12, !13}
 !11 = !{i32 10000, i64 100, i32 1}
 !12 = !{i32 999000, i64 100, i32 1}
 !13 = !{i32 999999, i64 1, i32 2}
 !14 = !{!"function_entry_count", i64 0}
-!15 = distinct !{!15, !16}
-!16 = !{!"llvm.loop.vectorize.enable", i1 true}
+!15 = distinct !{!15, !16, !17}
+!16 = !{!"llvm.loop.interleave.enable", i1 true}
+!17 = !{!"llvm.loop.vectorize.enable", i1 true}

llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll

 exit:                                             ; preds = %loop.1.latch
   ret void
 
 ; uselistorder directives
   uselistorder label %loop.3, { 1, 0 }
   uselistorder label %loop.2.header, { 1, 0 }
 }
 
-!0 = distinct !{!0, !1, !2}
+!0 = distinct !{!0, !1, !2, !3}
 !1 = !{!"llvm.loop.vectorize.width", i32 4}
 !2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll

for.inc8:                                         ; preds = %for.body3
   %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
   %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
   br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
 
 for.end10:                                        ; preds = %for.inc8
   ret void
 }
 
-!1 = distinct !{!1, !2, !3}
+!1 = distinct !{!1, !2, !3, !4}
 !2 = !{!"llvm.loop.vectorize.width", i32 4}
 !3 = !{!"llvm.loop.vectorize.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll

for.inc9:                                         ; preds = %for.cond1.for.inc9_crit_edge, %for.body
   %indvars.iv.next26 = add nuw nsw i64 %indvars.iv25, 1
   %exitcond28 = icmp eq i64 %indvars.iv.next26, %wide.trip.count27
   br i1 %exitcond28, label %for.end11, label %for.body, !llvm.loop !1
 
 for.end11:                                        ; preds = %for.inc9, %entry
   ret void
 }
 
-!1 = distinct !{!1, !2, !3}
+!1 = distinct !{!1, !2, !3, !4}
 !2 = !{!"llvm.loop.vectorize.width", i32 4}
 !3 = !{!"llvm.loop.vectorize.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll

 ; RUN: opt < %s -O2 -force-vector-interleave=2 -force-vector-width=4 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; CHECK: LV: Checking a loop in 'foo'
-; CHECK: LV: Loop hints: force=enabled
+; CHECK: LV: Loop hints: force_vectorization=enabled width=4 force_interleave=? interleave=2
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Ensure that "llvm.loop.vectorize.enable" metadata was not lost even
 ; if loop was not rotated.
 ;
 ; See http://reviews.llvm.org/D3348 for details.
 ;

llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll

 for1.latch:                                ; preds = %for.cond.cleanup9
   %c = call i1 @cond()
   br i1 %c, label %exit, label %for1.header, !llvm.loop !0
 
 exit:                                 ; preds = %for.cond.cleanup3, %entry
   ret void
 }
 
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll

outer.latch:
   %outer.iv.next = add nuw nsw i64 %outer.iv, 1
   %outer.ec = icmp eq i64 %outer.iv.next, 8
   br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
 
 exit:
   ret void
 }
 
-!1 = distinct !{!1, !2, !3}
+!1 = distinct !{!1, !2, !3, !4}
 !2 = !{!"llvm.loop.vectorize.width", i32 4}
 !3 = !{!"llvm.loop.vectorize.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll

for1.latch:                                               ; preds = %for2.header
   %indvar11 = add nuw nsw i64 %indvar1, 1
   %for1.cond = icmp eq i64 %indvar11, 1000
   br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
 
 exit:                                                    ; preds = %for1.latch
   ret void
 }
 
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll

for1.latch:
   %indvar11 = add nuw nsw i64 %indvar1, 1
   %for1.cond = icmp eq i64 %indvar11, 1000
   br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
 
 exit:
   ret void
 }
 
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.interleave.enable", i1 false}

llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll

 for1.latch:
   %indvar11 = add nuw nsw i64 %indvar1, 1
   %for1.cond = icmp eq i64 %indvar11, 1000
   br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
 
 exit:
   ret void
 }
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.interleave.enable", i1 false}