[mlir][llvm] Adapt loop metadata to match llvm

[Dinistro]

Summary

This commit adds support for the "llvm.loop.isvectorized" metadata and
ensures that the unroll followups match llvm's naming.

[gysit]

Thanks!

mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td

let description = [{
     the "!llvm.loop.unroll" metadata.
   }];
 
   let parameters = (ins
     OptionalParameter<"BoolAttr">:$disable,
     OptionalParameter<"IntegerAttr">:$count,
     OptionalParameter<"BoolAttr">:$runtimeDisable,
     OptionalParameter<"BoolAttr">:$full,
-    OptionalParameter<"LoopAnnotationAttr">:$followup,
-    OptionalParameter<"LoopAnnotationAttr">:$followupRemainder
+    OptionalParameter<"LoopAnnotationAttr">:$followupUnrolled,
+    OptionalParameter<"LoopAnnotationAttr">:$followupRemainder,
+    OptionalParameter<"LoopAnnotationAttr">:$followupAll
   );
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
 def LoopUnrollAndJamAttr : LLVM_Attr<"LoopUnrollAndJam", "loop_unroll_and_jam"> {
   let description = [{
     This attribute defines "unroll and jam" specific loop annotations that map to
let parameters = (ins
     OptionalParameter<"LoopVectorizeAttr">:$vectorize,
     OptionalParameter<"LoopInterleaveAttr">:$interleave,
     OptionalParameter<"LoopUnrollAttr">:$unroll,
     OptionalParameter<"LoopUnrollAndJamAttr">:$unrollAndJam,
     OptionalParameter<"LoopLICMAttr">:$licm,
     OptionalParameter<"LoopDistributeAttr">:$distribute,
     OptionalParameter<"LoopPipelineAttr">:$pipeline,
     OptionalParameter<"BoolAttr">:$mustProgress,
+    OptionalParameter<"BoolAttr">:$isVectorized,
     OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses
   );
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
 //===----------------------------------------------------------------------===//
 // DebugInfo Attributes

mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp

struct LoopMetadataConversion {
   /// Helper function to get and erase a property.
   const llvm::MDNode *lookupAndEraseProperty(StringRef name);
 
   /// Helper functions to lookup and convert MDNodes into a specifc attribute
   /// kind. These functions return null-attributes if there is no node with the
   /// specified name, or failure, if the node is ill-formatted.
   FailureOr<BoolAttr> lookupUnitNode(StringRef name);
   FailureOr<BoolAttr> lookupBoolNode(StringRef name, bool negated = false);
+  FailureOr<BoolAttr> lookupIntNodeAsBoolAttr(StringRef name);
   FailureOr<IntegerAttr> lookupIntNode(StringRef name);
   FailureOr<llvm::MDNode *> lookupMDNode(StringRef name);
   FailureOr<SmallVector<llvm::MDNode *>> lookupMDNodes(StringRef name);
   FailureOr<LoopAnnotationAttr> lookupFollowupNode(StringRef name);
   FailureOr<BoolAttr> lookupBooleanUnitNode(StringRef enableName,
                                             StringRef disableName,
                                             bool negated = false);
 
FailureOr<BoolAttr> LoopMetadataConversion::lookupBoolNode(StringRef name,
   llvm::ConstantInt *val =
       llvm::mdconst::dyn_extract<llvm::ConstantInt>(property->getOperand(1));
   if (!val || val->getBitWidth() != 1)
     return emitNodeWarning();
 
   return BoolAttr::get(ctx, val->getValue().getLimitedValue(1) ^ negated);
 }
 
+FailureOr<BoolAttr>
+LoopMetadataConversion::lookupIntNodeAsBoolAttr(StringRef name) {
+  const llvm::MDNode *property = lookupAndEraseProperty(name);
+  if (!property)
+    return BoolAttr(nullptr);
+
+  auto emitNodeWarning = [&]() {
+    return emitWarning(loc)
+           << "expected metadata node " << name << " to hold an integer value";
+  };
+
+  if (property->getNumOperands() != 2)
+    return emitNodeWarning();
+  llvm::ConstantInt *val =
+      llvm::mdconst::dyn_extract<llvm::ConstantInt>(property->getOperand(1));
+  if (!val || val->getBitWidth() != 32)
+    return emitNodeWarning();
+
+  return BoolAttr::get(ctx, val->getValue().getLimitedValue(1));
+}
+
 FailureOr<IntegerAttr> LoopMetadataConversion::lookupIntNode(StringRef name) {
   const llvm::MDNode *property = lookupAndEraseProperty(name);
   if (!property)
     return IntegerAttr(nullptr);
 
   auto emitNodeWarning = [&]() {
     return emitWarning(loc)
            << "expected metadata node " << name << " to hold an i32 value";

 
 FailureOr<LoopUnrollAttr> LoopMetadataConversion::convertUnrollAttr() {
   FailureOr<BoolAttr> disable = lookupBooleanUnitNode(
       "llvm.loop.unroll.enable", "llvm.loop.unroll.disable", /*negated=*/true);
   FailureOr<IntegerAttr> count = lookupIntNode("llvm.loop.unroll.count");
   FailureOr<BoolAttr> runtimeDisable =
       lookupUnitNode("llvm.loop.unroll.runtime.disable");
   FailureOr<BoolAttr> full = lookupUnitNode("llvm.loop.unroll.full");
-  FailureOr<LoopAnnotationAttr> followup =
-      lookupFollowupNode("llvm.loop.unroll.followup");
+  FailureOr<LoopAnnotationAttr> followupUnrolled =
+      lookupFollowupNode("llvm.loop.unroll.followup_unrolled");
   FailureOr<LoopAnnotationAttr> followupRemainder =
       lookupFollowupNode("llvm.loop.unroll.followup_remainder");
+  FailureOr<LoopAnnotationAttr> followupAll =
+      lookupFollowupNode("llvm.loop.unroll.followup_all");
 
   return createIfNonNull<LoopUnrollAttr>(ctx, disable, count, runtimeDisable,
-                                         full, followup, followupRemainder);
+                                         full, followupUnrolled,
+                                         followupRemainder, followupAll);
 }
 
 FailureOr<LoopUnrollAndJamAttr>
 LoopMetadataConversion::convertUnrollAndJamAttr() {
   FailureOr<BoolAttr> disable = lookupBooleanUnitNode(
       "llvm.loop.unroll_and_jam.enable", "llvm.loop.unroll_and_jam.disable",
       /*negated=*/true);
   FailureOr<IntegerAttr> count =
LoopAnnotationAttr LoopMetadataConversion::convert() {
   FailureOr<LoopVectorizeAttr> vecAttr = convertVectorizeAttr();
   FailureOr<LoopInterleaveAttr> interleaveAttr = convertInterleaveAttr();
   FailureOr<LoopUnrollAttr> unrollAttr = convertUnrollAttr();
   FailureOr<LoopUnrollAndJamAttr> unrollAndJamAttr = convertUnrollAndJamAttr();
   FailureOr<LoopLICMAttr> licmAttr = convertLICMAttr();
   FailureOr<LoopDistributeAttr> distributeAttr = convertDistributeAttr();
   FailureOr<LoopPipelineAttr> pipelineAttr = convertPipelineAttr();
   FailureOr<BoolAttr> mustProgress = lookupUnitNode("llvm.loop.mustprogress");
+  FailureOr<BoolAttr> isVectorized =
+      lookupIntNodeAsBoolAttr("llvm.loop.isvectorized");
   FailureOr<SmallVector<SymbolRefAttr>> parallelAccesses =
       convertParallelAccesses();
 
   // Drop the metadata if there are parts that cannot be imported.
   if (!propertyMap.empty()) {
     for (auto name : propertyMap.keys())
       emitWarning(loc) << "unknown loop annotation " << name;
     return {};
   }
 
   return createIfNonNull<LoopAnnotationAttr>(
       ctx, disableNonForced, vecAttr, interleaveAttr, unrollAttr,
       unrollAndJamAttr, licmAttr, distributeAttr, pipelineAttr, mustProgress,
-      parallelAccesses);
+      isVectorized, parallelAccesses);
 }
 
 LoopAnnotationAttr
 LoopAnnotationImporter::translateLoopAnnotation(const llvm::MDNode *node,
                                                 Location loc) {
   if (!node)
     return {};
 

mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp

struct LoopAnnotationConversion {
 
   /// Converts this struct's loop annotation into a corresponding LLVMIR
   /// metadata representation.
   llvm::MDNode *convert();
 
   /// Conversion functions for different payload attribute kinds.
   void addUnitNode(StringRef name);
   void addUnitNode(StringRef name, BoolAttr attr);
+  void addI32NodeWithVal(StringRef name, uint32_t val);
   void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false);
   void convertI32Node(StringRef name, IntegerAttr attr);
   void convertFollowupNode(StringRef name, LoopAnnotationAttr attr);
 
   /// Conversion functions for each for each loop annotation sub-attribute.
   void convertLoopOptions(LoopVectorizeAttr options);
   void convertLoopOptions(LoopInterleaveAttr options);
   void convertLoopOptions(LoopUnrollAttr options);
metadataNodes.push_back(
       llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name)}));
 }
 
 void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) {
   if (attr && attr.getValue())
     addUnitNode(name);
 }
 
+void LoopAnnotationConversion::addI32NodeWithVal(StringRef name, uint32_t val) {
+  llvm::Constant *cstValue = llvm::ConstantInt::get(
+      llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false);
+  metadataNodes.push_back(
+      llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name),
+                              llvm::ConstantAsMetadata::get(cstValue)}));
+}
+
 void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr,
                                                bool negated) {
   if (!attr)
     return;
   bool val = negated ^ attr.getValue();
   llvm::Constant *cstValue = llvm::ConstantInt::getBool(ctx, val);
   metadataNodes.push_back(
       llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name),
                               llvm::ConstantAsMetadata::get(cstValue)}));
 }
 
 void LoopAnnotationConversion::convertI32Node(StringRef name,
                                               IntegerAttr attr) {
   if (!attr)
     return;
-  uint32_t val = attr.getInt();
-  llvm::Constant *cstValue = llvm::ConstantInt::get(
-      llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false);
-  metadataNodes.push_back(
-      llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name),
-                              llvm::ConstantAsMetadata::get(cstValue)}));
+  addI32NodeWithVal(name, attr.getInt());
 }
 
 void LoopAnnotationConversion::convertFollowupNode(StringRef name,
                                                    LoopAnnotationAttr attr) {
   if (!attr)
     return;
 
   llvm::MDNode *node = loopAnnotationTranslation.translate(attr, op);

 void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) {
   if (auto disable = options.getDisable())
     addUnitNode(disable.getValue() ? "llvm.loop.unroll.disable"
                                    : "llvm.loop.unroll.enable");
   convertI32Node("llvm.loop.unroll.count", options.getCount());
   convertBoolNode("llvm.loop.unroll.runtime.disable",
                   options.getRuntimeDisable());
   addUnitNode("llvm.loop.unroll.full", options.getFull());
-  convertFollowupNode("llvm.loop.unroll.followup", options.getFollowup());
+  convertFollowupNode("llvm.loop.unroll.followup_unrolled",
+                      options.getFollowupUnrolled());
   convertFollowupNode("llvm.loop.unroll.followup_remainder",
                       options.getFollowupRemainder());
+  convertFollowupNode("llvm.loop.unroll.followup_all",
+                      options.getFollowupAll());
 }
 
 void LoopAnnotationConversion::convertLoopOptions(
     LoopUnrollAndJamAttr options) {
   if (auto disable = options.getDisable())
     addUnitNode(disable.getValue() ? "llvm.loop.unroll_and_jam.disable"
                                    : "llvm.loop.unroll_and_jam.enable");
   convertI32Node("llvm.loop.unroll_and_jam.count", options.getCount());

 llvm::MDNode *LoopAnnotationConversion::convert() {
 
   // Reserve operand 0 for loop id self reference.
   auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt);
   metadataNodes.push_back(dummy.get());
 
   addUnitNode("llvm.loop.disable_nonforced", attr.getDisableNonforced());
   addUnitNode("llvm.loop.mustprogress", attr.getMustProgress());
+  // "isvectorized" is encoded as an i32 value.
+  if (BoolAttr isVectorized = attr.getIsVectorized())
+    addI32NodeWithVal("llvm.loop.isvectorized", isVectorized.getValue());
 
   if (auto options = attr.getVectorize())
     convertLoopOptions(options);
   if (auto options = attr.getInterleave())
     convertLoopOptions(options);
   if (auto options = attr.getUnroll())
     convertLoopOptions(options);
   if (auto options = attr.getUnrollAndJam())

mlir/test/Dialect/LLVMIR/loop-metadata.mlir

 // RUN: mlir-opt %s | mlir-opt | FileCheck %s
 
 // CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
 #followup = #llvm.loop_annotation<disableNonforced = true>
 
 // CHECK-DAG: #[[VECTORIZE:.*]] = #llvm.loop_vectorize<disable = false, predicateEnable = false, scalableEnable = true, width = 16 : i32, followupVectorized = #[[FOLLOWUP]], followupEpilogue = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
 #vectorize = #llvm.loop_vectorize<
   disable = false, predicateEnable = false, scalableEnable = true, width = 16 : i32,
   followupVectorized = #followup, followupEpilogue = #followup, followupAll = #followup
 >
 
 // CHECK-DAG: #[[INTERLEAVE:.*]] = #llvm.loop_interleave<count = 32 : i32>
 #interleave = #llvm.loop_interleave<count = 32 : i32>
 
-// CHECK-DAG: #[[UNROLL:.*]] = #llvm.loop_unroll<disable = true, count = 32 : i32, runtimeDisable = true, full = false, followup = #[[FOLLOWUP]], followupRemainder = #[[FOLLOWUP]]>
+// CHECK-DAG: #[[UNROLL:.*]] = #llvm.loop_unroll<disable = true, count = 32 : i32, runtimeDisable = true, full = false, followupUnrolled = #[[FOLLOWUP]], followupRemainder = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
 #unroll = #llvm.loop_unroll<
   disable = true, count = 32 : i32, runtimeDisable = true, full = false,
-  followup = #followup, followupRemainder = #followup
+  followupUnrolled = #followup, followupRemainder = #followup, followupAll = #followup
 >
 
 // CHECK-DAG: #[[UNROLL_AND_JAM:.*]] = #llvm.loop_unroll_and_jam<disable = false, count = 16 : i32, followupOuter = #[[FOLLOWUP]], followupInner = #[[FOLLOWUP]], followupRemainderOuter = #[[FOLLOWUP]], followupRemainderInner = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
 #unrollAndJam = #llvm.loop_unroll_and_jam<
   disable = false, count = 16 : i32, followupOuter = #followup, followupInner = #followup,
   followupRemainderOuter = #followup, followupRemainderInner = #followup, followupAll = #followup
 >
 

 // CHECK: #[[LOOP_ANNOT:.*]] = #llvm.loop_annotation<
 // CHECK-DAG: disableNonforced = false
 // CHECK-DAG: mustProgress = true
 // CHECK-DAG: unroll = #[[UNROLL]]
 // CHECK-DAG: unrollAndJam = #[[UNROLL_AND_JAM]]
 // CHECK-DAG: licm = #[[LICM]]
 // CHECK-DAG: distribute = #[[DISTRIBUTE]]
 // CHECK-DAG: pipeline = #[[PIPELINE]]
+// CHECK-DAG: isVectorized = false
 // CHECK-DAG: parallelAccesses = @metadata::@group1, @metadata::@group2>
 #loopMD = #llvm.loop_annotation<disableNonforced = false,
         mustProgress = true,
         vectorize = #vectorize,
         interleave = #interleave,
         unroll = #unroll,
         unrollAndJam = #unrollAndJam,
         licm = #licm,
         distribute = #distribute,
         pipeline = #pipeline,
+        isVectorized = false,
         parallelAccesses = @metadata::@group1, @metadata::@group2>
 
 // CHECK: llvm.func @loop_annotation
 llvm.func @loop_annotation() {
   // CHECK: llvm.br ^bb1 {llvm.loop = #[[LOOP_ANNOT]]
   llvm.br ^bb1 {llvm.loop = #loopMD}
 ^bb1:
   llvm.return
 }
 
 llvm.metadata @metadata {
   llvm.access_group @group1
   llvm.access_group @group2
 }

mlir/test/Target/LLVMIR/Import/metadata-loop.ll

 !1 = !{!5, !3}
 !2 = distinct !{}
 !3 = distinct !{}
 !4 = distinct !{}
 !5 = distinct !{}
 
 ; // -----
 
-; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<disableNonforced = true, mustProgress = true>
+; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<disableNonforced = true, mustProgress = true, isVectorized = true>
 
 ; CHECK-LABEL: @simple
 define void @simple(i64 %n, ptr %A) {
 entry:
 ; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
   br label %end, !llvm.loop !1
 end:
   ret void
 }
 
-!1 = distinct !{!1, !2, !3}
+!1 = distinct !{!1, !2, !3, !4}
 !2 = !{!"llvm.loop.disable_nonforced"}
 !3 = !{!"llvm.loop.mustprogress"}
+!4 = !{!"llvm.loop.isvectorized", i32 1}
 
 ; // -----
 
 ; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
 ; CHECK-DAG: #[[VECTORIZE_ATTR:.*]] = #llvm.loop_vectorize<disable = false, predicateEnable = true, scalableEnable = false, width = 16 : i32, followupVectorized = #[[FOLLOWUP]], followupEpilogue = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
 ; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<vectorize = #[[VECTORIZE_ATTR]]>
 
 ; CHECK-LABEL: @vectorize

 }
 
 !1 = distinct !{!1, !2}
 !2 = !{!"llvm.loop.interleave.count", i32 8}
 
 ; // -----
 
 ; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
-; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll<disable = false, count = 16 : i32, runtimeDisable = true, full = true, followup = #[[FOLLOWUP]], followupRemainder = #[[FOLLOWUP]]>
+; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll<disable = false, count = 16 : i32, runtimeDisable = true, full = true, followupUnrolled = #[[FOLLOWUP]], followupRemainder = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
 ; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<unroll = #[[UNROLL_ATTR]]>
 
 ; CHECK-LABEL: @unroll
 define void @unroll(i64 %n, ptr %A) {
 entry:
 ; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
   br label %end, !llvm.loop !1
 end:
   ret void
 }
 
-!1 = distinct !{!1, !2, !3, !4, !5, !6, !7}
+!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8}
 !2 = !{!"llvm.loop.unroll.enable"}
 !3 = !{!"llvm.loop.unroll.count", i32 16}
 !4 = !{!"llvm.loop.unroll.runtime.disable"}
 !5 = !{!"llvm.loop.unroll.full"}
-!6 = !{!"llvm.loop.unroll.followup", !8}
-!7 = !{!"llvm.loop.unroll.followup_remainder", !8}
+!6 = !{!"llvm.loop.unroll.followup_unrolled", !9}
+!7 = !{!"llvm.loop.unroll.followup_remainder", !9}
+!8 = !{!"llvm.loop.unroll.followup_all", !9}
 
-!8 = distinct !{!8, !9}
-!9 = !{!"llvm.loop.disable_nonforced"}
+!9 = distinct !{!9, !10}
+!10 = !{!"llvm.loop.disable_nonforced"}
 
 ; // -----
 
 ; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll<disable = true>
 ; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<unroll = #[[UNROLL_ATTR]]>
 
 ; CHECK-LABEL: @unroll_disable
 define void @unroll_disable(i64 %n, ptr %A) {

mlir/test/Target/LLVMIR/loop-metadata.mlir

^bb1:
   llvm.return
 }
 
 // CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}}
 // CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.mustprogress"}
 
 // -----
 
+// CHECK-LABEL: @isvectorized
+llvm.func @isvectorized() {
+  // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+  llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<isVectorized = true>}
+^bb1:
+  llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}}
+// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.isvectorized", i32 1}
+
+// -----
 
 #followup = #llvm.loop_annotation<disableNonforced = true>
 
 // CHECK-LABEL: @vectorizeOptions
 llvm.func @vectorizeOptions() {
   // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
   llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<vectorize = <
     disable = false, predicateEnable = true, scalableEnable = false, width = 16 : i32, 

 
 #followup = #llvm.loop_annotation<disableNonforced = true>
 
 // CHECK-LABEL: @unrollOptions
 llvm.func @unrollOptions() {
   // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
   llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unroll = <
     disable = true, count = 64 : i32, runtimeDisable = false, full = false,
-    followup = #followup, followupRemainder = #followup>
+    followupUnrolled = #followup, followupRemainder = #followup, followupAll = #followup>
   >}
 ^bb1:
   llvm.return
 }
 
 // CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"}
 // CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]}
-// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} 
+// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
 // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.disable"}
 // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.count", i32 64}
 // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.runtime.disable", i1 false}
-// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_unrolled", ![[FOLLOWUP]]}
 // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_remainder", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_all", ![[FOLLOWUP]]}
 
 // -----
 
 // CHECK-LABEL: @unrollOptions2
 llvm.func @unrollOptions2() {
   // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
   llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unroll = <disable = false, full = true>>}
 ^bb1: