[Matrix] Add may-contain-matrix-intrinsics attribute, unconditionally run lowering

[fhahn]

Summary

This patch adds a new may-contain-matrix-intrinsics attribute to allow
frontends to mark a functions as requiring matrix lowering. This allows
us to run the LowerMatrixIntrinsics pass unconditionally, with it being
a no-op for functions without that attribute.

The main reason for introducing the attribute is to avoid doing an
unnecessary pass over each function in the lowering pass. This approach
should also works seamlessly with LTO combining modules built with and
without matrix support enabled.

This patch also extends the IR verifier to reject functions that contain
matrix intrinsics but are not marked as may-contain-matrix-intrinsics.

llvm/docs/LangRef.rst

 
 Matrix Intrinsics
 -----------------
 
 Operations on matrixes requiring shape information (like number of rows/columns
 or the memory layout) can be expressed using the matrix intrinsics. Matrixes are
 embedded in a flat vector and the intrinsics take the dimensions as arguments.
 Currently column-major layout is assumed. The intrinsics support both integer
-and floating point matrixes.
+and floating point matrixes. Functions containing calls to matrix intrinsics
+must have the 'may-contain-matrix-intrinsics' attribute set to true.
 
 
 '``llvm.matrix.transpose.*``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Syntax:
 """""""
 

llvm/include/llvm/IR/Attributes.td

 /// Target-independent string attributes.
 def LessPreciseFPMAD : StrBoolAttr<"less-precise-fpmad">;
 def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">;
 def NoNansFPMath : StrBoolAttr<"no-nans-fp-math">;
 def UnsafeFPMath : StrBoolAttr<"unsafe-fp-math">;
 def NoJumpTables : StrBoolAttr<"no-jump-tables">;
 def NoInlineLineTables : StrBoolAttr<"no-inline-line-tables">;
 def ProfileSampleAccurate : StrBoolAttr<"profile-sample-accurate">;
+def MayContainMatrixIntrinsics : StrBoolAttr<"may-contain-matrix-intrinsics">;
 
 class CompatRule<string F> {
   // The name of the function called to check the attribute of the caller and
   // callee and decide whether inlining should be allowed. The function's
   // signature must match "bool(const Function&, const Function &)", where the
   // first parameter is the reference to the caller and the second parameter is
   // the reference to the callee. It must return false if the attributes of the
   // caller and callee are incompatible, and true otherwise.

 def : MergeRule<"setAND<LessPreciseFPMADAttr>">;
 def : MergeRule<"setAND<NoInfsFPMathAttr>">;
 def : MergeRule<"setAND<NoNansFPMathAttr>">;
 def : MergeRule<"setAND<UnsafeFPMathAttr>">;
 def : MergeRule<"setOR<NoImplicitFloatAttr>">;
 def : MergeRule<"setOR<NoJumpTablesAttr>">;
 def : MergeRule<"setOR<ProfileSampleAccurateAttr>">;
 def : MergeRule<"setOR<SpeculativeLoadHardeningAttr>">;
+def : MergeRule<"setOR<MayContainMatrixIntrinsicsAttr>">;
 def : MergeRule<"adjustCallerSSPLevel">;
 def : MergeRule<"adjustCallerStackProbes">;
 def : MergeRule<"adjustCallerStackProbeSize">;
 def : MergeRule<"adjustMinLegalVectorWidth">;
 def : MergeRule<"adjustNullPointerValidAttr">;

llvm/lib/IR/Verifier.cpp

case Intrinsic::llrint: {
     break;
   }
   case Intrinsic::bswap: {
     Type *Ty = Call.getType();
     unsigned Size = Ty->getScalarSizeInBits();
     Assert(Size % 16 == 0, "bswap must be an even number of bytes", &Call);
     break;
   }
+  case Intrinsic::matrix_multiply:
+  case Intrinsic::matrix_transpose:
+  case Intrinsic::matrix_columnwise_load:
+  case Intrinsic::matrix_columnwise_store: {
+    Function *ParentFn = Call.getParent()->getParent();
+    Assert(ParentFn->getFnAttribute("may-contain-matrix-intrinsics")
+                   .getValueAsString() == "true",
+           "functions with matrix intrinsics must have the "
+           "may-contain-matrix-intrinsics attribute");
+    break;
+  }
   };
 }
 
 /// Carefully grab the subprogram from a local scope.
 ///
 /// This carefully grabs the subprogram from a local scope, avoiding the
 /// built-in assertions that would typically fire.
 static DISubprogram *getSubprogram(Metadata *LocalScope) {

llvm/lib/Transforms/IPO/PassManagerBuilder.cpp

cl::opt<bool> FlattenedProfileUsed(
     "flattened-profile-used", cl::init(false), cl::Hidden,
     cl::desc("Indicate the sample profile being used is flattened, i.e., "
              "no inline hierachy exists in the profile. "));
 
 cl::opt<bool> EnableOrderFileInstrumentation(
     "enable-order-file-instrumentation", cl::init(false), cl::Hidden,
     cl::desc("Enable order file instrumentation (default = off)"));
 
-static cl::opt<bool>
-    EnableMatrix("enable-matrix", cl::init(false), cl::Hidden,
-                 cl::desc("Enable lowering of the matrix intrinsics"));
-
 PassManagerBuilder::PassManagerBuilder() {
     OptLevel = 2;
     SizeLevel = 0;
     LibraryInfo = nullptr;
     Inliner = nullptr;
     DisableUnrollLoops = false;
     SLPVectorize = RunSLPVectorization;
     LoopVectorize = EnableLoopVectorization;
void PassManagerBuilder::populateModulePassManager(
   // this to work. Fortunately, it is trivial to preserve AliasAnalysis
   // (doing nothing preserves it as it is required to be conservatively
   // correct in the face of IR changes).
   MPM.add(createGlobalsAAWrapperPass());
 
   MPM.add(createFloat2IntPass());
   MPM.add(createLowerConstantIntrinsicsPass());
 
-  if (EnableMatrix) {
-    MPM.add(createLowerMatrixIntrinsicsPass());
-    // CSE the pointer arithmetic of the column vectors.  This allows alias
-    // analysis to establish no-aliasing between loads and stores of different
-    // columns of the same matrix.
-    MPM.add(createEarlyCSEPass(false));
-  }
 
   addExtensionsToPM(EP_VectorizerStart, MPM);
 
   // Re-rotate loops in all our loop nests. These may have fallout out of
   // rotated form due to GVN or other transformations, and the vectorizer relies
   // on the rotated form. Disable header duplication at -Oz.
   MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
 

llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp

bool Visit() {
     if (ORE) {
       RemarkGenerator RemarkGen(Inst2ColumnMatrix, *ORE, Func);
       RemarkGen.emitRemarks();
     }
 
     for (Instruction *Inst : reverse(ToRemove))
       Inst->eraseFromParent();
 
+    // All matrix intrinsics have been removed.
+    Func.removeFnAttr("may-contain-matrix-intrinsics");
     return Changed;
   }
 
   LoadInst *createColumnLoad(Value *ColumnPtr, Type *EltType,
                              IRBuilder<> &Builder) {
     return Builder.CreateAlignedLoad(
         ColumnPtr, Align(DL.getABITypeAlignment(EltType)), "col.load");
   }
linearize(Value *L,
       ExprLinearizer Lin(DL, Inst2ColumnMatrix, Shared, ExprsInSubprogram, L);
       Lin.linearizeExpr(L, 0, false, false);
       return Lin.getResult();
     }
   };
 };
 } // namespace
 
+static bool shouldRunMatrixLowering(Function &F) {
+  return F.getFnAttribute("may-contain-matrix-intrinsics").getValueAsString() ==
+         "true";
+}
+
 PreservedAnalyses LowerMatrixIntrinsicsPass::run(Function &F,
                                                  FunctionAnalysisManager &AM) {
   auto &TTI = AM.getResult<TargetIRAnalysis>(F);
   auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
   LowerMatrixIntrinsics LMT(F, TTI, &ORE);
   if (LMT.Visit()) {
     PreservedAnalyses PA;
     PA.preserveSet<CFGAnalyses>();
public:
   static char ID;
 
   LowerMatrixIntrinsicsLegacyPass() : FunctionPass(ID) {
     initializeLowerMatrixIntrinsicsLegacyPassPass(
         *PassRegistry::getPassRegistry());
   }
 
   bool runOnFunction(Function &F) override {
+    if (!shouldRunMatrixLowering(F))
+      return false;
     auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
     auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
     LowerMatrixIntrinsics LMT(F, TTI, &ORE);
     bool C = LMT.Visit();
     return C;
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {

llvm/test/Other/opt-O2-pipeline.ll

 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
+; CHECK-NEXT:       Optimization Remark Emitter
+; CHECK-NEXT:       Lower the matrix intrinsics
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       LCSSA Verifier
 ; CHECK-NEXT:       Loop-Closed SSA Form Pass
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Scalar Evolution Analysis
 ; CHECK-NEXT:       Loop Pass Manager
 ; CHECK-NEXT:         Rotate Loops

llvm/test/Other/opt-O3-pipeline.ll

 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
+; CHECK-NEXT:       Optimization Remark Emitter
+; CHECK-NEXT:       Lower the matrix intrinsics
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       LCSSA Verifier
 ; CHECK-NEXT:       Loop-Closed SSA Form Pass
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Scalar Evolution Analysis
 ; CHECK-NEXT:       Loop Pass Manager
 ; CHECK-NEXT:         Rotate Loops

llvm/test/Other/opt-Os-pipeline.ll

 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Lower constant intrinsics
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
+; CHECK-NEXT:       Lazy Branch Probability Analysis
+; CHECK-NEXT:       Lazy Block Frequency Analysis
+; CHECK-NEXT:       Optimization Remark Emitter
+; CHECK-NEXT:       Lower the matrix intrinsics
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       LCSSA Verifier
 ; CHECK-NEXT:       Loop-Closed SSA Form Pass
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Scalar Evolution Analysis
 ; CHECK-NEXT:       Loop Pass Manager
 ; CHECK-NEXT:         Rotate Loops

llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-define void @transpose_multiply(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) {
+define void @transpose_multiply(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) #0 {
 ; CHECK-LABEL: @transpose_multiply(
 ; CHECK-NEXT:  entry:
 
 ; Load columns of input matrixes %A and %B.
 
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double*
 ; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST]], align 8
entry:
   %c = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
   store <9 x double> %c, <9 x double>* %C.Ptr
   ret void
 }
 
 declare <9 x double> @llvm.matrix.transpose(<9 x double>, i32, i32)
 declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32, i32, i32)
 
-define void @transpose_multiply_add(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) {
+define void @transpose_multiply_add(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) #0 {
 ; CHECK-LABEL: @transpose_multiply_add(
 ; CHECK-NEXT:  entry:
 
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double*
 ; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[COL_CAST]], align 8
 ; CHECK-NEXT:    [[COL_GEP:%.*]] = getelementptr double, double* [[TMP0]], i32 3
 ; CHECK-NEXT:    [[COL_CAST1:%.*]] = bitcast double* [[COL_GEP]] to <3 x double>*
entry:
   %a.trans  = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
   %mult = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
   %c = load <9 x double>, <9 x double>* %C.Ptr
   %res = fadd <9 x double> %c, %mult
 
   store <9 x double> %res, <9 x double>* %C.Ptr
   ret void
 }
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/lowering-drops-attr.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+; Check that the running -lower-matrix-intrinsics removes the
+; "may-contain-matrix-intrinsics". Because it's the only attribute for
+; @transpose, the attribute reference will be dropped from the definition.
+
+define <8 x i32> @transpose(<8 x i32> %a) #0 {
+; CHECK-LABEL: define <8 x i32> @transpose(<8 x i32> %a) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP6]], i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> undef, i32 [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP10]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP12]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[TMP16]]
+;
+entry:
+  %c  = call <8 x i32> @llvm.matrix.transpose(<8 x i32> %a, i32 2, i32 4)
+  ret <8 x i32> %c
+}
+
+declare <8 x i32> @llvm.matrix.transpose(<8 x i32>, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
-define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
+define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) #0 {
 ; CHECK-LABEL: @multiply_2x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x double> [[B]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0

 ; CHECK-NEXT:    ret <4 x double> [[TMP24]]
 ;
 entry:
   %c = call contract <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   ret <4 x double> %c
 }
 
 declare <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double>, <4 x double>, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -matrix-allow-contract -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -matrix-allow-contract -S < %s | FileCheck %s
 
 
-define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
+define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) #0 {
 ; CHECK-LABEL: @multiply_2x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x double> [[B]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0

 ; CHECK-NEXT:    ret <4 x double> [[TMP24]]
 ;
 entry:
   %c = call <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   ret <4 x double> %c
 }
 
 declare <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double>, <4 x double>, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
-define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
+define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) #0 {
 ; CHECK-LABEL: @multiply_2x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x double> [[A]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x double> [[B]], <4 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0

 ;
 entry:
   %c = call <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   ret <4 x double> %c
 }
 
 declare <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v4f64(<4 x double>, <4 x double>, i32, i32, i32)
 
-define <4 x double> @multiply_1x2(<2 x double> %a, <2 x double> %b) {
+define <4 x double> @multiply_1x2(<2 x double> %a, <2 x double> %b) #0 {
 ; CHECK-LABEL: @multiply_1x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <2 x double> [[B]], <2 x double> undef, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x double> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i32 0

 ;
 entry:
   %c = call <4 x double> @llvm.matrix.multiply.v4f64.v2f64.v2f64(<2 x double> %a, <2 x double> %b, i32 2, i32 1, i32 2)
   ret <4 x double> %c
 }
 
 declare <4 x double> @llvm.matrix.multiply.v4f64.v2f64.v2f64(<2 x double>, <2 x double>, i32, i32, i32)
 
-define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
+define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) #0 {
 ; CHECK-LABEL: @multiply_2x3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x double> [[A:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x double> [[A]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <6 x double> [[B:%.*]], <6 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <6 x double> [[B]], <6 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <6 x double> [[B]], <6 x double> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> undef, <1 x i32> zeroinitializer

 ; CHECK-NEXT:    ret <9 x double> [[TMP65]]
 ;
 entry:
   %c = call <9 x double> @llvm.matrix.multiply.v6f64.v6f64.v6f64(<6 x double> %a, <6 x double> %b, i32 3, i32 2, i32 3)
   ret <9 x double> %c
 }
 
 declare <9 x double> @llvm.matrix.multiply.v6f64.v6f64.v6f64(<6 x double>, <6 x double>, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
-define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
+define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) #0 {
 ; CHECK-LABEL: @multiply_2x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0

 ; CHECK-NEXT:    ret <4 x float> [[TMP24]]
 ;
 entry:
   %c = call contract <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
   ret <4 x float> %c
 }
 
 declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -matrix-allow-contract -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -matrix-allow-contract -S < %s | FileCheck %s
 
 
-define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
+define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) #0 {
 ; CHECK-LABEL: @multiply_2x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0

 ; CHECK-NEXT:    ret <4 x float> [[TMP24]]
 ;
 entry:
   %c = call contract <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
   ret <4 x float> %c
 }
 
 declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
-define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
+define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) #0 {
 ; CHECK-LABEL: @multiply_2x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0

 ;
 entry:
   %c = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
   ret <4 x float> %c
 }
 
 declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
 
-define <4 x float> @multiply_1x2(<2 x float> %a, <2 x float> %b) {
+define <4 x float> @multiply_1x2(<2 x float> %a, <2 x float> %b) #0 {
 ; CHECK-LABEL: @multiply_1x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <2 x float> [[A:%.*]], <2 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <2 x float> [[B:%.*]], <2 x float> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <2 x float> [[B]], <2 x float> undef, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x float> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x float> undef, float [[TMP0]], i32 0

 ;
 entry:
   %c = call <4 x float> @llvm.matrix.multiply.v4f32.v2f32.v2f32(<2 x float> %a, <2 x float> %b, i32 2, i32 1, i32 2)
   ret <4 x float> %c
 }
 
 declare <4 x float> @llvm.matrix.multiply.v4f32.v2f32.v2f32(<2 x float>, <2 x float>, i32, i32, i32)
 
-define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
+define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) #0 {
 ; CHECK-LABEL: @multiply_2x3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x float> [[A:%.*]], <6 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x float> [[A]], <6 x float> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <6 x float> [[B:%.*]], <6 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <6 x float> [[B]], <6 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <6 x float> [[B]], <6 x float> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> undef, <1 x i32> zeroinitializer

 ; CHECK-NEXT:    ret <9 x float> [[TMP65]]
 ;
 entry:
   %c = call <9 x float> @llvm.matrix.multiply.v6f32.v6f32.v6f32(<6 x float> %a, <6 x float> %b, i32 3, i32 2, i32 3)
   ret <9 x float> %c
 }
 
 declare <9 x float> @llvm.matrix.multiply.v6f32.v6f32.v6f32(<6 x float>, <6 x float>, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
-define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
+define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) #0 {
 ; CHECK-LABEL: @multiply_2x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0

 ;
 entry:
   %c = call <4 x i32> @llvm.matrix.multiply.v4i32.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 2, i32 2, i32 2)
   ret <4 x i32> %c
 }
 
 declare <4 x i32> @llvm.matrix.multiply.v4i32.v4i32.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32)
 
-define <4 x i32> @multiply_1x2(<2 x i32> %a, <2 x i32> %b) {
+define <4 x i32> @multiply_1x2(<2 x i32> %a, <2 x i32> %b) #0 {
 ; CHECK-LABEL: @multiply_1x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <2 x i32> [[A:%.*]], <2 x i32> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <2 x i32> [[B:%.*]], <2 x i32> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> undef, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> undef, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x i32> undef, i32 [[TMP0]], i32 0

 ;
 entry:
   %c = call <4 x i32> @llvm.matrix.multiply.v4i32.v2i32.v2i32(<2 x i32> %a, <2 x i32> %b, i32 2, i32 1, i32 2)
   ret <4 x i32> %c
 }
 
 declare <4 x i32> @llvm.matrix.multiply.v4i32.v2i32.v2i32(<2 x i32>, <2 x i32>, i32, i32, i32)
 
-define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
+define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) #0 {
 ; CHECK-LABEL: @multiply_2x3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x i32> [[A:%.*]], <6 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x i32> [[A]], <6 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <6 x i32> [[B:%.*]], <6 x i32> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <6 x i32> [[B]], <6 x i32> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <6 x i32> [[B]], <6 x i32> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> undef, <1 x i32> zeroinitializer

 ; CHECK-NEXT:    ret <9 x i32> [[TMP65]]
 ;
 entry:
   %c = call <9 x i32> @llvm.matrix.multiply.v6i32.v6i32.v6i32(<6 x i32> %a, <6 x i32> %b, i32 3, i32 2, i32 3)
   ret <9 x i32> %c
 }
 
 declare <9 x i32> @llvm.matrix.multiply.v6i32.v6i32.v6i32(<6 x i32>, <6 x i32>, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-define <8 x double> @fadd_transpose(<8 x double> %a, <8 x double> %b) {
+define <8 x double> @fadd_transpose(<8 x double> %a, <8 x double> %b) #0 {
 ; CHECK-LABEL: @fadd_transpose(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <8 x double> [[B]], <8 x double> undef, <2 x i32> <i32 2, i32 3>

 ; CHECK-NEXT:    ret <8 x double> [[TMP20]]
 ;
 entry:
   %add = fadd <8 x double> %a, %b
   %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %add, i32 2, i32 4)
   ret <8 x double> %c
 }
 
-define <8 x double> @load_fadd_transpose(<8 x double>* %A.Ptr, <8 x double> %b) {
+define <8 x double> @load_fadd_transpose(<8 x double>* %A.Ptr, <8 x double> %b) #0 {
 ; CHECK-LABEL: @load_fadd_transpose(
 ; CHECK-NEXT:  entry:
 
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x double>* [[A_PTR:%.*]] to double*
 ; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP0]] to <2 x double>*
 ; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[COL_CAST]], align 8
 ; CHECK-NEXT:    [[COL_GEP:%.*]] = getelementptr double, double* [[TMP0]], i32 2
 ; CHECK-NEXT:    [[COL_CAST1:%.*]] = bitcast double* [[COL_GEP]] to <2 x double>*

 entry:
   %a = load <8 x double>, <8 x double>* %A.Ptr
   %add = fadd <8 x double> %a, %b
   %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %add, i32 2, i32 4)
   ret <8 x double> %c
 }
 
 declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 ; Check that we we use flattened vectors for PHI operands and extract the columns afterwards.
-define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B, <9 x double> %C) {
+define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B, <9 x double> %C) #0 {
 ; CHECK-LABEL: @unsupported_phi(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 
 ; CHECK-LABEL: if.then:
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>

 
 if.end:                                        ; preds = %if.then, %if.else
   %merge = phi <9 x double> [ %A.trans, %if.then], [ %B.trans, %if.else ]
   %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %C, <9 x double> %merge, i32 3, i32 3, i32 3)
   ret <9 x double> %res
 }
 
 ; Make sure we use a flattened vector when calling @foo and the use its flat vector result properly.
-define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B) {
+define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B) #0 {
 ; CHECK-LABEL: @unsupported_call(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <3 x double> undef, double [[TMP1]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <3 x double> [[TMP2]], double [[TMP3]], i64 1
;
   %A.foo = call <9 x double> @foo(<9 x double> %A.trans)
   %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %B, <9 x double> %A.foo, i32 3, i32 3, i32 3)
   ret <9 x double> %res
 }
 
 declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32 immarg, i32 immarg, i32 immarg)
 declare <9 x double> @llvm.matrix.transpose.v9f64(<9 x double>, i32 immarg, i32 immarg)
 declare <9 x double> @foo(<9 x double>)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 ; Check that we do not emit shufflevectors to flatten the result of the
 ; transpose and store the columns directly.
-define void @transpose_store(<8 x double> %a, <8 x double>* %Ptr) {
+define void @transpose_store(<8 x double> %a, <8 x double>* %Ptr) #0 {
 ; CHECK-LABEL: @transpose_store(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0

 entry:
   %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
   store <8 x double> %c, <8 x double>* %Ptr
   ret void
 }
 
 declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32)
 
-define <8 x double> @transpose_fadd(<8 x double> %a) {
+define <8 x double> @transpose_fadd(<8 x double> %a) #0 {
 ; CHECK-LABEL: @transpose_fadd(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0

 ; CHECK-NEXT:    ret <8 x double> [[TMP18]]
 ;
 entry:
   %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
   %res = fadd <8 x double> %c, %a
   ret <8 x double> %res
 }
 
-define <8 x double> @transpose_fmul(<8 x double> %a) {
+define <8 x double> @transpose_fmul(<8 x double> %a) #0 {
 ; CHECK-LABEL: @transpose_fmul(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0

 ; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> [[TMP17]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x double> [[TMP18]]
 ;
 entry:
   %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
   %res = fmul <8 x double> %c, %a
   ret <8 x double> %res
 }
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 ; Currently we only lower stores with shape information, but need to embed the
 ; matrix in a flat vector for function calls and returns.
-define <8 x double> @strided_load_4x4(<8 x double> %in, <8 x double>* %Ptr) {
+define <8 x double> @strided_load_4x4(<8 x double> %in, <8 x double>* %Ptr) #0 {
 ; CHECK-LABEL: @strided_load_4x4(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[IN:%.*]], <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[IN]], <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[TMP1]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP2]], double [[TMP3]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[SPLIT]], i64 1
;
   store <8 x double> %transposed, <8 x double>* %Ptr
   call void @foo(<8 x double> %transposed)
   ret <8 x double> %transposed
 }
 
 declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32)
 
 declare void @foo(<8 x double>)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
 ; Make sure we propagate in multiple iterations. First, we back-propagate the
 ; shape information from the transpose to %A, in the next iteration we
 ; forward-propagate it to %Mul, and then back to %B.
-define <16 x double> @backpropagation_iterations(<16 x double>* %A.Ptr, <16 x double>* %B.Ptr) {
+define <16 x double> @backpropagation_iterations(<16 x double>* %A.Ptr, <16 x double>* %B.Ptr) #0 {
 ; CHECK-LABEL: @backpropagation_iterations(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x double>* [[A_PTR:%.*]] to double*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[TMP1]] to <4 x double>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x double>, <4 x double>* [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, double* [[TMP1]], i32 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[TMP5]] to <4 x double>*
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x double>, <4 x double>* [[TMP6]], align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr double, double* [[TMP1]], i32 8
;
   %A.trans = tail call <16 x double> @llvm.matrix.transpose.v16f64(<16 x double> %A, i32 4, i32 4)
   %B = load <16 x double>, <16 x double>* %B.Ptr
   %Mul = fmul <16 x double> %A, %B
   ret <16 x double> %Mul
 }
 
 declare <16 x double> @llvm.matrix.multiply.v16f64.v16f64.v16f64(<16 x double>, <16 x double>, i32 immarg, i32 immarg, i32 immarg)
 declare <16 x double> @llvm.matrix.transpose.v16f64(<16 x double>, i32 immarg, i32 immarg)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/remarks-inlining.ll

 ; CHECK-LABEL: remark: store.h:66:11: Lowered with 1 stores, 0 loads, 0 compute ops
 ; CHECK-NEXT:  store(
 ; CHECK-NEXT:  scalar,
 ; CHECK-NEXT:  addr %D)
 
 ; CHECK-LABEL: remark: transpose.h:13:11: Lowered with 0 stores, 0 loads, 8 compute ops
 ; CHECK-NEXT:  transpose.1x2.float(transpose.2x1.float(addr %D))
 
-define void @toplevel(<15 x double>* %A, <15 x double>* %B, <15 x double>* %C, <2 x float>* %D) !dbg !16 {
+define void @toplevel(<15 x double>* %A, <15 x double>* %B, <15 x double>* %C, <2 x float>* %D) #0 !dbg !16 {
 entry:
   %a = load <15 x double>, <15 x double> *%A, align 16, !dbg !3791
   %b = call <15 x double> @llvm.matrix.columnwise.load(<15 x double>* %B, i32 5, i32 3, i32 5), !dbg !3793
   %c  = fadd <15 x double> %a, %b, !dbg !100
   store <15 x double> %c, <15 x double> *%C, align 16, !dbg !102
 
   %load = load <2 x float>, <2 x float>* %D, !dbg !104
   %t1 = call <2 x float> @llvm.matrix.transpose(<2 x float> %load, i32 2, i32 1), !dbg !106

 !104 = !DILocation(line: 41, column: 11, scope: !5, inlinedAt: !101)
 !105 = !DILocation(line: 500, column: 11, scope: !16)
 
 !106 = !DILocation(line: 13, column: 11, scope: !25, inlinedAt: !101)
 !107 = !DILocation(line: 510, column: 11, scope: !16)
 
 !108 = !DILocation(line: 66, column: 11, scope: !23, inlinedAt: !109)
 !109 = !DILocation(line: 510, column: 0, scope: !16)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll

 ; STDERR-NEXT:   fsub(
 ; STDERR-NEXT:    columnwise.load.4x15.double(addr %arg2, 20),
 ; STDERR-NEXT:    multiply.4x2.2x15.double(
 ; STDERR-NEXT:     shared with remark at line 35 column 71 (transpose.2x4.double(columnwise.load.2x4.double(addr %arg1,
 ; STDERR-NEXT:       scalar)),
 ; STDERR-NEXT:     columnwise.load.2x15.double(addr %arg3, scalar))),
 ; STDERR-NEXT:   addr %arg2,
 ; STDERR-NEXT:   10)
-define void @test_2leafs(double* %arg1, double* %arg2, double* %arg3, i32 %stride, i32 %offset) !dbg !8 {
+define void @test_2leafs(double* %arg1, double* %arg2, double* %arg3, i32 %stride, i32 %offset) #0 !dbg !8 {
 bb:
   %shared.load = tail call <8 x double> @llvm.matrix.columnwise.load.v8f64.p0f64(double* %arg1, i32 %stride, i32 2, i32 4), !dbg !10, !noalias !10
   %shared.load.2 = tail call <30 x double> @llvm.matrix.columnwise.load.v30f64.p0f64(double* %arg3, i32 %stride, i32 2, i32 15), !dbg !10, !noalias !10
   %tmp17 = tail call <8 x double> @llvm.matrix.transpose.v8f64(<8 x double> %shared.load, i32 2, i32 4), !dbg !10
   tail call void @llvm.matrix.columnwise.store.v8f64.p0f64(<8 x double> %tmp17, double* %arg3, i32 10, i32 4, i32 2), !dbg !10
   %tmp18 = tail call <60 x double> @llvm.matrix.columnwise.load.v60f64.p0f64(double* %arg2, i32 20, i32 4, i32 15), !dbg !11
   %tmp48 = tail call <60 x double> @llvm.matrix.multiply.v60f64.v8f64.v30f64(<8 x double> %tmp17, <30 x double> %shared.load.2, i32 4, i32 2, i32 15), !dbg !11
   %tmp49 = fsub <60 x double> %tmp18, %tmp48, !dbg !11
   tail call void @llvm.matrix.columnwise.store.v60f64.p0f64(<60 x double> %tmp49, double* %arg2, i32 10, i32 4, i32 15), !dbg !11
   ret void
 }
 
 declare <8 x double> @llvm.matrix.transpose.v8f64(<8 x double>, i32 immarg, i32 immarg)
 declare <8 x double> @llvm.matrix.columnwise.load.v8f64.p0f64(double*, i32, i32 immarg, i32 immarg)
 declare <30 x double> @llvm.matrix.columnwise.load.v30f64.p0f64(double*, i32, i32 immarg, i32 immarg)
 declare <60 x double> @llvm.matrix.columnwise.load.v60f64.p0f64(double*, i32, i32 immarg, i32 immarg)
 declare void @llvm.matrix.columnwise.store.v60f64.p0f64(<60 x double>, double* writeonly, i32, i32 immarg, i32 immarg)
 declare void @llvm.matrix.columnwise.store.v8f64.p0f64(<8 x double>, double* writeonly, i32, i32 immarg, i32 immarg)
 declare <60 x double> @llvm.matrix.multiply.v60f64.v8f64.v30f64(<8 x double>, <30 x double>, i32 immarg, i32 immarg, i32 immarg)
 
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }
+
 !llvm.module.flags = !{!0, !1, !2, !3}
 !llvm.dbg.cu = !{!4}
 !llvm.ident = !{!7}
 
 !0 = !{i32 2, !"SDK Version", [2 x i32] [i32 13, i32 0]}
 !1 = !{i32 2, !"Debug Info Version", i32 3}
 !2 = !{i32 1, !"wchar_size", i32 4}
 !3 = !{i32 7, !"PIC Level", i32 2}

llvm/test/Transforms/LowerMatrixIntrinsics/remarks.ll

 ; REQUIRES: aarch64-registered-target
 
 ; This test needs to be target specific due to the cost estimate in the output.
 
 ; RUN: opt -lower-matrix-intrinsics -pass-remarks=lower-matrix-intrinsics -mtriple=arm64-apple-iphoneos < %s 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: remark: test.h:40:20: Lowered with 6 stores, 6 loads, 24 compute ops
 ; CHECK-NEXT: store(
 ; CHECK-NEXT:  transpose.2x6.double(load(addr %A)),
 ; CHECK-NEXT:  addr %B)
-define void @transpose(<12 x double>* %A, <12 x double>* %B) !dbg !23 {
+define void @transpose(<12 x double>* %A, <12 x double>* %B) #0 !dbg !23 {
   %load = load <12 x double>, <12 x double>* %A, !dbg !24
   %t = call <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double> %load, i32 2, i32 6), !dbg !24
   store <12 x double> %t, <12 x double>* %B, !dbg !24
   ret void
 }
 
 declare <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double>, i32, i32)
 
 
 ; CHECK-LABEL: remark: test.h:50:20: Lowered with 2 stores, 12 loads, 22 compute ops
 ; CHECK-NEXT:  store(
 ; CHECK-NEXT:   multiply.2x6.6x2.double(
 ; CHECK-NEXT:    load(addr %A),
 ; CHECK-NEXT:    load(addr %B)),
 ; CHECK-NEXT:   addr %C)
-define void @multiply(<12 x double>* %A, <12 x double>* %B, <4 x double>* %C) !dbg !25 {
+define void @multiply(<12 x double>* %A, <12 x double>* %B, <4 x double>* %C) #0 !dbg !25 {
   %A.matrix = load <12 x double>, <12 x double>* %A, !dbg !26
   %B.matrix = load <12 x double>, <12 x double>* %B, !dbg !26
   %t = call <4 x double> @llvm.matrix.multiply(<12 x double> %A.matrix, <12 x double> %B.matrix, i32 2, i32 6, i32 2), !dbg !26
   store <4 x double> %t, <4 x double>* %C, !dbg !26
   ret void
 }
 
 declare <4 x double> @llvm.matrix.multiply(<12 x double>, <12 x double>, i32, i32, i32)
 
 ; CHECK-LABEL: remark: test.h:60:20: Lowered with 6 stores, 6 loads, 0 compute ops
 ; CHECK-NEXT:  store(
 ; CHECK-NEXT:   columnwise.load.3x3.double(addr %A, 5),
 ; CHECK-NEXT:   addr %B)
-define void @columnwise.load(<9 x double>* %A, <9 x double>* %B) !dbg !27 {
+define void @columnwise.load(<9 x double>* %A, <9 x double>* %B) #0 !dbg !27 {
   %A.matrix = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %A, i32 5, i32 3, i32 3), !dbg !28
   store <9 x double> %A.matrix, <9 x double>* %B, !dbg !28
   ret void
 }
 
 declare <9 x double> @llvm.matrix.columnwise.load(<9 x double>*, i32, i32, i32)
 
 ; CHECK-LABEL: remark: test.h:70:20: Lowered with 6 stores, 6 loads, 0 compute ops
 ; CHECK-NEXT:  columnwise.store.3x3.double(
 ; CHECK-NEXT:   columnwise.load.3x3.double(addr %A, 5),
 ; CHECK-NEXT:   addr %B,
 ; CHECK-NEXT:   10)
-define void @columnwise.store(<9 x double>* %A, <9 x double>* %B) !dbg !29 {
+define void @columnwise.store(<9 x double>* %A, <9 x double>* %B) #0 !dbg !29 {
   %A.matrix = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %A, i32 5, i32 3, i32 3), !dbg !30
   call void @llvm.matrix.columnwise.store(<9 x double> %A.matrix, <9 x double>* %B, i32 10, i32 3, i32 3), !dbg !30
   ret void
 }
 
 declare void @llvm.matrix.columnwise.store(<9 x double>, <9 x double>*, i32, i32, i32)
 
 ; CHECK-LABEL: remark: test.h:80:20: Lowered with 6 stores, 6 loads, 12 compute ops
 ; CHECK-NEXT:  columnwise.store.3x3.double(
 ; CHECK-NEXT:   fmul(
 ; CHECK-NEXT:    fadd(
 ; CHECK-NEXT:     columnwise.load.3x3.double(addr %A, 5)
 ; CHECK-NEXT:     (reused) columnwise.load.3x3.double(addr %A, 5)),
 ; CHECK-NEXT:    (reused) columnwise.load.3x3.double(addr %A, 5)),
 ; CHECK-NEXT:   addr %B,
 ; CHECK-NEXT:   10)
 
-define void @binaryops(<9 x double>* %A, <9 x double>* %B) !dbg !31 {
+define void @binaryops(<9 x double>* %A, <9 x double>* %B) #0 !dbg !31 {
   %A.matrix = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %A, i32 5, i32 3, i32 3), !dbg !32
   %R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !32
   %R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !32
   call void @llvm.matrix.columnwise.store(<9 x double> %R2.matrix, <9 x double>* %B, i32 10, i32 3, i32 3), !dbg !32
   ret void
 }
 
 ; CHECK-LABEL: remark: test.h:90:20: Lowered with 6 stores, 6 loads, 12 compute ops
 ; CHECK-NEXT:  columnwise.store.3x3.double(
 ; CHECK-NEXT:   fmul(
 ; CHECK-NEXT:    fadd(
 ; CHECK-NEXT:     columnwise.load.3x3.double(addr %A, 5)
 ; CHECK-NEXT:     (reused) columnwise.load.3x3.double(addr %A, 5)),
 ; CHECK-NEXT:    (reused) columnwise.load.3x3.double(addr %A, 5)),
 ; CHECK-NEXT:   addr %B,
 ; CHECK-NEXT:   10)
 ; CHECK-NEXT:  remark: test.h:90:20: Lowered with 2 stores, 12 loads, 22 compute ops
 ; CHECK-NEXT:  store(
 ; CHECK-NEXT:   multiply.2x6.6x2.double(
 ; CHECK-NEXT:    load(addr %C),
 ; CHECK-NEXT:    load(addr %D)),
 ; CHECK-NEXT:   addr %E)
 
-define void @multiple_expressions(<9 x double>* %A, <9 x double>* %B, <12 x double>* %C, <12 x double>* %D, <4 x double>* %E) !dbg !33 {
+define void @multiple_expressions(<9 x double>* %A, <9 x double>* %B, <12 x double>* %C, <12 x double>* %D, <4 x double>* %E) #0 !dbg !33 {
   %A.matrix = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %A, i32 5, i32 3, i32 3), !dbg !34
   %R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !34
   %R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !34
   call void @llvm.matrix.columnwise.store(<9 x double> %R2.matrix, <9 x double>* %B, i32 10, i32 3, i32 3), !dbg !34
 
   %C.matrix = load <12 x double>, <12 x double>* %C, !dbg !34
   %D.matrix = load <12 x double>, <12 x double>* %D, !dbg !34
   %Mult.matrix = call <4 x double> @llvm.matrix.multiply(<12 x double> %C.matrix, <12 x double> %D.matrix, i32 2, i32 6, i32 2), !dbg !34
   store <4 x double> %Mult.matrix, <4 x double>* %E, !dbg !34
 
   ret void
 }
 
 ; CHECK-LABEL: remark: test.h:100:20: Lowered with 6 stores, 6 loads, 12 compute ops
 ; CHECK-NEXT:  columnwise.store.3x3.double(
 ; CHECK-NEXT:   fmul(
 ; CHECK-NEXT:    fadd(
 ; CHECK-NEXT:     columnwise.load.3x3.double(addr %A, 5)
 ; CHECK-NEXT:     (reused) columnwise.load.3x3.double(addr %A, 5)),
 ; CHECK-NEXT:    (reused) columnwise.load.3x3.double(addr %A, 5)),
 ; CHECK-NEXT:   stack addr %B,
 ; CHECK-NEXT:   10)
-define void @stackaddresses(<9 x double>* %A) !dbg !35 {
+define void @stackaddresses(<9 x double>* %A) #0 !dbg !35 {
   %B = alloca <9 x double>
   %A.matrix = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %A, i32 5, i32 3, i32 3), !dbg !36
   %R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !36
   %R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !36
   call void @llvm.matrix.columnwise.store(<9 x double> %R2.matrix, <9 x double>* %B, i32 10, i32 3, i32 3), !dbg !36
   ret void
 }
 
 ; CHECK-LABEL: remark: test.h:30:20: Lowered with 10 stores, 9 loads, 30 compute ops
 ; CHECK-NEXT:  store(
 ; CHECK-NEXT:   transpose.5x3.double(load(addr %A)),
 ; CHECK-NEXT:   stack addr %s1)
 %S1 = type {<15 x double>*}
-define void @get_underlying_object(%S1* %A) !dbg !21 {
+define void @get_underlying_object(%S1* %A) #0 !dbg !21 {
 entry:
   %s1 = alloca <15 x double>, !dbg !22
   %a1 = getelementptr %S1, %S1* %A, i32 0, i32 0, !dbg !22
   %a2 = load <15 x double>*, <15 x double>** %a1, !dbg !22
   %av = load <15 x double>, <15 x double>* %a2, !dbg !22
 
   %s2 = bitcast <15 x double>* %s1 to i64*, !dbg !22
   %s3 = bitcast i64* %s2 to <15 x double>*, !dbg !22
 
   %t = call <15 x double> @llvm.matrix.transpose.v15f64.v15f64(<15 x double> %av, i32 5, i32 3), !dbg !22
 
   store <15 x double> %t, <15 x double>* %s3, !dbg !22
   ret void
 }
 
 declare <15 x double> @llvm.matrix.transpose.v15f64.v15f64(<15 x double>, i32, i32)
 
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }
+
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "test.h", directory: "/test")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}

llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-define <9 x double> @strided_load_3x3(<9 x double>* %in, i32 %stride) {
+define <9 x double> @strided_load_3x3(<9 x double>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_3x3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <3 x double>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <3 x double>, <3 x double>* [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]

 ;
 entry:
   %load = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %in, i32 %stride, i32 3, i32 3)
   ret <9 x double> %load
 }
 
 declare <9 x double> @llvm.matrix.columnwise.load(<9 x double>*, i32, i32, i32)
 
-define <9 x double> @strided_load_9x1(<9 x double>* %in, i32 %stride) {
+define <9 x double> @strided_load_9x1(<9 x double>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_9x1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <9 x double>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <9 x double>, <9 x double>* [[TMP3]], align 8
 ; CHECK-NEXT:    ret <9 x double> [[TMP4]]
 ;
 entry:
   %load = call <9 x double> @llvm.matrix.columnwise.load(<9 x double>* %in, i32 %stride, i32 9, i32 1)
   ret <9 x double> %load
 }
 
 declare <8 x double> @llvm.matrix.columnwise.load.v8f64(<8 x double>*, i32, i32, i32)
 
-define <8 x double> @strided_load_4x2(<8 x double>* %in, i32 %stride) {
+define <8 x double> @strided_load_4x2(<8 x double>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_4x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x double>* [[IN:%.*]] to double*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <4 x double>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x double>, <4 x double>* [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, double* [[TMP0]], i32 [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[TMP6]] to <4 x double>*
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x double>, <4 x double>* [[TMP7]], align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x double> [[TMP9]]
 ;
 entry:
   %load = call <8 x double> @llvm.matrix.columnwise.load.v8f64(<8 x double>* %in, i32 %stride, i32 4, i32 2)
   ret <8 x double> %load
 }
 
 ; CHECK: declare <9 x double> @llvm.matrix.columnwise.load.v9f64.p0v9f64(<9 x double>*, i32, i32 immarg, i32 immarg) [[READONLY:#[0-9]]]
 
 ; CHECK: declare <8 x double> @llvm.matrix.columnwise.load.v8f64.p0v8f64(<8 x double>*, i32, i32 immarg, i32 immarg) [[READONLY]]
 
 ; CHECK: attributes [[READONLY]] = { argmemonly nounwind readonly willreturn }
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-define <9 x float> @strided_load_3x3(<9 x float>* %in, i32 %stride) {
+define <9 x float> @strided_load_3x3(<9 x float>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_3x3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x float>* [[IN:%.*]] to float*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <3 x float>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <3 x float>, <3 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]

 ;
 entry:
   %load = call <9 x float> @llvm.matrix.columnwise.load(<9 x float>* %in, i32 %stride, i32 3, i32 3)
   ret <9 x float> %load
 }
 
 declare <9 x float> @llvm.matrix.columnwise.load(<9 x float>*, i32, i32, i32)
 
-define <9 x float> @strided_load_9x1(<9 x float>* %in, i32 %stride) {
+define <9 x float> @strided_load_9x1(<9 x float>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_9x1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x float>* [[IN:%.*]] to float*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <9 x float>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <9 x float>, <9 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    ret <9 x float> [[TMP4]]
 ;
 entry:
   %load = call <9 x float> @llvm.matrix.columnwise.load(<9 x float>* %in, i32 %stride, i32 9, i32 1)
   ret <9 x float> %load
 }
 
 declare <8 x float> @llvm.matrix.columnwise.load.v8f32(<8 x float>*, i32, i32, i32)
 
-define <8 x float> @strided_load_4x2(<8 x float>* %in, i32 %stride) {
+define <8 x float> @strided_load_4x2(<8 x float>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_4x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x float>* [[IN:%.*]] to float*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr float, float* [[TMP0]], i32 [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x float>, <4 x float>* [[TMP7]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x float> [[TMP9]]
 ;
 entry:
   %load = call <8 x float> @llvm.matrix.columnwise.load.v8f32(<8 x float>* %in, i32 %stride, i32 4, i32 2)
   ret <8 x float> %load
 }
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-define <9 x i32> @strided_load_3x3(<9 x i32>* %in, i32 %stride) {
+define <9 x i32> @strided_load_3x3(<9 x i32>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_3x3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x i32>* [[IN:%.*]] to i32*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <3 x i32>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <3 x i32>, <3 x i32>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]

 ;
 entry:
   %load = call <9 x i32> @llvm.matrix.columnwise.load(<9 x i32>* %in, i32 %stride, i32 3, i32 3)
   ret <9 x i32> %load
 }
 
 declare <9 x i32> @llvm.matrix.columnwise.load(<9 x i32>*, i32, i32, i32)
 
-define <9 x i32> @strided_load_9x1(<9 x i32>* %in, i32 %stride) {
+define <9 x i32> @strided_load_9x1(<9 x i32>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_9x1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x i32>* [[IN:%.*]] to i32*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <9 x i32>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <9 x i32>, <9 x i32>* [[TMP3]], align 4
 ; CHECK-NEXT:    ret <9 x i32> [[TMP4]]
 ;
 entry:
   %load = call <9 x i32> @llvm.matrix.columnwise.load(<9 x i32>* %in, i32 %stride, i32 9, i32 1)
   ret <9 x i32> %load
 }
 
 declare <8 x i32> @llvm.matrix.columnwise.load.v8i32(<8 x i32>*, i32, i32, i32)
 
-define <8 x i32> @strided_load_4x2(<8 x i32>* %in, i32 %stride) {
+define <8 x i32> @strided_load_4x2(<8 x i32>* %in, i32 %stride) #0 {
 ; CHECK-LABEL: @strided_load_4x2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i32>* [[IN:%.*]] to i32*
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 1, [[STRIDE]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP9]]
 ;
 entry:
   %load = call <8 x i32> @llvm.matrix.columnwise.load.v8i32(<8 x i32>* %in, i32 %stride, i32 4, i32 2)
   ret <8 x i32> %load
 }
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-define void @strided_store_3x2(<6 x double> %in, double* %out) {
+define void @strided_store_3x2(<6 x double> %in, double* %out) #0 {
 ; CHECK-LABEL: @strided_store_3x2(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[OUT:%.*]] to <3 x double>*
 ; CHECK-NEXT:    store <3 x double> [[SPLIT]], <3 x double>* [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[OUT]], i32 5
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <3 x double>*
 ; CHECK-NEXT:    store <3 x double> [[SPLIT1]], <3 x double>* [[TMP3]], align 8
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.matrix.columnwise.store(<6 x double> %in, double* %out, i32 5, i32 3, i32 2)
   ret void
 }
 
-define void @strided_store_3x2_nonconst_stride(<6 x double> %in, i32 %stride, double* %out) {
+define void @strided_store_3x2_nonconst_stride(<6 x double> %in, i32 %stride, double* %out) #0 {
 ; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[OUT:%.*]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <3 x double>*
 ; CHECK-NEXT:    store <3 x double> [[SPLIT]], <3 x double>* [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 1, [[STRIDE]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, double* [[OUT]], i32 [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[TMP5]] to <3 x double>*
 ; CHECK-NEXT:    store <3 x double> [[SPLIT1]], <3 x double>* [[TMP6]], align 8
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.matrix.columnwise.store(<6 x double> %in, double* %out, i32 %stride, i32 3, i32 2)
   ret void
 }
 
 
 declare void @llvm.matrix.columnwise.store(<6 x double>, double*, i32, i32, i32)
 
-define void @strided_store_2x3(<10 x double> %in, double* %out) {
+define void @strided_store_2x3(<10 x double> %in, double* %out) #0 {
 ; CHECK-LABEL: @strided_store_2x3(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <10 x double> [[IN:%.*]], <10 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 8, i32 9>
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[OUT:%.*]] to <2 x double>*
 ; CHECK-NEXT:    store <2 x double> [[SPLIT]], <2 x double>* [[TMP1]], align 8

 
 declare void @llvm.matrix.columnwise.store.v10f64(<10 x double>, double*, i32, i32, i32)
 
 ; CHECK: declare void @llvm.matrix.columnwise.store.v6f64.p0f64(<6 x double>, double* writeonly, i32, i32 immarg, i32 immarg) [[WRITEONLY:#[0-9]]]
 
 ; CHECK: declare void @llvm.matrix.columnwise.store.v10f64.p0f64(<10 x double>, double* writeonly, i32, i32 immarg, i32 immarg) [[WRITEONLY]]
 
 ; CHECK: attributes [[WRITEONLY]] = { argmemonly nounwind willreturn writeonly }
+
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-define void @strided_store_3x2(<6 x float> %in, float* %out) {
+define void @strided_store_3x2(<6 x float> %in, float* %out) #0 {
 ; CHECK-LABEL: @strided_store_3x2(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x float> [[IN:%.*]], <6 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x float> [[IN]], <6 x float> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[OUT:%.*]] to <3 x float>*
 ; CHECK-NEXT:    store <3 x float> [[SPLIT]], <3 x float>* [[TMP1]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[OUT]], i32 5
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <3 x float>*
 ; CHECK-NEXT:    store <3 x float> [[SPLIT1]], <3 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.matrix.columnwise.store(<6 x float> %in, float* %out, i32 5, i32 3, i32 2)
   ret void
 }
 
-define void @strided_store_3x2_nonconst_stride(<6 x float> %in, i32 %stride, float* %out) {
+define void @strided_store_3x2_nonconst_stride(<6 x float> %in, i32 %stride, float* %out) #0 {
 ; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x float> [[IN:%.*]], <6 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x float> [[IN]], <6 x float> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, float* [[OUT:%.*]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <3 x float>*
 ; CHECK-NEXT:    store <3 x float> [[SPLIT]], <3 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 1, [[STRIDE]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr float, float* [[OUT]], i32 [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <3 x float>*
 ; CHECK-NEXT:    store <3 x float> [[SPLIT1]], <3 x float>* [[TMP6]], align 4
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.matrix.columnwise.store(<6 x float> %in, float* %out, i32 %stride, i32 3, i32 2)
   ret void
 }
 
 
 declare void @llvm.matrix.columnwise.store(<6 x float>, float*, i32, i32, i32)
 
-define void @strided_store_2x3(<10 x float> %in, float* %out) {
+define void @strided_store_2x3(<10 x float> %in, float* %out) #0 {
 ; CHECK-LABEL: @strided_store_2x3(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <10 x float> [[IN:%.*]], <10 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> undef, <2 x i32> <i32 8, i32 9>
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[OUT:%.*]] to <2 x float>*
 ; CHECK-NEXT:    store <2 x float> [[SPLIT]], <2 x float>* [[TMP1]], align 4

 ; CHECK-NEXT:    store <2 x float> [[SPLIT4]], <2 x float>* [[TMP9]], align 4
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.matrix.columnwise.store.v10f32(<10 x float> %in, float* %out, i32 4, i32 2, i32 5)
   ret void
 }
 
 declare void @llvm.matrix.columnwise.store.v10f32(<10 x float>, float*, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-define void @strided_store_3x2(<6 x i32> %in, i32* %out) {
+define void @strided_store_3x2(<6 x i32> %in, i32* %out) #0 {
 ; CHECK-LABEL: @strided_store_3x2(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[OUT:%.*]] to <3 x i32>*
 ; CHECK-NEXT:    store <3 x i32> [[SPLIT]], <3 x i32>* [[TMP1]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[OUT]], i32 5
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <3 x i32>*
 ; CHECK-NEXT:    store <3 x i32> [[SPLIT1]], <3 x i32>* [[TMP3]], align 4
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.matrix.columnwise.store(<6 x i32> %in, i32* %out, i32 5, i32 3, i32 2)
   ret void
 }
 
-define void @strided_store_3x2_nonconst_stride(<6 x i32> %in, i32 %stride, i32* %out) {
+define void @strided_store_3x2_nonconst_stride(<6 x i32> %in, i32 %stride, i32* %out) #0 {
 ; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 0, [[STRIDE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <3 x i32>*
 ; CHECK-NEXT:    store <3 x i32> [[SPLIT]], <3 x i32>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 1, [[STRIDE]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, i32* [[OUT]], i32 [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <3 x i32>*
 ; CHECK-NEXT:    store <3 x i32> [[SPLIT1]], <3 x i32>* [[TMP6]], align 4
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.matrix.columnwise.store(<6 x i32> %in, i32* %out, i32 %stride, i32 3, i32 2)
   ret void
 }
 
 
 declare void @llvm.matrix.columnwise.store(<6 x i32>, i32*, i32, i32, i32)
 
-define void @strided_store_2x3(<10 x i32> %in, i32* %out) {
+define void @strided_store_2x3(<10 x i32> %in, i32* %out) #0 {
 ; CHECK-LABEL: @strided_store_2x3(
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <10 x i32> [[IN:%.*]], <10 x i32> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> undef, <2 x i32> <i32 8, i32 9>
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[OUT:%.*]] to <2 x i32>*
 ; CHECK-NEXT:    store <2 x i32> [[SPLIT]], <2 x i32>* [[TMP1]], align 4

 ; CHECK-NEXT:    store <2 x i32> [[SPLIT4]], <2 x i32>* [[TMP9]], align 4
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.matrix.columnwise.store.v10i32(<10 x i32> %in, i32* %out, i32 4, i32 2, i32 5)
   ret void
 }
 
 declare void @llvm.matrix.columnwise.store.v10i32(<10 x i32>, i32*, i32, i32, i32)
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
-define <8 x double> @transpose(<8 x double> %a) {
+define <8 x double> @transpose(<8 x double> %a) #0 {
 ; CHECK-LABEL: @transpose(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x double> [[A]], <8 x double> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0

 ;
 entry:
   %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
   ret <8 x double> %c
 }
 
 declare <8 x double> @llvm.matrix.transpose(<8 x double>, i32, i32)
 
-define <8 x double> @transpose_single_column(<8 x double> %a) {
+define <8 x double> @transpose_single_column(<8 x double> %a) #0 {
 ; CHECK-LABEL: @transpose_single_column(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[SPLIT]], i64 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <1 x double> undef, double [[TMP2]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[SPLIT]], i64 2

 ;
 entry:
   %c  = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 8, i32 1)
   ret <8 x double> %c
 }
 
 declare <12 x double> @llvm.matrix.transpose.v12f64(<12 x double>, i32, i32)
 
-define <12 x double> @transpose_double_3x4(<12 x double> %a) {
+define <12 x double> @transpose_double_3x4(<12 x double> %a) #0 {
 ; CHECK-LABEL: @transpose_double_3x4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <12 x double> [[A:%.*]], <12 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <12 x double> [[A]], <12 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <12 x double> [[A]], <12 x double> undef, <3 x i32> <i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <12 x double> [[A]], <12 x double> undef, <3 x i32> <i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> undef, double [[TMP0]], i64 0

 ; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x double> [[TMP23]], <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x double> [[TMP24]], <8 x double> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <12 x double> [[TMP26]]
 ;
 entry:
   %c  = call <12 x double> @llvm.matrix.transpose.v12f64(<12 x double> %a, i32 3, i32 4)
   ret <12 x double> %c
 }
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
-define <8 x float> @transpose(<8 x float> %a) {
+define <8 x float> @transpose(<8 x float> %a) #0 {
 ; CHECK-LABEL: @transpose(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[TMP0]], i64 0

 ;
 entry:
   %c  = call <8 x float> @llvm.matrix.transpose(<8 x float> %a, i32 2, i32 4)
   ret <8 x float> %c
 }
 
 declare <8 x float> @llvm.matrix.transpose(<8 x float>, i32, i32)
 
-define <8 x float> @transpose_single_column(<8 x float> %a) {
+define <8 x float> @transpose_single_column(<8 x float> %a) #0 {
 ; CHECK-LABEL: @transpose_single_column(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x float> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <1 x float> undef, float [[TMP0]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[SPLIT]], i64 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <1 x float> undef, float [[TMP2]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[SPLIT]], i64 2

 ;
 entry:
   %c  = call <8 x float> @llvm.matrix.transpose(<8 x float> %a, i32 8, i32 1)
   ret <8 x float> %c
 }
 
 declare <12 x float> @llvm.matrix.transpose.v12f32(<12 x float>, i32, i32)
 
-define <12 x float> @transpose_float_3x4(<12 x float> %a) {
+define <12 x float> @transpose_float_3x4(<12 x float> %a) #0 {
 ; CHECK-LABEL: @transpose_float_3x4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <12 x float> [[A:%.*]], <12 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <12 x float> [[A]], <12 x float> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <12 x float> [[A]], <12 x float> undef, <3 x i32> <i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <12 x float> [[A]], <12 x float> undef, <3 x i32> <i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x float> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[TMP0]], i64 0

 ; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x float> [[TMP23]], <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x float> [[TMP24]], <8 x float> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <12 x float> [[TMP26]]
 ;
 entry:
   %c  = call <12 x float> @llvm.matrix.transpose.v12f32(<12 x float> %a, i32 3, i32 4)
   ret <12 x float> %c
 }
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }

llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
 
-define <8 x i32> @transpose(<8 x i32> %a) {
+define <8 x i32> @transpose(<8 x i32> %a) #0 {
 ; CHECK-LABEL: @transpose(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[TMP0]], i64 0

 ;
 entry:
   %c  = call <8 x i32> @llvm.matrix.transpose(<8 x i32> %a, i32 2, i32 4)
   ret <8 x i32> %c
 }
 
 declare <8 x i32> @llvm.matrix.transpose(<8 x i32>, i32, i32)
 
-define <8 x i32> @transpose_single_column(<8 x i32> %a) {
+define <8 x i32> @transpose_single_column(<8 x i32> %a) #0 {
 ; CHECK-LABEL: @transpose_single_column(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <1 x i32> undef, i32 [[TMP0]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <1 x i32> undef, i32 [[TMP2]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[SPLIT]], i64 2

 ;
 entry:
   %c  = call <8 x i32> @llvm.matrix.transpose(<8 x i32> %a, i32 8, i32 1)
   ret <8 x i32> %c
 }
 
 declare <12 x i32> @llvm.matrix.transpose.v12i32(<12 x i32>, i32, i32)
 
-define <12 x i32> @transpose_i32_3x4(<12 x i32> %a) {
+define <12 x i32> @transpose_i32_3x4(<12 x i32> %a) #0 {
 ; CHECK-LABEL: @transpose_i32_3x4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <12 x i32> [[A:%.*]], <12 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <12 x i32> [[A]], <12 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <12 x i32> [[A]], <12 x i32> undef, <3 x i32> <i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <12 x i32> [[A]], <12 x i32> undef, <3 x i32> <i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x i32> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[TMP0]], i64 0

 ; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x i32> [[TMP24]], <8 x i32> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <12 x i32> [[TMP26]]
 ;
 entry:
   %c  = call <12 x i32> @llvm.matrix.transpose.v12i32(<12 x i32> %a, i32 3, i32 4)
   ret <12 x i32> %c
 }
+
+attributes #0 = { "may-contain-matrix-intrinsics"="true" }