[Matrix] Add __builtin_matrix_column_store to Clang.

[fhahn]

Summary

This patch add __builtin_matrix_column_major_store to Clang,
as described in clang/docs/MatrixTypes.rst. In the initial version,
the stride is not optional yet.

[merge_guards_bot]

Unit tests: unknown.

clang-tidy: unknown.

clang-format: unknown.

Build artifacts: diff.json, console-log.txt

[fhahn]

ping. Simplify code, extend tests. This should now be ready for review.

[fhahn]

Ping :)

Rebased and applied feedback from D72781.

[rjmccall]

LGTM

clang/include/clang/Basic/DiagnosticSemaKinds.td

10792–10798

You can write %ord0 to get the diagnostic to print 1st, 2nd, etc, if that's acceptable vs. the spelled-out version.

[fhahn]

Rebased after recent changes landed upstream. Will commit shortly

[fhahn]

clang/include/clang/Basic/DiagnosticSemaKinds.td

10792–10798

Thanks for pointing me to %ord! I think most diagnostics for matrixes currently use the spelled out versions, but I'll prepare a follow-up patch to update all matrix diagnostics to consistently use %ord.

clang/include/clang/Basic/Builtins.def

 BUILTIN(__builtin_shufflevector, "v."   , "nct")
 BUILTIN(__builtin_convertvector, "v."   , "nct")
 BUILTIN(__builtin_alloca, "v*z"   , "Fn")
 BUILTIN(__builtin_alloca_with_align, "v*zIz", "Fn")
 BUILTIN(__builtin_call_with_static_chain, "v.", "nt")
 
 BUILTIN(__builtin_matrix_transpose, "v.", "nFt")
 BUILTIN(__builtin_matrix_column_major_load, "v.", "nFt")
+BUILTIN(__builtin_matrix_column_major_store, "v.", "nFt")
 
 // "Overloaded" Atomic operator builtins.  These are overloaded to support data
 // types of i8, i16, i32, i64, and i128.  The front-end sees calls to the
 // non-suffixed version of these (which has a bogus type) and transforms them to
 // the right overloaded version in Sema (plus casts).
 
 // FIXME: These assume that char -> i8, short -> i16, int -> i32,
 // long long -> i64.

clang/include/clang/Basic/DiagnosticSemaKinds.td

 def warn_mismatched_import : Warning<
   "import %select{module|name}0 (%1) does not match the import %select{module|name}0 (%2) of the "
   "previous declaration">,
   InGroup<IgnoredAttributes>;
 def warn_import_on_definition : Warning<
   "import %select{module|name}0 cannot be applied to a function with a definition">,
   InGroup<IgnoredAttributes>;
 
-def err_builtin_matrix_arg: Error<
-  "%select{first|second}0 argument must be a matrix">;
+def err_builtin_matrix_arg: Error<"first argument must be a matrix">;
 
 def err_builtin_matrix_scalar_unsigned_arg: Error<
   "%0 argument must be a constant unsigned integer expression">;
 
 def err_builtin_matrix_pointer_arg: Error<
-  "%select{first|second}0 argument must be a pointer to a valid matrix element type">;
+  "%0 argument must be a pointer to a valid matrix element type">;
+
+def err_builtin_matrix_pointer_arg_mismatch: Error<
+  "the pointee of the second argument must match the element type of the first argument (%0 != %1)">;
+
+def err_builtin_matrix_store_to_const: Error<
+  "cannot store matrix to read-only pointer">;
 
 def err_builtin_matrix_stride_too_small: Error<
   "stride must be greater or equal to the number of rows">;
 
 def err_builtin_matrix_invalid_dimension: Error<
   "%0 dimension is outside the allowed range [1, %1]">;
 
 def err_preserve_field_info_not_field : Error<

clang/include/clang/Sema/Sema.h

bool SemaBuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall,
                                 bool AllowName);
   bool SemaBuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
 
   // Matrix builtin handling.
   ExprResult SemaBuiltinMatrixTranspose(CallExpr *TheCall,
                                         ExprResult CallResult);
   ExprResult SemaBuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
                                               ExprResult CallResult);
+  ExprResult SemaBuiltinMatrixColumnMajorStore(CallExpr *TheCall,
+                                               ExprResult CallResult);
 
 public:
   enum FormatStringType {
     FST_Scanf,
     FST_Printf,
     FST_NSString,
     FST_Strftime,
     FST_Strfmon,

clang/lib/CodeGen/CGBuiltin.cpp

EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(),
                         E->getArg(0)->getExprLoc(), FD, 0);
     Value *Result = MB.CreateColumnMajorLoad(
         Src.getPointer(), Align(Src.getAlignment().getQuantity()), Stride,
         IsVolatile, ResultTy->getNumRows(), ResultTy->getNumColumns(),
         "matrix");
     return RValue::get(Result);
   }
 
+  case Builtin::BI__builtin_matrix_column_major_store: {
+    MatrixBuilder<CGBuilderTy> MB(Builder);
+    Value *Matrix = EmitScalarExpr(E->getArg(0));
+    Address Dst = EmitPointerWithAlignment(E->getArg(1));
+    Value *Stride = EmitScalarExpr(E->getArg(2));
+
+    const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
+    auto *PtrTy = E->getArg(1)->getType()->getAs<PointerType>();
+    assert(PtrTy && "arg1 must be of pointer type");
+    bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
+
+    EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(),
+                        E->getArg(1)->getExprLoc(), FD, 0);
+    Value *Result = MB.CreateColumnMajorStore(
+        Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()),
+        Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
+    return RValue::get(Result);
+  }
+
   case Builtin::BIfinite:
   case Builtin::BI__finite:
   case Builtin::BIfinitef:
   case Builtin::BI__finitef:
   case Builtin::BIfinitel:
   case Builtin::BI__finitel:
   case Builtin::BI__builtin_isinf:
   case Builtin::BI__builtin_isfinite: {

clang/lib/Sema/SemaChecking.cpp

case Builtin::BI__builtin_return_address: {
     break;
   }
 
   case Builtin::BI__builtin_matrix_transpose:
     return SemaBuiltinMatrixTranspose(TheCall, TheCallResult);
 
   case Builtin::BI__builtin_matrix_column_major_load:
     return SemaBuiltinMatrixColumnMajorLoad(TheCall, TheCallResult);
+
+  case Builtin::BI__builtin_matrix_column_major_store:
+    return SemaBuiltinMatrixColumnMajorStore(TheCall, TheCallResult);
   }
 
   // Since the target specific builtins for each arch overlap, only check those
   // of the arch we are compiling for.
   if (Context.BuiltinInfo.isTSBuiltin(BuiltinID)) {
     if (Context.BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
       assert(Context.getAuxTargetInfo() &&
              "Aux Target Builtin, but not an aux target?");
ExprResult Sema::SemaBuiltinMatrixTranspose(CallExpr *TheCall,
 
   ExprResult MatrixArg = DefaultLvalueConversion(TheCall->getArg(0));
   if (MatrixArg.isInvalid())
     return MatrixArg;
   Expr *Matrix = MatrixArg.get();
 
   auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
   if (!MType) {
-    Diag(Matrix->getBeginLoc(), diag::err_builtin_matrix_arg) << 0;
+    Diag(Matrix->getBeginLoc(), diag::err_builtin_matrix_arg);
     return ExprError();
   }
 
   // Create returned matrix type by swapping rows and columns of the argument
   // matrix type.
   QualType ResultType = Context.getConstantMatrixType(
       MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
 
if (PtrExpr->isTypeDependent()) {
       TheCall->setType(Context.DependentTy);
       return TheCall;
     }
   }
 
   auto *PtrTy = PtrExpr->getType()->getAs<PointerType>();
   QualType ElementTy;
   if (!PtrTy) {
-    Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_pointer_arg) << 0;
+    Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_pointer_arg)
+        << "first";
     ArgError = true;
   } else {
     ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
 
     if (!ConstantMatrixType::isValidElementType(ElementTy)) {
-      Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_pointer_arg) << 0;
+      Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_pointer_arg)
+          << "first";
       ArgError = true;
     }
   }
 
   // Apply default Lvalue conversions and convert the expression to size_t.
   auto ApplyArgumentConversions = [this](Expr *E) {
     ExprResult Conv = DefaultLvalueConversion(E);
     if (Conv.isInvalid())
ExprResult Sema::SemaBuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
 
   if (ArgError || !MaybeRows || !MaybeColumns)
     return ExprError();
 
   TheCall->setType(
       Context.getConstantMatrixType(ElementTy, *MaybeRows, *MaybeColumns));
   return CallResult;
 }
+
+ExprResult Sema::SemaBuiltinMatrixColumnMajorStore(CallExpr *TheCall,
+                                                   ExprResult CallResult) {
+  if (checkArgCount(*this, TheCall, 3))
+    return ExprError();
+
+  Expr *MatrixExpr = TheCall->getArg(0);
+  Expr *PtrExpr = TheCall->getArg(1);
+  Expr *StrideExpr = TheCall->getArg(2);
+
+  bool ArgError = false;
+
+  {
+    ExprResult MatrixConv = DefaultLvalueConversion(MatrixExpr);
+    if (MatrixConv.isInvalid())
+      return MatrixConv;
+    MatrixExpr = MatrixConv.get();
+    TheCall->setArg(0, MatrixExpr);
+  }
+  if (MatrixExpr->isTypeDependent()) {
+    TheCall->setType(Context.DependentTy);
+    return TheCall;
+  }
+
+  auto *MatrixTy = MatrixExpr->getType()->getAs<ConstantMatrixType>();
+  if (!MatrixTy) {
+    Diag(MatrixExpr->getBeginLoc(), diag::err_builtin_matrix_arg) << 0;
+    ArgError = true;
+  }
+
+  {
+    ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
+    if (PtrConv.isInvalid())
+      return PtrConv;
+    PtrExpr = PtrConv.get();
+    TheCall->setArg(1, PtrExpr);
+    if (PtrExpr->isTypeDependent()) {
+      TheCall->setType(Context.DependentTy);
+      return TheCall;
+    }
+  }
+
+  // Check pointer argument.
+  auto *PtrTy = PtrExpr->getType()->getAs<PointerType>();
+  if (!PtrTy) {
+    Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_pointer_arg)
+        << "second";
+    ArgError = true;
+  } else {
+    QualType ElementTy = PtrTy->getPointeeType();
+    if (ElementTy.isConstQualified()) {
+      Diag(PtrExpr->getBeginLoc(), diag::err_builtin_matrix_store_to_const);
+      ArgError = true;
+    }
+    ElementTy = ElementTy.getUnqualifiedType().getCanonicalType();
+    if (MatrixTy &&
+        !Context.hasSameType(ElementTy, MatrixTy->getElementType())) {
+      Diag(PtrExpr->getBeginLoc(),
+           diag::err_builtin_matrix_pointer_arg_mismatch)
+          << ElementTy << MatrixTy->getElementType();
+      ArgError = true;
+    }
+  }
+
+  // Apply default Lvalue conversions and convert the stride expression to
+  // size_t.
+  {
+    ExprResult StrideConv = DefaultLvalueConversion(StrideExpr);
+    if (StrideConv.isInvalid())
+      return StrideConv;
+
+    StrideConv = tryConvertExprToType(StrideConv.get(), Context.getSizeType());
+    if (StrideConv.isInvalid())
+      return StrideConv;
+    StrideExpr = StrideConv.get();
+    TheCall->setArg(2, StrideExpr);
+  }
+
+  // Check stride argument.
+  llvm::APSInt Value(64);
+  if (MatrixTy && StrideExpr->isIntegerConstantExpr(Value, Context)) {
+    uint64_t Stride = Value.getZExtValue();
+    if (Stride < MatrixTy->getNumRows()) {
+      Diag(StrideExpr->getBeginLoc(),
+           diag::err_builtin_matrix_stride_too_small);
+      ArgError = true;
+    }
+  }
+
+  if (ArgError)
+    return ExprError();
+
+  return CallResult;
+}

clang/test/CodeGen/matrix-type-builtins.c

 
 void column_major_load_volatile(volatile double *Ptr) {
   // CHECK-LABEL: define void @column_major_load_volatile(double* %Ptr)
   // CHECK:         [[PTR:%.*]] = load double*, double** %Ptr.addr, align 8
   // CHECK-NEXT:    call <25 x double> @llvm.matrix.column.major.load.v25f64.p0f64(double* align 8 [[PTR]], i64 5, i1 true, i32 5, i32 5)
 
   dx5x5_t m_a1 = __builtin_matrix_column_major_load(Ptr, 5, 5, 5);
 }
+
+void column_major_store_with_const_stride_double(double *Ptr) {
+  // CHECK-LABEL: define void @column_major_store_with_const_stride_double(double* %Ptr)
+  // CHECK:         [[M:%.*]] = load <25 x double>, <25 x double>* {{.*}}, align 8
+  // CHECK-NEXT:    [[PTR:%.*]] = load double*, double** %Ptr.addr, align 8
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v25f64.p0f64(<25 x double> [[M]], double* align 8 [[PTR]], i64 5, i1 false, i32 5, i32 5)
+
+  dx5x5_t m;
+  __builtin_matrix_column_major_store(m, Ptr, 5);
+}
+
+void column_major_store_with_const_stride2_double(double *Ptr) {
+  // CHECK-LABEL: define void @column_major_store_with_const_stride2_double(double* %Ptr)
+  // CHECK:         [[M:%.*]] = load <25 x double>, <25 x double>* {{.*}}, align 8
+  // CHECK-NEXT:    [[PTR:%.*]] = load double*, double** %Ptr.addr, align 8
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v25f64.p0f64(<25 x double> [[M]], double* align 8 [[PTR]], i64 15, i1 false, i32 5, i32 5)
+  //
+  dx5x5_t m;
+  __builtin_matrix_column_major_store(m, Ptr, 2 * 3 + 9);
+}
+
+void column_major_store_with_stride_math_int(int *Ptr, int S) {
+  // CHECK-LABEL: define void @column_major_store_with_stride_math_int(i32* %Ptr, i32 %S)
+  // CHECK:         [[M:%.*]] = load <80 x i32>, <80 x i32>* {{.*}}, align 4
+  // CHECK-NEXT:    [[PTR:%.*]] = load i32*, i32** %Ptr.addr, align 8
+  // CHECK-NEXT:    [[S:%.*]] = load i32, i32* %S.addr, align 4
+  // CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[S]], 32
+  // CHECK-NEXT:    [[IDX:%.*]] = sext i32 [[ADD]] to i64
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v80i32.p0i32(<80 x i32> [[M]], i32* align 4 [[PTR]], i64 [[IDX]], i1 false, i32 4, i32 20)
+
+  ix4x20_t m;
+  __builtin_matrix_column_major_store(m, Ptr, S + 32);
+}
+
+void column_major_store_with_stride_math_s_int(int *Ptr, short S) {
+  // CHECK-LABEL: define void @column_major_store_with_stride_math_s_int(i32* %Ptr, i16 signext %S)
+  // CHECK:         [[M:%.*]] = load <80 x i32>, <80 x i32>* {{.*}}, align 4
+  // CHECK-NEXT:    [[PTR:%.*]] = load i32*, i32** %Ptr.addr, align 8
+  // CHECK-NEXT:    [[S:%.*]] = load i16, i16* %S.addr, align 2
+  // CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[S]] to i32
+  // CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[EXT]], 2
+  // CHECK-NEXT:    [[IDX:%.*]] = sext i32 [[ADD]] to i64
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v80i32.p0i32(<80 x i32> [[M]], i32* align 4 [[PTR]], i64 [[IDX]], i1 false, i32 4, i32 20)
+
+  ix4x20_t m;
+  __builtin_matrix_column_major_store(m, Ptr, S + 2);
+}
+
+void column_major_store_array1(double Ptr[25]) {
+  // CHECK-LABEL: define void @column_major_store_array1(double* %Ptr)
+  // CHECK:         [[M:%.*]] = load <25 x double>, <25 x double>* {{.*}}, align 8
+  // CHECK-NEXT:    [[PTR:%.*]] = load double*, double** %Ptr.addr, align 8
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v25f64.p0f64(<25 x double> [[M]], double* align 8 [[PTR]], i64 5, i1 false, i32 5, i32 5)
+
+  dx5x5_t m;
+  __builtin_matrix_column_major_store(m, Ptr, 5);
+}
+
+void column_major_store_array2() {
+  // CHECK-LABEL: define void @column_major_store_array2()
+  // CHECK:         [[M:%.*]] = load <25 x double>, <25 x double>* {{.*}}, align 8
+  // CHECK-NEXT:    [[PTR:%.*]] = getelementptr inbounds [25 x double], [25 x double]* %Ptr, i64 0, i64 0
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v25f64.p0f64(<25 x double> [[M]], double* align 16 [[PTR]], i64 5, i1 false, i32 5, i32 5)
+
+  double Ptr[25];
+  dx5x5_t m;
+  __builtin_matrix_column_major_store(m, Ptr, 5);
+}
+
+void column_major_store_volatile(volatile double *Ptr) {
+  // CHECK-LABEL: define void @column_major_store_volatile(double* %Ptr) #0 {
+  // CHECK:         [[M:%.*]] = load <25 x double>, <25 x double>* {{.*}}, align 8
+  // CHECK-NEXT:    [[PTR:%.*]] = load double*, double** %Ptr.addr, align 8
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v25f64.p0f64(<25 x double> [[M]], double* align 8 [[PTR]], i64 5, i1 true, i32 5, i32 5)
+
+  dx5x5_t m;
+  __builtin_matrix_column_major_store(m, Ptr, 5);
+}

clang/test/CodeGenCXX/matrix-type-builtins.cpp

 template <typename PtrT, unsigned R, unsigned C, unsigned S>
 matrix_t<typename remove_pointer<PtrT>::type, R, C> column_major_load_with_stride2(PtrT Ptr) {
   return __builtin_matrix_column_major_load(Ptr, R, C, S);
 }
 
 void call_column_major_load_with_stride2(float *Ptr) {
   matrix_t<float, 2, 2> m = column_major_load_with_stride2<float *, 2, 2, 2>(Ptr);
 }
+
+template <typename T, unsigned R, unsigned C, unsigned S>
+void column_major_store_with_stride(matrix_t<T, R, C> &m, T *Ptr) {
+  __builtin_matrix_column_major_store(m, Ptr, S);
+}
+
+void test_column_major_store_with_stride_template_double(double *Ptr) {
+  // CHECK-LABEL: define void @_Z51test_column_major_store_with_stride_template_doublePd(double* %Ptr)
+  // CHECK:         [[PTR:%.*]] = load double*, double** %Ptr.addr, align 8
+  // CHECK-NEXT:    call void @_Z30column_major_store_with_strideIdLj10ELj4ELj15EEvRU11matrix_typeXT0_EXT1_ET_PS0_([40 x double]* nonnull align 8 dereferenceable(320) %M1, double* [[PTR]])
+
+  // CHECK-LABEL:  define linkonce_odr void @_Z30column_major_store_with_strideIdLj10ELj4ELj15EEvRU11matrix_typeXT0_EXT1_ET_PS0_([40 x double]* nonnull align 8 dereferenceable(320) %m, double* %Ptr)
+  // CHECK:         [[M:%.*]] = load <40 x double>, <40 x double>* {{.*}}, align 8
+  // CHECK-NEXT:    [[PTR:%.*]] = load double*, double** %Ptr.addr, align 8
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v40f64.p0f64(<40 x double> [[M]], double* align 8 [[PTR]], i64 15, i1 false, i32 10, i32 4)
+
+  matrix_t<double, 10, 4> M1;
+  column_major_store_with_stride<double, 10, 4, 15>(M1, Ptr);
+}
+
+void test_column_major_store_with_stride_template_int(int *Ptr) {
+  // CHECK-LABEL: define void @_Z48test_column_major_store_with_stride_template_intPi(i32* %Ptr)
+  // CHECK:         [[PTR:%.*]] = load i32*, i32** %Ptr.addr, align 8
+  // CHECK-NEXT:    call void @_Z30column_major_store_with_strideIiLj3ELj2ELj3EEvRU11matrix_typeXT0_EXT1_ET_PS0_([6 x i32]* nonnull align 4 dereferenceable(24) %M1, i32* [[PTR]])
+
+  // CHECK-LABEL:  define linkonce_odr void @_Z30column_major_store_with_strideIiLj3ELj2ELj3EEvRU11matrix_typeXT0_EXT1_ET_PS0_([6 x i32]* nonnull align 4 dereferenceable(24) %m, i32* %Ptr)
+  // CHECK:         [[M:%.*]] = load <6 x i32>, <6 x i32>* {{.*}}, align 4
+  // CHECK-NEXT:    [[PTR:%.*]] = load i32*, i32** %Ptr.addr, align 8
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v6i32.p0i32(<6 x i32> [[M]], i32* align 4 [[PTR]], i64 3, i1 false, i32 3, i32 2)
+
+  matrix_t<int, 3, 2> M1;
+  column_major_store_with_stride<int, 3, 2, 3>(M1, Ptr);
+}
+
+void test_column_major_store_stride_wrapper(int *Ptr, UnsignedWrapper &W) {
+  // CHECK-LABEL: define void @_Z38test_column_major_store_stride_wrapperPiR15UnsignedWrapper(i32* %Ptr, %struct.UnsignedWrapper* nonnull align 1 dereferenceable(1) %W)
+  // CHECK:         [[M:%.*]] = load <4 x i32>, <4 x i32>* {{.*}}, align 4
+  // CHECK-NEXT:    [[PTR:%.*]] = load i32*, i32** %Ptr.addr, align 8
+  // CHECK-NEXT:    [[W:%.*]] = load %struct.UnsignedWrapper*, %struct.UnsignedWrapper** %W.addr, align 8
+  // CHECK-NEXT:    [[IDX:%.*]] = call i32 @_ZN15UnsignedWrappercvjEv(%struct.UnsignedWrapper* [[W]])
+  // CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v4i32.p0i32(<4 x i32> [[M]], i32* align 4 [[PTR]], i64 [[IDX_EXT]], i1 false, i32 2, i32 2)
+
+  matrix_t<int, 2, 2> M1;
+  __builtin_matrix_column_major_store(M1, Ptr, W);
+}
+
+void test_column_major_store_constexpr_stride_constexpr(int *Ptr) {
+  // CHECK-LABEL: define void @_Z50test_column_major_store_constexpr_stride_constexprPi(i32* %Ptr)
+  // CHECK:         [[M:%.*]] = load <4 x i32>, <4 x i32>* %0, align 4
+  // CHECK-NEXT:    [[PTR:%.*]] = load i32*, i32** %Ptr.addr, align 8
+  // CHECK-NEXT:    [[IDX:%.*]] = call i32 @_Z10constexpr3v()
+  // CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[IDX]] to i64
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v4i32.p0i32(<4 x i32> [[M]], i32* align 4 [[PTR]], i64 [[IDX_EXT]], i1 false, i32 2, i32 2)
+
+  matrix_t<int, 2, 2> M;
+  __builtin_matrix_column_major_store(M, Ptr, constexpr3());
+}

clang/test/CodeGenObjC/matrix-type-builtins.m

void test_column_major_load(PtrValue *Ptr, IntValue *Stride) {
   // CHECK-LABEL: define void @test_column_major_load(%2* %Ptr, %3* %Stride) #4 {
   // CHECK:         [[STRIDE:%.*]] = call i32 bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i32 (i8*, i8*)*)
   // CHECK-NEXT:    [[STRIDE_EXT:%.*]] = sext i32 [[STRIDE]] to i64
   // CHECK:         [[PTR:%.*]] = call i32* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i32* (i8*, i8*)*)
   // CHECK-NEXT:    call <12 x i32> @llvm.matrix.column.major.load.v12i32.p0i32(i32* align 4 [[PTR]], i64 [[STRIDE_EXT]], i1 false, i32 3, i32 4)
 
   u3x4 m = __builtin_matrix_column_major_load(Ptr.value, 3, 4, Stride.value);
 }
+
+void test_column_major_store(UnsignedMatrixValue *M, PtrValue *Ptr, IntValue *Stride) {
+  // CHECK-LABEL: define void @test_column_major_store(%1* %M, %2* %Ptr, %3* %Stride) #3 {
+  // CHECK:         [[M:%.*]] = call <12 x i32> bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to <12 x i32> (i8*, i8*)*)
+  // CHECK:         [[PTR:%.*]] = call i32* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i32* (i8*, i8*)*)
+  // CHECK:         [[IDX:%.*]] = call i32 bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i32 (i8*, i8*)*)
+  // CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[IDX]] to i64
+  // CHECK-NEXT:    call void @llvm.matrix.column.major.store.v12i32.p0i32(<12 x i32> [[M]], i32* align 4 [[PTR]], i64 [[IDX_EXT]], i1 false, i32 3, i32 4)
+
+  __builtin_matrix_column_major_store(M.value, Ptr.value, Stride.value);
+}

clang/test/Sema/matrix-type-builtins.c

void column_major_load(float *p1, int *p2, _Bool *p3, struct Foo *p4) {
 
   sx5x10_t a13 = __builtin_matrix_column_major_load(
       10,  // expected-error {{first argument must be a pointer to a valid matrix element type}}
       *p4, // expected-error {{casting 'struct Foo' to incompatible type 'unsigned long'}}
       "",  // expected-error {{column argument must be a constant unsigned integer expression}}
            // expected-warning@-1 {{incompatible pointer to integer conversion casting 'char [1]' to type 'unsigned long'}}
       10);
 }
+
+void column_major_store(sx5x10_t *m1, ix3x2_t *m2, float *p1, int *p2, struct Foo *p3, const float *p4) {
+  __builtin_matrix_column_major_store(*m1, p1, 1);
+  // expected-error@-1 {{stride must be greater or equal to the number of rows}}
+  __builtin_matrix_column_major_store(*m1, p1, 0);
+  // expected-error@-1 {{stride must be greater or equal to the number of rows}}
+  __builtin_matrix_column_major_store(*m1, p2, 10);
+  // expected-error@-1 {{the pointee of the second argument must match the element type of the first argument ('int' != 'float')}}
+  __builtin_matrix_column_major_store(p1, p2, 10);
+  // expected-error@-1 {{first argument must be a matrix}}
+
+  __builtin_matrix_column_major_store(
+      "",   // expected-error {{first argument must be a matrix}}
+      10,   // expected-error {{second argument must be a pointer to a valid matrix element type}}
+      *p3); // expected-error {{casting 'struct Foo' to incompatible type 'unsigned long'}}
+
+  __builtin_matrix_column_major_store(
+      *m1,
+      10, // expected-error {{second argument must be a pointer to a valid matrix element type}}
+      10);
+
+  *m1 = __builtin_matrix_column_major_store(*m1, p1, 10);
+  // expected-error@-1 {{assigning to 'sx5x10_t' (aka 'float __attribute__((matrix_type(5, 10)))') from incompatible type 'void'}}
+
+  int x = __builtin_matrix_column_major_store(*m1, p1, 10);
+  // expected-error@-1 {{initializing 'int' with an expression of incompatible type 'void'}}
+
+  __builtin_matrix_column_major_store(*m1, p4, 20);
+  // expected-error@-1 {{cannot store matrix to read-only pointer}}
+}

clang/test/SemaCXX/matrix-type-builtins.cpp

 
 void call_column_major_load_temp(unsigned *Ptr, unsigned X) {
   (void)__builtin_matrix_column_major_load(Ptr, X, X, X);
   // expected-error@-1 {{row argument must be a constant unsigned integer expression}}
   // expected-error@-2 {{column argument must be a constant unsigned integer expression}}
   (void)__builtin_matrix_column_major_load(X, 2, 2, 2);
   // expected-error@-1 {{first argument must be a pointer to a valid matrix element type}}
 }
+
+template <typename EltTy0, unsigned R0, unsigned C0, typename PtrTy>
+void column_major_store(MyMatrix<EltTy0, R0, C0> &A, PtrTy Ptr, unsigned Stride) {
+  __builtin_matrix_column_major_store(A.value, Ptr, Stride);
+  // expected-error@-1 {{the pointee of the second argument must match the element type of the first argument ('float' != 'unsigned int')}}
+}
+
+template <typename MTy, typename PtrTy, unsigned Stride>
+void column_major_store(MTy &A, PtrTy Ptr) {
+  __builtin_matrix_column_major_store(A.value, Ptr, Stride);
+  // expected-error@-1 {{stride must be greater or equal to the number of rows}}
+}
+
+void test_column_major_stores_template(MyMatrix<unsigned, 2, 3> &M1, unsigned *Ptr1, MyMatrix<float, 3, 4> &M2, float *Ptr2) {
+  column_major_store(M1, Ptr2, 10);
+  // expected-note@-1 {{in instantiation of function template specialization 'column_major_store<unsigned int, 2, 3, float *>' requested here}}
+
+  column_major_store<decltype(M2), float *, 1>(M2, Ptr2);
+  // expected-note@-1 {{in instantiation of function template specialization 'column_major_store<MyMatrix<float, 3, 4> &, float *, 1>' requested here}}
+}
+
+template <typename EltTy0, unsigned R0, unsigned C0, typename EltTy1>
+void column_major_store(MyMatrix<EltTy0, R0, C0> &A, EltTy1 *Ptr) {
+  __builtin_matrix_column_major_store(A.value, Ptr, 1);
+  // expected-error@-1 3 {{stride must be greater or equal to the number of rows}}
+  // expected-error@-2 {{the pointee of the second argument must match the element type of the first argument ('float' != 'unsigned int')}}
+  // expected-error@-3 {{the pointee of the second argument must match the element type of the first argument ('unsigned int' != 'float')}}
+
+  char *s;
+  return __builtin_matrix_column_major_store(A.value, s, 20);
+  // expected-error@-1 {{the pointee of the second argument must match the element type of the first argument ('char' != 'unsigned int')}}
+  // expected-error@-2 {{the pointee of the second argument must match the element type of the first argument ('char' != 'unsigned int')}}
+  // expected-error@-3 {{he pointee of the second argument must match the element type of the first argument ('char' != 'float')}}
+}
+
+void test_column_major_store_template(unsigned *Ptr1, float *Ptr2) {
+  MyMatrix<unsigned, 2, 3> Mat1;
+  column_major_store<unsigned, 2, 3, unsigned>(Mat1, Ptr1);
+  // expected-note@-1 {{in instantiation of function template specialization 'column_major_store<unsigned int, 2, 3, unsigned int>'}}
+  column_major_store<unsigned, 2, 3, float>(Mat1, Ptr2);
+  // expected-note@-1 {{in instantiation of function template specialization 'column_major_store<unsigned int, 2, 3, float>'}}
+
+  MyMatrix<float, 2, 3> Mat2;
+  column_major_store<float, 2, 3, unsigned>(Mat2, Ptr1);
+  // expected-note@-1 {{in instantiation of function template specialization 'column_major_store<float, 2, 3, unsigned int>'}}
+}
+
+void test_column_major_store_constexpr(unsigned *Ptr, MyMatrix<unsigned, 3, 3> &M) {
+  __builtin_matrix_column_major_store(M.value, Ptr, constexpr1());
+  // expected-error@-1 {{stride must be greater or equal to the number of rows}}
+  __builtin_matrix_column_major_store(constexpr1(), Ptr, 1);
+  // expected-error@-1 {{first argument must be a matrix}}
+  __builtin_matrix_column_major_store(M.value, constexpr1(), 1);
+  // expected-error@-1 {{second argument must be a pointer to a valid matrix element type}}
+  // expected-error@-2 {{stride must be greater or equal to the number of rows}}
+}
+
+void test_column_major_store_wrapper(unsigned *Ptr, MyMatrix<unsigned, 3, 3> &M, IntWrapper &W) {
+  __builtin_matrix_column_major_store(M.value, Ptr, W);
+
+  __builtin_matrix_column_major_store(W, Ptr, W);
+  // expected-error@-1 {{first argument must be a matrix}}
+}

clang/test/SemaObjC/matrix-type-builtins.m

 }
 
 typedef double double3x3 __attribute__((matrix_type(3, 3)));
 
 double test_dimension_mismatch(double3x3 m, MatrixValue *mv) {
   m = __builtin_matrix_transpose(mv.value);
   // expected-error@-1 {{assigning to 'double3x3' (aka 'double __attribute__((matrix_type(3, 3)))') from incompatible type 'double __attribute__((matrix_type(4, 4)))'}}
 }
+
+double test_store(MatrixValue *mv, float *Ptr) {
+  __builtin_matrix_column_major_store(mv.value, Ptr, 1);
+  // expected-error@-1 {{the pointee of the second argument must match the element type of the first argument ('float' != 'double')}}
+  // expected-error@-2 {{stride must be greater or equal to the number of rows}}
+
+  __builtin_matrix_column_major_store(mv.value, mv.value, mv.value);
+  // expected-error@-1 {{second argument must be a pointer to a valid matrix element type}}
+  // expected-error@-2 {{casting 'double4x4' (aka 'double __attribute__((matrix_type(4, 4)))') to incompatible type 'unsigned long'}}
+}