This is an archive of the discontinued LLVM Phabricator instance.

Differential D83785

[Matrix] Change the definitions of the load/store intrinsics
ClosedPublic

Authored by SjoerdMeijer on Jul 14 2020, 9:26 AM.

Details

Reviewers
fhahn
anemet
Gerolf
jdoerfert
Commits
rG2b3c505d0f6e: [Matrix] Intrinsic descriptions
Summary

This changes the definitions of the matrix load/store intrinsics to load/store from/to a pointer, and not to a vector, as discussed in D83477. I will commit this and D83477 as 1 commit, put am putting this new bit up for review separately because that makes this part easier to review.

Diff Detail

Event Timeline

SjoerdMeijer created this revision.Jul 14 2020, 9:26 AM
Herald added a project: Restricted Project. · View Herald TranscriptJul 14 2020, 9:26 AM
Herald added subscribers: tschuett, jdoerfert, hiraditya. · View Herald Transcript
SjoerdMeijer updated this revision to Diff 277873.Jul 14 2020, 9:31 AM

don't need new block scopes in the Verifier anymore.

fhahn accepted this revision.Jul 14 2020, 9:49 AM

LGTM, thank you very much! This has been discussed already in D83477. Passing in a pointer to a vector is confusing, as it seems to suggest we only access elements of the vector pointer, but we may access other elements, if the stride > rows.

This revision is now accepted and ready to land.Jul 14 2020, 9:49 AM
SjoerdMeijer added a comment.Jul 14 2020, 11:39 AM

Thanks for reviewing!

Closed by commit rG2b3c505d0f6e: [Matrix] Intrinsic descriptions (authored by SjoerdMeijer). · Explain WhyJul 14 2020, 11:58 AM
This revision was automatically updated to reflect the committed changes.
Herald added a reviewer: jdoerfert. · View Herald TranscriptJul 14 2020, 11:58 AM

Revision Contents

PathSize
llvm/
include/
llvm/
IR/
4 lines
lib/
IR/
16 lines
test/
Transforms/
LowerMatrixIntrinsics/
48 lines
6 lines
45 lines
37 lines
31 lines
31 lines
16 lines
Verifier/
109 lines

Diff 277873

llvm/include/llvm/IR/Intrinsics.td

Show First 20 Lines • Show All 1,452 Lines • ▼ Show 20 Linesdef int_matrix_multiply
: Intrinsic<[llvm_anyvector_ty], : Intrinsic<[llvm_anyvector_ty],
[llvm_anyvector_ty, llvm_anyvector_ty, llvm_i32_ty, llvm_i32_ty, [llvm_anyvector_ty, llvm_anyvector_ty, llvm_i32_ty, llvm_i32_ty,
llvm_i32_ty], llvm_i32_ty],
[IntrNoSync, IntrWillReturn, IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<2>>, [IntrNoSync, IntrWillReturn, IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<2>>,
ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
def int_matrix_column_major_load def int_matrix_column_major_load
: Intrinsic<[llvm_anyvector_ty], : Intrinsic<[llvm_anyvector_ty],
[LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i64_ty, llvm_i1_ty, [LLVMPointerToElt<0>, llvm_i64_ty, llvm_i1_ty,
llvm_i32_ty, llvm_i32_ty], llvm_i32_ty, llvm_i32_ty],
[IntrNoSync, IntrWillReturn, IntrArgMemOnly, IntrReadMem, [IntrNoSync, IntrWillReturn, IntrArgMemOnly, IntrReadMem,
NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>,
ImmArg<ArgIndex<4>>]>; ImmArg<ArgIndex<4>>]>;
def int_matrix_column_major_store def int_matrix_column_major_store
: Intrinsic<[], : Intrinsic<[],
[llvm_anyvector_ty, LLVMAnyPointerType<LLVMMatchType<0>>, [llvm_anyvector_ty, LLVMPointerToElt<0>,
llvm_i64_ty, llvm_i1_ty, llvm_i32_ty, llvm_i32_ty], llvm_i64_ty, llvm_i1_ty, llvm_i32_ty, llvm_i32_ty],
[IntrNoSync, IntrWillReturn, IntrArgMemOnly, IntrWriteMem, [IntrNoSync, IntrWillReturn, IntrArgMemOnly, IntrWriteMem,
WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>, WriteOnly<ArgIndex<1>>, NoCapture<ArgIndex<1>>,
ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>; ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
//===---------- Intrinsics to control hardware supported loops ----------===// //===---------- Intrinsics to control hardware supported loops ----------===//
// Specify that the value given is the number of iterations that the next loop // Specify that the value given is the number of iterations that the next loop
▲ Show 20 Lines • Show All 72 Lines • Show Last 20 Lines

llvm/lib/IR/Verifier.cpp

Show First 20 Lines • Show All 5,035 Lines • ▼ Show 20 Lines case Intrinsic::matrix_multiply:
break; break;
case Intrinsic::matrix_transpose: case Intrinsic::matrix_transpose:
NumRows = cast<ConstantInt>(Call.getArgOperand(1)); NumRows = cast<ConstantInt>(Call.getArgOperand(1));
NumColumns = cast<ConstantInt>(Call.getArgOperand(2)); NumColumns = cast<ConstantInt>(Call.getArgOperand(2));
ResultTy = cast<VectorType>(Call.getType()); ResultTy = cast<VectorType>(Call.getType());
Op0ElemTy = Op0ElemTy =
cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType();
break; break;
case Intrinsic::matrix_column_major_load: { case Intrinsic::matrix_column_major_load:
Stride = dyn_cast<ConstantInt>(Call.getArgOperand(1)); Stride = dyn_cast<ConstantInt>(Call.getArgOperand(1));
NumRows = cast<ConstantInt>(Call.getArgOperand(3)); NumRows = cast<ConstantInt>(Call.getArgOperand(3));
NumColumns = cast<ConstantInt>(Call.getArgOperand(4)); NumColumns = cast<ConstantInt>(Call.getArgOperand(4));
ResultTy = cast<VectorType>(Call.getType()); ResultTy = cast<VectorType>(Call.getType());
auto *VecTy = cast<VectorType>( Op0ElemTy =
cast<PointerType>(Call.getArgOperand(0)->getType())->getElementType()); cast<PointerType>(Call.getArgOperand(0)->getType())->getElementType();
Op0ElemTy = VecTy->getElementType();
}
break; break;
case Intrinsic::matrix_column_major_store: { case Intrinsic::matrix_column_major_store:
Stride = dyn_cast<ConstantInt>(Call.getArgOperand(2)); Stride = dyn_cast<ConstantInt>(Call.getArgOperand(2));
NumRows = cast<ConstantInt>(Call.getArgOperand(4)); NumRows = cast<ConstantInt>(Call.getArgOperand(4));
NumColumns = cast<ConstantInt>(Call.getArgOperand(5)); NumColumns = cast<ConstantInt>(Call.getArgOperand(5));
ResultTy = cast<VectorType>(Call.getArgOperand(0)->getType()); ResultTy = cast<VectorType>(Call.getArgOperand(0)->getType());
Op0ElemTy = Op0ElemTy =
cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType();
auto *VecTy = cast<VectorType>( Op1ElemTy =
cast<PointerType>(Call.getArgOperand(1)->getType())->getElementType()); cast<PointerType>(Call.getArgOperand(1)->getType())->getElementType();
Op1ElemTy = VecTy->getElementType();
}
break; break;
default: default:
llvm_unreachable("unexpected intrinsic"); llvm_unreachable("unexpected intrinsic");
} }
Assert(ResultTy->getElementType()->isIntegerTy() || Assert(ResultTy->getElementType()->isIntegerTy() ||
ResultTy->getElementType()->isFloatingPointTy(), ResultTy->getElementType()->isFloatingPointTy(),
"Result type must be an integer or floating-point type!", IF); "Result type must be an integer or floating-point type!", IF);
▲ Show 20 Lines • Show All 834 Lines • Show Last 20 Lines

llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll

; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
define <9 x double> @strided_load_3x3_volatile(<9 x double>* %in, i64 %stride) { define <9 x double> @strided_load_3x3_volatile(double* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3_volatile( ; CHECK-LABEL: @strided_load_3x3_volatile(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST]], align 8 ; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST]], align 8
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>*
; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST3]], align 8 ; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST3]], align 8
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]] ; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START5]] ; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* %in, i64 [[VEC_START5]]
; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>*
; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST7]], align 8 ; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST7]], align 8
; CHECK-NOT: = load ; CHECK-NOT: = load
; ;
entry: entry:
%load = call <9 x double> @llvm.matrix.column.major.load.v9f64(<9 x double>* %in, i64 %stride, i1 true, i32 3, i32 3) %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(double* %in, i64 %stride, i1 true, i32 3, i32 3)
ret <9 x double> %load ret <9 x double> %load
} }
declare <9 x double> @llvm.matrix.column.major.load.v9f64(<9 x double>*, i64, i1, i32, i32) declare <9 x double> @llvm.matrix.column.major.load.v9f64(double*, i64, i1, i32, i32)
define <4 x double> @load_volatile_multiply(<4 x double>* %in) { define <4 x double> @load_volatile_multiply(<4 x double>* %in) {
; CHECK-LABEL: @load_volatile_multiply( ; CHECK-LABEL: @load_volatile_multiply(
; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x double>* [[IN:%.*]] to double* ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x double>* [[IN:%.*]] to double*
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP1]] to <2 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP1]] to <2 x double>*
; CHECK-NEXT: load volatile <2 x double>, <2 x double>* [[VEC_CAST]], align 8 ; CHECK-NEXT: load volatile <2 x double>, <2 x double>* [[VEC_CAST]], align 8
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP1]], i64 2 ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP1]], i64 2
; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>* ; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
; CHECK-NEXT: load volatile <2 x double>, <2 x double>* [[VEC_CAST1]], align 8 ; CHECK-NEXT: load volatile <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
; CHECK-NOT: = load ; CHECK-NOT: = load
; ;
%in.m = load volatile <4 x double>, <4 x double>* %in, align 8 %in.m = load volatile <4 x double>, <4 x double>* %in, align 8
%res = call <4 x double> @llvm.matrix.multiply(<4 x double> %in.m, <4 x double> %in.m, i32 2, i32 2, i32 2) %res = call <4 x double> @llvm.matrix.multiply(<4 x double> %in.m, <4 x double> %in.m, i32 2, i32 2, i32 2)
ret <4 x double> %res ret <4 x double> %res
} }
declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32) declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32)
define <9 x double> @strided_load_3x3_align32(<9 x double>* %in, i64 %stride) { define <9 x double> @strided_load_3x3_align32(double* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3_align32( ; CHECK-LABEL: @strided_load_3x3_align32(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST]], align 32 ; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST]], align 32
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>*
; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST3]], align 8 ; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST3]], align 8
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]] ; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START5]] ; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* %in, i64 [[VEC_START5]]
; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>*
; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST7]], align 8 ; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST7]], align 8
; CHECK-NOT: = load ; CHECK-NOT: = load
; ;
entry: entry:
%load = call <9 x double> @llvm.matrix.column.major.load.v9f64(<9 x double>* align 32 %in, i64 %stride, i1 false, i32 3, i32 3) %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(double* align 32 %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x double> %load ret <9 x double> %load
} }
define <9 x double> @strided_load_3x3_align2(<9 x double>* %in, i64 %stride) { define <9 x double> @strided_load_3x3_align2(double* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3_align2( ; CHECK-LABEL: @strided_load_3x3_align2(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST]], align 2 ; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST]], align 2
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>*
; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST3]], align 2 ; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST3]], align 2
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]] ; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START5]] ; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* %in, i64 [[VEC_START5]]
; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>*
; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST7]], align 2 ; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST7]], align 2
; CHECK-NOT: = load ; CHECK-NOT: = load
; ;
entry: entry:
%load = call <9 x double> @llvm.matrix.column.major.load.v9f64(<9 x double>* align 2 %in, i64 %stride, i1 false, i32 3, i32 3) %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(double* align 2 %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x double> %load ret <9 x double> %load
} }
define <4 x double> @load_align2_multiply(<4 x double>* %in) { define <4 x double> @load_align2_multiply(<4 x double>* %in) {
; CHECK-LABEL: @load_align2_multiply( ; CHECK-LABEL: @load_align2_multiply(
; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x double>* [[IN:%.*]] to double* ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x double>* [[IN:%.*]] to double*
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP1]] to <2 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP1]] to <2 x double>*
; CHECK-NEXT: load <2 x double>, <2 x double>* [[VEC_CAST]], align 2 ; CHECK-NEXT: load <2 x double>, <2 x double>* [[VEC_CAST]], align 2
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP1]], i64 2 ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP1]], i64 2
; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>* ; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
; CHECK-NEXT: load <2 x double>, <2 x double>* [[VEC_CAST1]], align 2 ; CHECK-NEXT: load <2 x double>, <2 x double>* [[VEC_CAST1]], align 2
; CHECK-NOT: = load ; CHECK-NOT: = load
; ;
%in.m = load <4 x double>, <4 x double>* %in, align 2 %in.m = load <4 x double>, <4 x double>* %in, align 2
%res = call <4 x double> @llvm.matrix.multiply(<4 x double> %in.m, <4 x double> %in.m, i32 2, i32 2, i32 2) %res = call <4 x double> @llvm.matrix.multiply(<4 x double> %in.m, <4 x double> %in.m, i32 2, i32 2, i32 2)
ret <4 x double> %res ret <4 x double> %res
} }
define <6 x float> @strided_load_2x3_align16_stride2(<6 x float>* %in) { define <6 x float> @strided_load_2x3_align16_stride2(float* %in) {
; CHECK-LABEL: @strided_load_2x3_align16_stride2( ; CHECK-LABEL: @strided_load_2x3_align16_stride2(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <6 x float>* [[IN:%.*]] to float* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* %in to <2 x float>*
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[TMP0]] to <2 x float>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST]], align 16 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST]], align 16
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[TMP0]], i64 2 ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* %in, i64 2
; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast float* [[VEC_GEP]] to <2 x float>* ; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast float* [[VEC_GEP]] to <2 x float>*
; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST1]], align 8 ; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST1]], align 8
; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, float* [[TMP0]], i64 4 ; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, float* %in, i64 4
; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast float* [[VEC_GEP3]] to <2 x float>* ; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast float* [[VEC_GEP3]] to <2 x float>*
; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST4]], align 16 ; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST4]], align 16
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[COL_LOAD5]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[COL_LOAD5]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5> ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: ret <6 x float> [[TMP3]] ; CHECK-NEXT: ret <6 x float> [[TMP3]]
; ;
entry: entry:
%load = call <6 x float> @llvm.matrix.column.major.load.v6f32(<6 x float>* align 16 %in, i64 2, i1 false, i32 2, i32 3) %load = call <6 x float> @llvm.matrix.column.major.load.v6f32(float* align 16 %in, i64 2, i1 false, i32 2, i32 3)
ret <6 x float> %load ret <6 x float> %load
} }
declare <6 x float> @llvm.matrix.column.major.load.v6f32(<6 x float>*, i64, i1, i32, i32) declare <6 x float> @llvm.matrix.column.major.load.v6f32(float*, i64, i1, i32, i32)

llvm/test/Transforms/LowerMatrixIntrinsics/remarks-inlining.ll

Show First 20 Lines • Show All 86 Lines • ▼ Show 20 Lines
; CHECK-LABEL: remark: store.h:66:11: Lowered with 1 stores, 0 loads, 0 compute ops ; CHECK-LABEL: remark: store.h:66:11: Lowered with 1 stores, 0 loads, 0 compute ops
; CHECK-NEXT: store( ; CHECK-NEXT: store(
; CHECK-NEXT: scalar, ; CHECK-NEXT: scalar,
; CHECK-NEXT: addr %D) ; CHECK-NEXT: addr %D)
; CHECK-LABEL: remark: transpose.h:13:11: Lowered with 0 stores, 0 loads, 8 compute ops ; CHECK-LABEL: remark: transpose.h:13:11: Lowered with 0 stores, 0 loads, 8 compute ops
; CHECK-NEXT: transpose.1x2.float(transpose.2x1.float(addr %D)) ; CHECK-NEXT: transpose.1x2.float(transpose.2x1.float(addr %D))
define void @toplevel(<15 x double>* %A, <15 x double>* %B, <15 x double>* %C, <2 x float>* %D) !dbg !16 { define void @toplevel(<15 x double>* %A, double* %B, <15 x double>* %C, <2 x float>* %D) !dbg !16 {
entry: entry:
%a = load <15 x double>, <15 x double> *%A, align 16, !dbg !3791 %a = load <15 x double>, <15 x double> *%A, align 16, !dbg !3791
%b = call <15 x double> @llvm.matrix.column.major.load(<15 x double>* %B, i64 5, i1 false, i32 3, i32 5), !dbg !3793 %b = call <15 x double> @llvm.matrix.column.major.load(double* %B, i64 5, i1 false, i32 3, i32 5), !dbg !3793
%c = fadd <15 x double> %a, %b, !dbg !100 %c = fadd <15 x double> %a, %b, !dbg !100
store <15 x double> %c, <15 x double> *%C, align 16, !dbg !102 store <15 x double> %c, <15 x double> *%C, align 16, !dbg !102
%load = load <2 x float>, <2 x float>* %D, !dbg !104 %load = load <2 x float>, <2 x float>* %D, !dbg !104
%t1 = call <2 x float> @llvm.matrix.transpose(<2 x float> %load, i32 2, i32 1), !dbg !106 %t1 = call <2 x float> @llvm.matrix.transpose(<2 x float> %load, i32 2, i32 1), !dbg !106
%t2 = call <2 x float> @llvm.matrix.transpose(<2 x float> %t1, i32 1, i32 2), !dbg !106 %t2 = call <2 x float> @llvm.matrix.transpose(<2 x float> %t1, i32 1, i32 2), !dbg !106
store <2 x float> %t2, <2 x float>* %D, !dbg !108 store <2 x float> %t2, <2 x float>* %D, !dbg !108
ret void ret void
} }
declare <15 x double> @llvm.matrix.column.major.load(<15 x double>*, i64, i1, i32, i32) declare <15 x double> @llvm.matrix.column.major.load(double*, i64, i1, i32, i32)
declare <2 x float> @llvm.matrix.transpose(<2 x float>, i32, i32) declare <2 x float> @llvm.matrix.transpose(<2 x float>, i32, i32)
!llvm.dbg.cu = !{!0} !llvm.dbg.cu = !{!0}
!llvm.module.flags = !{!3, !4} !llvm.module.flags = !{!3, !4}
!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
!1 = !DIFile(filename: "load.h", directory: "/test") !1 = !DIFile(filename: "load.h", directory: "/test")
!2 = !{} !2 = !{}
▲ Show 20 Lines • Show All 49 Lines • Show Last 20 Lines

llvm/test/Transforms/LowerMatrixIntrinsics/remarks.ll

Show All 9 Lines
; CHECK-NEXT: addr %B) ; CHECK-NEXT: addr %B)
define void @transpose(<12 x double>* %A, <12 x double>* %B) !dbg !23 { define void @transpose(<12 x double>* %A, <12 x double>* %B) !dbg !23 {
%load = load <12 x double>, <12 x double>* %A, !dbg !24 %load = load <12 x double>, <12 x double>* %A, !dbg !24
%t = call <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double> %load, i32 2, i32 6), !dbg !24 %t = call <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double> %load, i32 2, i32 6), !dbg !24
store <12 x double> %t, <12 x double>* %B, !dbg !24 store <12 x double> %t, <12 x double>* %B, !dbg !24
ret void ret void
} }
declare <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double>, i32, i32)
; CHECK-LABEL: remark: test.h:50:20: Lowered with 2 stores, 12 loads, 22 compute ops ; CHECK-LABEL: remark: test.h:50:20: Lowered with 2 stores, 12 loads, 22 compute ops
; CHECK-NEXT: store( ; CHECK-NEXT: store(
; CHECK-NEXT: multiply.2x6.6x2.double( ; CHECK-NEXT: multiply.2x6.6x2.double(
; CHECK-NEXT: load(addr %A), ; CHECK-NEXT: load(addr %A),
; CHECK-NEXT: load(addr %B)), ; CHECK-NEXT: load(addr %B)),
; CHECK-NEXT: addr %C) ; CHECK-NEXT: addr %C)
define void @multiply(<12 x double>* %A, <12 x double>* %B, <4 x double>* %C) !dbg !25 { define void @multiply(<12 x double>* %A, <12 x double>* %B, <4 x double>* %C) !dbg !25 {
%A.matrix = load <12 x double>, <12 x double>* %A, !dbg !26 %A.matrix = load <12 x double>, <12 x double>* %A, !dbg !26
%B.matrix = load <12 x double>, <12 x double>* %B, !dbg !26 %B.matrix = load <12 x double>, <12 x double>* %B, !dbg !26
%t = call <4 x double> @llvm.matrix.multiply(<12 x double> %A.matrix, <12 x double> %B.matrix, i32 2, i32 6, i32 2), !dbg !26 %t = call <4 x double> @llvm.matrix.multiply(<12 x double> %A.matrix, <12 x double> %B.matrix, i32 2, i32 6, i32 2), !dbg !26
store <4 x double> %t, <4 x double>* %C, !dbg !26 store <4 x double> %t, <4 x double>* %C, !dbg !26
ret void ret void
} }
declare <4 x double> @llvm.matrix.multiply(<12 x double>, <12 x double>, i32, i32, i32)
; CHECK-LABEL: remark: test.h:60:20: Lowered with 6 stores, 6 loads, 0 compute ops ; CHECK-LABEL: remark: test.h:60:20: Lowered with 6 stores, 6 loads, 0 compute ops
; CHECK-NEXT: store( ; CHECK-NEXT: store(
; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5), ; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5),
; CHECK-NEXT: addr %B) ; CHECK-NEXT: addr %B)
define void @column.major.load(<9 x double>* %A, <9 x double>* %B) !dbg !27 { define void @column.major.load(double* %A, <9 x double>* %B) !dbg !27 {
%A.matrix = call <9 x double> @llvm.matrix.column.major.load(<9 x double>* %A, i64 5, i1 false, i32 3, i32 3), !dbg !28 %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !28
store <9 x double> %A.matrix, <9 x double>* %B, !dbg !28 store <9 x double> %A.matrix, <9 x double>* %B, !dbg !28
ret void ret void
} }
declare <9 x double> @llvm.matrix.column.major.load(<9 x double>*, i64, i1, i32, i32)
; CHECK-LABEL: remark: test.h:70:20: Lowered with 6 stores, 6 loads, 0 compute ops ; CHECK-LABEL: remark: test.h:70:20: Lowered with 6 stores, 6 loads, 0 compute ops
; CHECK-NEXT: column.major.store.3x3.double( ; CHECK-NEXT: column.major.store.3x3.double(
; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5), ; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5),
; CHECK-NEXT: addr %B, ; CHECK-NEXT: addr %B,
; CHECK-NEXT: 10) ; CHECK-NEXT: 10)
define void @column.major.store(<9 x double>* %A, <9 x double>* %B) !dbg !29 { define void @column.major.store(double* %A, double* %B) !dbg !29 {
%A.matrix = call <9 x double> @llvm.matrix.column.major.load(<9 x double>* %A, i64 5, i1 false, i32 3, i32 3), !dbg !30 %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !30
call void @llvm.matrix.column.major.store(<9 x double> %A.matrix, <9 x double>* %B, i64 10, i1 false, i32 3, i32 3), !dbg !30 call void @llvm.matrix.column.major.store(<9 x double> %A.matrix, double* %B, i64 10, i1 false, i32 3, i32 3), !dbg !30
ret void ret void
} }
declare void @llvm.matrix.column.major.store(<9 x double>, <9 x double>*, i64, i1, i32, i32)
; CHECK-LABEL: remark: test.h:80:20: Lowered with 6 stores, 6 loads, 12 compute ops ; CHECK-LABEL: remark: test.h:80:20: Lowered with 6 stores, 6 loads, 12 compute ops
; CHECK-NEXT: column.major.store.3x3.double( ; CHECK-NEXT: column.major.store.3x3.double(
; CHECK-NEXT: fmul( ; CHECK-NEXT: fmul(
; CHECK-NEXT: fadd( ; CHECK-NEXT: fadd(
; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5) ; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5)
; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)), ; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)),
; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)), ; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)),
; CHECK-NEXT: addr %B, ; CHECK-NEXT: addr %B,
; CHECK-NEXT: 10) ; CHECK-NEXT: 10)
define void @binaryops(<9 x double>* %A, <9 x double>* %B) !dbg !31 { define void @binaryops(double* %A, double* %B) !dbg !31 {
%A.matrix = call <9 x double> @llvm.matrix.column.major.load(<9 x double>* %A, i64 5, i1 false, i32 3, i32 3), !dbg !32 %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !32
%R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !32 %R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !32
%R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !32 %R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !32
call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, <9 x double>* %B, i64 10, i1 false, i32 3, i32 3), !dbg !32 call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, double* %B, i64 10, i1 false, i32 3, i32 3), !dbg !32
ret void ret void
} }
; CHECK-LABEL: remark: test.h:90:20: Lowered with 6 stores, 6 loads, 12 compute ops ; CHECK-LABEL: remark: test.h:90:20: Lowered with 6 stores, 6 loads, 12 compute ops
; CHECK-NEXT: column.major.store.3x3.double( ; CHECK-NEXT: column.major.store.3x3.double(
; CHECK-NEXT: fmul( ; CHECK-NEXT: fmul(
; CHECK-NEXT: fadd( ; CHECK-NEXT: fadd(
; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5) ; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5)
; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)), ; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)),
; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)), ; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)),
; CHECK-NEXT: addr %B, ; CHECK-NEXT: addr %B,
; CHECK-NEXT: 10) ; CHECK-NEXT: 10)
; CHECK-NEXT: remark: test.h:90:20: Lowered with 2 stores, 12 loads, 22 compute ops ; CHECK-NEXT: remark: test.h:90:20: Lowered with 2 stores, 12 loads, 22 compute ops
; CHECK-NEXT: store( ; CHECK-NEXT: store(
; CHECK-NEXT: multiply.2x6.6x2.double( ; CHECK-NEXT: multiply.2x6.6x2.double(
; CHECK-NEXT: load(addr %C), ; CHECK-NEXT: load(addr %C),
; CHECK-NEXT: load(addr %D)), ; CHECK-NEXT: load(addr %D)),
; CHECK-NEXT: addr %E) ; CHECK-NEXT: addr %E)
define void @multiple_expressions(<9 x double>* %A, <9 x double>* %B, <12 x double>* %C, <12 x double>* %D, <4 x double>* %E) !dbg !33 { define void @multiple_expressions(double* %A, double* %B, <12 x double>* %C, <12 x double>* %D, <4 x double>* %E) !dbg !33 {
%A.matrix = call <9 x double> @llvm.matrix.column.major.load(<9 x double>* %A, i64 5, i1 false, i32 3, i32 3), !dbg !34 %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !34
%R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !34 %R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !34
%R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !34 %R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !34
call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, <9 x double>* %B, i64 10, i1 false, i32 3, i32 3), !dbg !34 call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, double* %B, i64 10, i1 false, i32 3, i32 3), !dbg !34
%C.matrix = load <12 x double>, <12 x double>* %C, !dbg !34 %C.matrix = load <12 x double>, <12 x double>* %C, !dbg !34
%D.matrix = load <12 x double>, <12 x double>* %D, !dbg !34 %D.matrix = load <12 x double>, <12 x double>* %D, !dbg !34
%Mult.matrix = call <4 x double> @llvm.matrix.multiply(<12 x double> %C.matrix, <12 x double> %D.matrix, i32 2, i32 6, i32 2), !dbg !34 %Mult.matrix = call <4 x double> @llvm.matrix.multiply(<12 x double> %C.matrix, <12 x double> %D.matrix, i32 2, i32 6, i32 2), !dbg !34
store <4 x double> %Mult.matrix, <4 x double>* %E, !dbg !34 store <4 x double> %Mult.matrix, <4 x double>* %E, !dbg !34
ret void ret void
} }
; CHECK-LABEL: remark: test.h:100:20: Lowered with 6 stores, 6 loads, 12 compute ops ; CHECK-LABEL: remark: test.h:100:20: Lowered with 6 stores, 6 loads, 12 compute ops
; CHECK-NEXT: column.major.store.3x3.double( ; CHECK-NEXT: column.major.store.3x3.double(
; CHECK-NEXT: fmul( ; CHECK-NEXT: fmul(
; CHECK-NEXT: fadd( ; CHECK-NEXT: fadd(
; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5) ; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5)
; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)), ; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)),
; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)), ; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)),
; CHECK-NEXT: stack addr %B, ; CHECK-NEXT: addr %B,
; CHECK-NEXT: 10) ; CHECK-NEXT: 10)
define void @stackaddresses(<9 x double>* %A) !dbg !35 { define void @stackaddresses(double* %A, double* %B) !dbg !35 {
%B = alloca <9 x double> %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !36
%A.matrix = call <9 x double> @llvm.matrix.column.major.load(<9 x double>* %A, i64 5, i1 false, i32 3, i32 3), !dbg !36
%R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !36 %R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !36
%R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !36 %R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !36
call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, <9 x double>* %B, i64 10, i1 false, i32 3, i32 3), !dbg !36 call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, double* %B, i64 10, i1 false, i32 3, i32 3), !dbg !36
ret void ret void
} }
; CHECK-LABEL: remark: test.h:30:20: Lowered with 10 stores, 9 loads, 30 compute ops ; CHECK-LABEL: remark: test.h:30:20: Lowered with 10 stores, 9 loads, 30 compute ops
; CHECK-NEXT: store( ; CHECK-NEXT: store(
; CHECK-NEXT: transpose.5x3.double(load(addr %A)), ; CHECK-NEXT: transpose.5x3.double(load(addr %A)),
; CHECK-NEXT: stack addr %s1) ; CHECK-NEXT: stack addr %s1)
%S1 = type {<15 x double>*} %S1 = type {<15 x double>*}
define void @get_underlying_object(%S1* %A) !dbg !21 { define void @get_underlying_object(%S1* %A) !dbg !21 {
entry: entry:
%s1 = alloca <15 x double>, !dbg !22 %s1 = alloca <15 x double>, !dbg !22
%a1 = getelementptr %S1, %S1* %A, i32 0, i32 0, !dbg !22 %a1 = getelementptr %S1, %S1* %A, i32 0, i32 0, !dbg !22
%a2 = load <15 x double>*, <15 x double>** %a1, !dbg !22 %a2 = load <15 x double>*, <15 x double>** %a1, !dbg !22
%av = load <15 x double>, <15 x double>* %a2, !dbg !22 %av = load <15 x double>, <15 x double>* %a2, !dbg !22
%s2 = bitcast <15 x double>* %s1 to i64*, !dbg !22 %s2 = bitcast <15 x double>* %s1 to i64*, !dbg !22
%s3 = bitcast i64* %s2 to <15 x double>*, !dbg !22 %s3 = bitcast i64* %s2 to <15 x double>*, !dbg !22
%t = call <15 x double> @llvm.matrix.transpose.v15f64.v15f64(<15 x double> %av, i32 5, i32 3), !dbg !22 %t = call <15 x double> @llvm.matrix.transpose.v15f64.v15f64(<15 x double> %av, i32 5, i32 3), !dbg !22
store <15 x double> %t, <15 x double>* %s3, !dbg !22 store <15 x double> %t, <15 x double>* %s3, !dbg !22
ret void ret void
} }
declare <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double>, i32, i32)
declare <4 x double> @llvm.matrix.multiply(<12 x double>, <12 x double>, i32, i32, i32)
declare <9 x double> @llvm.matrix.column.major.load(double*, i64, i1, i32, i32)
declare <15 x double> @llvm.matrix.transpose.v15f64.v15f64(<15 x double>, i32, i32) declare <15 x double> @llvm.matrix.transpose.v15f64.v15f64(<15 x double>, i32, i32)
declare void @llvm.matrix.column.major.store(<9 x double>, double*, i64, i1, i32, i32)
!llvm.dbg.cu = !{!0} !llvm.dbg.cu = !{!0}
!llvm.module.flags = !{!3, !4} !llvm.module.flags = !{!3, !4}
!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
!1 = !DIFile(filename: "test.h", directory: "/test") !1 = !DIFile(filename: "test.h", directory: "/test")
!2 = !{} !2 = !{}
!3 = !{i32 2, !"Dwarf Version", i32 4} !3 = !{i32 2, !"Dwarf Version", i32 4}
Show All 39 Lines

llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
define <9 x double> @strided_load_3x3(<9 x double>* %in, i64 %stride) { define <9 x double> @strided_load_3x3(double* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3( ; CHECK-LABEL: @strided_load_3x3(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 8 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 8
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>*
; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST3]], align 8 ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST3]], align 8
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]] ; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START5]] ; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* %in, i64 [[VEC_START5]]
; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>*
; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST7]], align 8 ; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST7]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD]], <3 x double> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD]], <3 x double> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x double> [[COL_LOAD8]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x double> [[COL_LOAD8]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x double> [[TMP1]], <6 x double> [[TMP2]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x double> [[TMP1]], <6 x double> [[TMP2]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: ret <9 x double> [[TMP3]] ; CHECK-NEXT: ret <9 x double> [[TMP3]]
; ;
entry: entry:
%load = call <9 x double> @llvm.matrix.column.major.load(<9 x double>* %in, i64 %stride, i1 false, i32 3, i32 3) %load = call <9 x double> @llvm.matrix.column.major.load(double* %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x double> %load ret <9 x double> %load
} }
declare <9 x double> @llvm.matrix.column.major.load(<9 x double>*, i64, i1, i32, i32) declare <9 x double> @llvm.matrix.column.major.load(double*, i64, i1, i32, i32)
define <9 x double> @strided_load_9x1(<9 x double>* %in, i64 %stride) { define <9 x double> @strided_load_9x1(double* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_9x1( ; CHECK-LABEL: @strided_load_9x1(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[IN:%.*]] to double*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <9 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <9 x double>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x double>, <9 x double>* [[VEC_CAST]], align 8 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x double>, <9 x double>* [[VEC_CAST]], align 8
; CHECK-NEXT: ret <9 x double> [[COL_LOAD]] ; CHECK-NEXT: ret <9 x double> [[COL_LOAD]]
; ;
entry: entry:
%load = call <9 x double> @llvm.matrix.column.major.load(<9 x double>* %in, i64 %stride, i1 false, i32 9, i32 1) %load = call <9 x double> @llvm.matrix.column.major.load(double* %in, i64 %stride, i1 false, i32 9, i32 1)
ret <9 x double> %load ret <9 x double> %load
} }
declare <8 x double> @llvm.matrix.column.major.load.v8f64(<8 x double>*, i64, i1, i32, i32) declare <8 x double> @llvm.matrix.column.major.load.v8f64(double*, i64, i1, i32, i32)
; CHECK: declare <8 x double> @llvm.matrix.column.major.load.v8f64(double* nocapture, i64, i1 immarg, i32 immarg, i32 immarg) [[READONLY:#[0-9]]]
define <8 x double> @strided_load_4x2(<8 x double>* %in, i64 %stride) { define <8 x double> @strided_load_4x2(double* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_4x2( ; CHECK-LABEL: @strided_load_4x2(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x double>* [[IN:%.*]] to double*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <4 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <4 x double>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST]], align 8 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST]], align 8
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <4 x double>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <4 x double>*
; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST3]], align 8 ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST3]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[COL_LOAD]], <4 x double> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[COL_LOAD]], <4 x double> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: ret <8 x double> [[TMP1]] ; CHECK-NEXT: ret <8 x double> [[TMP1]]
; ;
entry: entry:
%load = call <8 x double> @llvm.matrix.column.major.load.v8f64(<8 x double>* %in, i64 %stride, i1 false, i32 4, i32 2) %load = call <8 x double> @llvm.matrix.column.major.load.v8f64(double* %in, i64 %stride, i1 false, i32 4, i32 2)
ret <8 x double> %load ret <8 x double> %load
} }
; CHECK: declare <9 x double> @llvm.matrix.column.major.load.v9f64.p0v9f64(<9 x double>* nocapture, i64, i1 immarg, i32 immarg, i32 immarg) [[READONLY:#[0-9]]] ; CHECK: declare <9 x double> @llvm.matrix.column.major.load.v9f64(double* nocapture, i64, i1 immarg, i32 immarg, i32 immarg) [[READONLY]]
; CHECK: declare <8 x double> @llvm.matrix.column.major.load.v8f64.p0v8f64(<8 x double>* nocapture, i64, i1 immarg, i32 immarg, i32 immarg) [[READONLY]]
; CHECK: attributes [[READONLY]] = { argmemonly nosync nounwind readonly willreturn } ; CHECK: attributes [[READONLY]] = { argmemonly nosync nounwind readonly willreturn }

llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
define <9 x float> @strided_load_3x3(<9 x float>* %in, i64 %stride) { define <9 x float> @strided_load_3x3(float* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3( ; CHECK-LABEL: @strided_load_3x3(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x float>* [[IN:%.*]] to float*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <3 x float>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <3 x float>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST]], align 4 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST]], align 4
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr float, float* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr float, float* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast float* [[VEC_GEP2]] to <3 x float>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast float* [[VEC_GEP2]] to <3 x float>*
; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST3]], align 4 ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST3]], align 4
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]] ; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr float, float* [[TMP0]], i64 [[VEC_START5]] ; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr float, float* %in, i64 [[VEC_START5]]
; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast float* [[VEC_GEP6]] to <3 x float>* ; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast float* [[VEC_GEP6]] to <3 x float>*
; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST7]], align 4 ; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST7]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x float> [[COL_LOAD]], <3 x float> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x float> [[COL_LOAD]], <3 x float> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x float> [[COL_LOAD8]], <3 x float> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x float> [[COL_LOAD8]], <3 x float> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x float> [[TMP1]], <6 x float> [[TMP2]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x float> [[TMP1]], <6 x float> [[TMP2]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: ret <9 x float> [[TMP3]] ; CHECK-NEXT: ret <9 x float> [[TMP3]]
; ;
entry: entry:
%load = call <9 x float> @llvm.matrix.column.major.load(<9 x float>* %in, i64 %stride, i1 false, i32 3, i32 3) %load = call <9 x float> @llvm.matrix.column.major.load(float* %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x float> %load ret <9 x float> %load
} }
declare <9 x float> @llvm.matrix.column.major.load(<9 x float>*, i64, i1, i32, i32) declare <9 x float> @llvm.matrix.column.major.load(float*, i64, i1, i32, i32)
define <9 x float> @strided_load_9x1(<9 x float>* %in, i64 %stride) { define <9 x float> @strided_load_9x1(float* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_9x1( ; CHECK-LABEL: @strided_load_9x1(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x float>* [[IN:%.*]] to float*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <9 x float>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <9 x float>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x float>, <9 x float>* [[VEC_CAST]], align 4 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x float>, <9 x float>* [[VEC_CAST]], align 4
; CHECK-NEXT: ret <9 x float> [[COL_LOAD]] ; CHECK-NEXT: ret <9 x float> [[COL_LOAD]]
; ;
entry: entry:
%load = call <9 x float> @llvm.matrix.column.major.load(<9 x float>* %in, i64 %stride, i1 false, i32 9, i32 1) %load = call <9 x float> @llvm.matrix.column.major.load(float* %in, i64 %stride, i1 false, i32 9, i32 1)
ret <9 x float> %load ret <9 x float> %load
} }
declare <8 x float> @llvm.matrix.column.major.load.v8f32(<8 x float>*, i64, i1, i32, i32) declare <8 x float> @llvm.matrix.column.major.load.v8f32(float*, i64, i1, i32, i32)
define <8 x float> @strided_load_4x2(<8 x float>* %in, i64 %stride) { define <8 x float> @strided_load_4x2(float* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_4x2( ; CHECK-LABEL: @strided_load_4x2(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x float>* [[IN:%.*]] to float*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <4 x float>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <4 x float>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x float>, <4 x float>* [[VEC_CAST]], align 4 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x float>, <4 x float>* [[VEC_CAST]], align 4
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr float, float* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr float, float* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast float* [[VEC_GEP2]] to <4 x float>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast float* [[VEC_GEP2]] to <4 x float>*
; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x float>, <4 x float>* [[VEC_CAST3]], align 4 ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x float>, <4 x float>* [[VEC_CAST3]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[COL_LOAD]], <4 x float> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[COL_LOAD]], <4 x float> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: ret <8 x float> [[TMP1]] ; CHECK-NEXT: ret <8 x float> [[TMP1]]
; ;
entry: entry:
%load = call <8 x float> @llvm.matrix.column.major.load.v8f32(<8 x float>* %in, i64 %stride, i1 false, i32 4, i32 2) %load = call <8 x float> @llvm.matrix.column.major.load.v8f32(float* %in, i64 %stride, i1 false, i32 4, i32 2)
ret <8 x float> %load ret <8 x float> %load
} }

llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
define <9 x i32> @strided_load_3x3(<9 x i32>* %in, i64 %stride) { define <9 x i32> @strided_load_3x3(i32* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3( ; CHECK-LABEL: @strided_load_3x3(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x i32>* [[IN:%.*]] to i32*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <3 x i32>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <3 x i32>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST]], align 4 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST]], align 4
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, i32* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast i32* [[VEC_GEP2]] to <3 x i32>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast i32* [[VEC_GEP2]] to <3 x i32>*
; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST3]], align 4 ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST3]], align 4
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]] ; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[VEC_START5]] ; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr i32, i32* %in, i64 [[VEC_START5]]
; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast i32* [[VEC_GEP6]] to <3 x i32>* ; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast i32* [[VEC_GEP6]] to <3 x i32>*
; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST7]], align 4 ; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST7]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x i32> [[COL_LOAD]], <3 x i32> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x i32> [[COL_LOAD]], <3 x i32> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x i32> [[COL_LOAD8]], <3 x i32> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x i32> [[COL_LOAD8]], <3 x i32> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x i32> [[TMP1]], <6 x i32> [[TMP2]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x i32> [[TMP1]], <6 x i32> [[TMP2]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: ret <9 x i32> [[TMP3]] ; CHECK-NEXT: ret <9 x i32> [[TMP3]]
; ;
entry: entry:
%load = call <9 x i32> @llvm.matrix.column.major.load(<9 x i32>* %in, i64 %stride, i1 false, i32 3, i32 3) %load = call <9 x i32> @llvm.matrix.column.major.load(i32* %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x i32> %load ret <9 x i32> %load
} }
declare <9 x i32> @llvm.matrix.column.major.load(<9 x i32>*, i64, i1, i32, i32) declare <9 x i32> @llvm.matrix.column.major.load(i32*, i64, i1, i32, i32)
define <9 x i32> @strided_load_9x1(<9 x i32>* %in, i64 %stride) { define <9 x i32> @strided_load_9x1(i32* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_9x1( ; CHECK-LABEL: @strided_load_9x1(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x i32>* [[IN:%.*]] to i32*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <9 x i32>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <9 x i32>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x i32>, <9 x i32>* [[VEC_CAST]], align 4 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x i32>, <9 x i32>* [[VEC_CAST]], align 4
; CHECK-NEXT: ret <9 x i32> [[COL_LOAD]] ; CHECK-NEXT: ret <9 x i32> [[COL_LOAD]]
; ;
entry: entry:
%load = call <9 x i32> @llvm.matrix.column.major.load(<9 x i32>* %in, i64 %stride, i1 false, i32 9, i32 1) %load = call <9 x i32> @llvm.matrix.column.major.load(i32* %in, i64 %stride, i1 false, i32 9, i32 1)
ret <9 x i32> %load ret <9 x i32> %load
} }
declare <8 x i32> @llvm.matrix.column.major.load.v8i32(<8 x i32>*, i64, i1, i32, i32) declare <8 x i32> @llvm.matrix.column.major.load.v8i32(i32*, i64, i1, i32, i32)
define <8 x i32> @strided_load_4x2(<8 x i32>* %in, i64 %stride) { define <8 x i32> @strided_load_4x2(i32* %in, i64 %stride) {
; CHECK-LABEL: @strided_load_4x2( ; CHECK-LABEL: @strided_load_4x2(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x i32>* [[IN:%.*]] to i32*
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* %in, i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <4 x i32>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <4 x i32>*
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[VEC_CAST]], align 4 ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[VEC_CAST]], align 4
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, i32* [[TMP0]], i64 [[VEC_START1]] ; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, i32* %in, i64 [[VEC_START1]]
; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast i32* [[VEC_GEP2]] to <4 x i32>* ; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast i32* [[VEC_GEP2]] to <4 x i32>*
; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x i32>, <4 x i32>* [[VEC_CAST3]], align 4 ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x i32>, <4 x i32>* [[VEC_CAST3]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[COL_LOAD]], <4 x i32> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[COL_LOAD]], <4 x i32> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: ret <8 x i32> [[TMP1]] ; CHECK-NEXT: ret <8 x i32> [[TMP1]]
; ;
entry: entry:
%load = call <8 x i32> @llvm.matrix.column.major.load.v8i32(<8 x i32>* %in, i64 %stride, i1 false, i32 4, i32 2) %load = call <8 x i32> @llvm.matrix.column.major.load.v8i32(i32* %in, i64 %stride, i1 false, i32 4, i32 2)
ret <8 x i32> %load ret <8 x i32> %load
} }

llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s ; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
define void @strided_store_3x2(<6 x double> %in, double* %out) { define void @strided_store_3x2(<6 x double> %in, double* %out) {
; CHECK-LABEL: @strided_store_3x2( ; CHECK-LABEL: @strided_store_3x2(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> ; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> ; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[OUT:%.*]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[OUT:%.*]] to <3 x double>*
; CHECK-NEXT: store <3 x double> [[SPLIT]], <3 x double>* [[VEC_CAST]], align 8 ; CHECK-NEXT: store <3 x double> [[SPLIT]], <3 x double>* [[VEC_CAST]], align 8
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[OUT]], i64 5 ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[OUT]], i64 5
; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
; CHECK-NEXT: store <3 x double> [[SPLIT1]], <3 x double>* [[VEC_CAST2]], align 8 ; CHECK-NEXT: store <3 x double> [[SPLIT1]], <3 x double>* [[VEC_CAST2]], align 8
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
call void @llvm.matrix.column.major.store(<6 x double> %in, double* %out, i64 5, i1 false, i32 3, i32 2) call void @llvm.matrix.column.major.store.v6f64(<6 x double> %in, double* %out, i64 5, i1 false, i32 3, i32 2)
ret void ret void
} }
define void @strided_store_3x2_nonconst_stride(<6 x double> %in, i64 %stride, double* %out) { define void @strided_store_3x2_nonconst_stride(<6 x double> %in, i64 %stride, double* %out) {
; CHECK-LABEL: @strided_store_3x2_nonconst_stride( ; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> ; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> undef, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> ; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> undef, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]] ; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[OUT:%.*]], i64 [[VEC_START]] ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[OUT:%.*]], i64 [[VEC_START]]
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
; CHECK-NEXT: store <3 x double> [[SPLIT]], <3 x double>* [[VEC_CAST]], align 8 ; CHECK-NEXT: store <3 x double> [[SPLIT]], <3 x double>* [[VEC_CAST]], align 8
; CHECK-NEXT: [[VEC_START2:%.*]] = mul i64 1, [[STRIDE]] ; CHECK-NEXT: [[VEC_START2:%.*]] = mul i64 1, [[STRIDE]]
; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[OUT]], i64 [[VEC_START2]] ; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[OUT]], i64 [[VEC_START2]]
; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>* ; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
; CHECK-NEXT: store <3 x double> [[SPLIT1]], <3 x double>* [[VEC_CAST4]], align 8 ; CHECK-NEXT: store <3 x double> [[SPLIT1]], <3 x double>* [[VEC_CAST4]], align 8
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
call void @llvm.matrix.column.major.store(<6 x double> %in, double* %out, i64 %stride, i1 false, i32 3, i32 2) call void @llvm.matrix.column.major.store.v6f64(<6 x double> %in, double* %out, i64 %stride, i1 false, i32 3, i32 2)
ret void ret void
} }
declare void @llvm.matrix.column.major.store(<6 x double>, double*, i64, i1, i32, i32)
define void @strided_store_2x3(<10 x double> %in, double* %out) { define void @strided_store_2x3(<10 x double> %in, double* %out) {
; CHECK-LABEL: @strided_store_2x3( ; CHECK-LABEL: @strided_store_2x3(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <10 x double> [[IN:%.*]], <10 x double> undef, <2 x i32> <i32 0, i32 1> ; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <10 x double> [[IN:%.*]], <10 x double> undef, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 2, i32 3> ; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 4, i32 5> ; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 6, i32 7> ; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 6, i32 7>
; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 8, i32 9> ; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> undef, <2 x i32> <i32 8, i32 9>
; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[OUT:%.*]] to <2 x double>* ; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[OUT:%.*]] to <2 x double>*
Show All 11 Lines
; CHECK-NEXT: [[VEC_CAST11:%.*]] = bitcast double* [[VEC_GEP10]] to <2 x double>* ; CHECK-NEXT: [[VEC_CAST11:%.*]] = bitcast double* [[VEC_GEP10]] to <2 x double>*
; CHECK-NEXT: store <2 x double> [[SPLIT4]], <2 x double>* [[VEC_CAST11]], align 8 ; CHECK-NEXT: store <2 x double> [[SPLIT4]], <2 x double>* [[VEC_CAST11]], align 8
; CHECK-NEXT: ret void ; CHECK-NEXT: ret void
; ;
call void @llvm.matrix.column.major.store.v10f64(<10 x double> %in, double* %out, i64 4, i1 false, i32 2, i32 5) call void @llvm.matrix.column.major.store.v10f64(<10 x double> %in, double* %out, i64 4, i1 false, i32 2, i32 5)
ret void ret void
} }
declare void @llvm.matrix.column.major.store.v6f64(<6 x double>, double*, i64, i1, i32, i32)
declare void @llvm.matrix.column.major.store.v10f64(<10 x double>, double*, i64, i1, i32, i32) declare void @llvm.matrix.column.major.store.v10f64(<10 x double>, double*, i64, i1, i32, i32)
; CHECK: declare void @llvm.matrix.column.major.store.v6f64.p0f64(<6 x double>, double* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) [[WRITEONLY:#[0-9]]] ; CHECK: declare void @llvm.matrix.column.major.store.v6f64(<6 x double>, double* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) #0
; CHECK: declare void @llvm.matrix.column.major.store.v10f64(<10 x double>, double* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) #0
; CHECK: declare void @llvm.matrix.column.major.store.v10f64.p0f64(<10 x double>, double* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) [[WRITEONLY]] ; CHECK: attributes #0 = { argmemonly nosync nounwind willreturn writeonly }
; CHECK: attributes [[WRITEONLY]] = { argmemonly nosync nounwind willreturn writeonly }

llvm/test/Verifier/matrix-intrinsics.ll

; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s ; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
declare <4 x float> @llvm.matrix.transpose.v4f32(<4 x float>, i32, i32)
define <4 x float> @transpose(<4 x float> %m, i32 %arg) { define <4 x float> @transpose(<4 x float> %m, i32 %arg) {
; CHECK: assembly parsed, but does not verify as correct! ; CHECK: assembly parsed, but does not verify as correct!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: immarg operand has non-immediate parameter ; CHECK-NEXT: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %arg ; CHECK-NEXT: i32 %arg
; CHECK-NEXT: %result.3 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.2, i32 %arg, i32 2) ; CHECK-NEXT: %result.3 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.2, i32 %arg, i32 2)
; CHECK-NEXT: immarg operand has non-immediate parameter ; CHECK-NEXT: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %arg ; CHECK-NEXT: i32 %arg
; CHECK-NEXT: %result.4 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.3, i32 2, i32 %arg) ; CHECK-NEXT: %result.4 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.3, i32 2, i32 %arg)
%result.0 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %m, i32 0, i32 0) %result.0 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %m, i32 0, i32 0)
%result.1 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.0, i32 3, i32 2) %result.1 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.0, i32 3, i32 2)
%result.2 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.1, i32 2, i32 1) %result.2 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.1, i32 2, i32 1)
%result.3 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.2, i32 %arg, i32 2) %result.3 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.2, i32 %arg, i32 2)
%result.4 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.3, i32 2, i32 %arg) %result.4 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.3, i32 2, i32 %arg)
ret <4 x float> %result.4 ret <4 x float> %result.4
} }
declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
define <4 x float> @multiply(<4 x float> %m, i32 %arg) { define <4 x float> @multiply(<4 x float> %m, i32 %arg) {
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: immarg operand has non-immediate parameter ; CHECK-NEXT: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %arg ; CHECK-NEXT: i32 %arg
; CHECK-NEXT: %result.3 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.2, <4 x float> %m, i32 %arg, i32 2, i32 1) ; CHECK-NEXT: %result.3 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.2, <4 x float> %m, i32 %arg, i32 2, i32 1)
%result.0 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %m, <4 x float> %m, i32 0, i32 0, i32 0) %result.0 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %m, <4 x float> %m, i32 0, i32 0, i32 0)
%result.1 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.0, <4 x float> %m, i32 3, i32 2, i32 2) %result.1 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.0, <4 x float> %m, i32 3, i32 2, i32 2)
%result.2 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.1, <4 x float> %m, i32 2, i32 2, i32 1) %result.2 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.1, <4 x float> %m, i32 2, i32 2, i32 1)
%result.3 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.2, <4 x float> %m, i32 %arg, i32 2, i32 1) %result.3 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.2, <4 x float> %m, i32 %arg, i32 2, i32 1)
ret <4 x float> %result.3 ret <4 x float> %result.3
} }
declare <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4f32(<4 x float>*, i64, i1, i32, i32) define <4 x float> @column.major_load(float* %m, float* %n, i32 %arg) {
declare <6 x float> @llvm.matrix.column.major.load.v6f32.p0v6f32(<6 x float>*, i64, i1, i32, i32)
define <4 x float> @column.major_load(<4 x float>* %m, <6 x float>* %n, i32 %arg) {
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: immarg operand has non-immediate parameter ; CHECK-NEXT: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %arg ; CHECK-NEXT: i32 %arg
; CHECK-NEXT: %result.3 = call <6 x float> @llvm.matrix.column.major.load.v6f32.p0v6f32(<6 x float>* %n, i64 2, i1 true, i32 3, i32 %arg) ; CHECK-NEXT: %result.3 = call <6 x float> @llvm.matrix.column.major.load.v6f32(float* %n, i64 2, i1 true, i32 3, i32 %arg)
%result.0 = call <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4f32(<4 x float>* %m, i64 0, i1 false, i32 0, i32 0) %result.0 = call <4 x float> @llvm.matrix.column.major.load.v4f32(float* %m, i64 0, i1 false, i32 0, i32 0)
%result.1 = call <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4f32(<4 x float>* %m, i64 2, i1 false, i32 1, i32 2) %result.1 = call <4 x float> @llvm.matrix.column.major.load.v4f32(float* %m, i64 2, i1 false, i32 1, i32 2)
%result.2 = call <6 x float> @llvm.matrix.column.major.load.v6f32.p0v6f32(<6 x float>* %n, i64 2, i1 true, i32 3, i32 3) %result.2 = call <6 x float> @llvm.matrix.column.major.load.v6f32(float* %n, i64 2, i1 true, i32 3, i32 3)
%result.3 = call <6 x float> @llvm.matrix.column.major.load.v6f32.p0v6f32(<6 x float>* %n, i64 2, i1 true, i32 3, i32 %arg) %result.3 = call <6 x float> @llvm.matrix.column.major.load.v6f32(float* %n, i64 2, i1 true, i32 3, i32 %arg)
ret <4 x float> %result.1 ret <4 x float> %result.1
} }
declare void @llvm.matrix.column.major.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i64, i1, i32, i32) define void @column.major_store(float* %m, float* %n, i64 %arg) {
declare void @llvm.matrix.column.major.store.v6f32.p0v6f32(<6 x float>, <6 x float>*, i64, i1, i32, i32)
define void @column.major_store(<4 x float>* %m, <6 x float>* %n, i64 %arg) {
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector! ; CHECK-NEXT: Result of a matrix operation does not fit in the returned vector!
call void @llvm.matrix.column.major.store.v4f32.p0v4f32(<4 x float> zeroinitializer, <4 x float>* %m, i64 0, i1 false, i32 0, i32 0) call void @llvm.matrix.column.major.store.v4f32(<4 x float> zeroinitializer, float* %m, i64 0, i1 false, i32 0, i32 0)
call void @llvm.matrix.column.major.store.v4f32.p0v4f32(<4 x float> zeroinitializer, <4 x float>* %m, i64 2, i1 false, i32 1, i32 2) call void @llvm.matrix.column.major.store.v4f32(<4 x float> zeroinitializer, float* %m, i64 2, i1 false, i32 1, i32 2)
call void @llvm.matrix.column.major.store.v6f32.p0v6f32(<6 x float> zeroinitializer, <6 x float>* %n, i64 2, i1 false, i32 3, i32 3) call void @llvm.matrix.column.major.store.v6f32(<6 x float> zeroinitializer, float* %n, i64 2, i1 false, i32 3, i32 3)
call void @llvm.matrix.column.major.store.v6f32.p0v6f32(<6 x float> zeroinitializer, <6 x float>* %n, i64 %arg, i1 false, i32 3, i32 3) call void @llvm.matrix.column.major.store.v6f32(<6 x float> zeroinitializer, float* %n, i64 %arg, i1 false, i32 3, i32 3)
ret void ret void
} }
declare <4 x float> @llvm.matrix.transpose.v4f32.v4i32(<4 x i32>, i32, i32)
declare <4 x i32> @llvm.matrix.transpose.v4i32.v4f32(<4 x float>, i32, i32)
define <4 x float> @transpose_mixed_types(<4 x float> %fvec, <4 x i32> %ivec, i32 %arg) { define <4 x float> @transpose_mixed_types(<4 x float> %fvec, <4 x i32> %ivec, i32 %arg) {
; ;
; CHECK-NEXT: Intrinsic has incorrect argument type! ; CHECK-NEXT: Intrinsic has incorrect argument type!
; CHECK-NEXT: <4 x float> (<4 x i32>, i32, i32)* @llvm.matrix.transpose.v4f32.v4i32 ; CHECK-NEXT: <4 x float> (<4 x i32>, i32, i32)* @llvm.matrix.transpose.v4f32.v4i32
; CHECK-NEXT: Intrinsic has incorrect argument type! ; CHECK-NEXT: Intrinsic has incorrect argument type!
; CHECK-NEXT: <4 x i32> (<4 x float>, i32, i32)* @llvm.matrix.transpose.v4i32.v4f32 ; CHECK-NEXT: <4 x i32> (<4 x float>, i32, i32)* @llvm.matrix.transpose.v4i32.v4f32
; ;
%result.0 = call <4 x float> @llvm.matrix.transpose.v4f32.v4i32(<4 x i32> %ivec, i32 0, i32 0) %result.0 = call <4 x float> @llvm.matrix.transpose.v4f32.v4i32(<4 x i32> %ivec, i32 0, i32 0)
%result.1 = call <4 x i32> @llvm.matrix.transpose.v4i32.v4f32(<4 x float> %result.0, i32 3, i32 2) %result.1 = call <4 x i32> @llvm.matrix.transpose.v4i32.v4f32(<4 x float> %result.0, i32 3, i32 2)
ret <4 x float> %result.0 ret <4 x float> %result.0
} }
declare <4 x i32> @llvm.matrix.multiply.v4i32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
declare <4 x float> @llvm.matrix.multiply.v4f32.v4i32.v4f32(<4 x i32>, <4 x float>, i32, i32, i32)
declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4i32(<4 x float>, <4 x i32>, i32, i32, i32)
declare <4 x float> @llvm.matrix.multiply.v4f32.v4i32.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32)
define <4 x float> @multiply_mixed_types(<4 x i32> %ivec, <4 x float> %fvec, i32 %arg) { define <4 x float> @multiply_mixed_types(<4 x i32> %ivec, <4 x float> %fvec, i32 %arg) {
; ;
; CHECK-NEXT: Vector element type mismatch of the result and first operand vector! ; CHECK-NEXT: Vector element type mismatch of the result and first operand vector!
; CHECK-NEXT: <4 x i32> (<4 x float>, <4 x float>, i32, i32, i32)* @llvm.matrix.multiply.v4i32.v4f32.v4f32 ; CHECK-NEXT: <4 x i32> (<4 x float>, <4 x float>, i32, i32, i32)* @llvm.matrix.multiply.v4i32.v4f32.v4f32
; CHECK-NEXT: Vector element type mismatch of the result and first operand vector! ; CHECK-NEXT: Vector element type mismatch of the result and first operand vector!
; CHECK-NEXT: <4 x float> (<4 x i32>, <4 x float>, i32, i32, i32)* @llvm.matrix.multiply.v4f32.v4i32.v4f32 ; CHECK-NEXT: <4 x float> (<4 x i32>, <4 x float>, i32, i32, i32)* @llvm.matrix.multiply.v4f32.v4i32.v4f32
; CHECK-NEXT: Vector element type mismatch of the result and second operand vector! ; CHECK-NEXT: Vector element type mismatch of the result and second operand vector!
; CHECK-NEXT: <4 x float> (<4 x float>, <4 x i32>, i32, i32, i32)* @llvm.matrix.multiply.v4f32.v4f32.v4i32 ; CHECK-NEXT: <4 x float> (<4 x float>, <4 x i32>, i32, i32, i32)* @llvm.matrix.multiply.v4f32.v4f32.v4i32
; CHECK-NEXT: Vector element type mismatch of the result and first operand vector! ; CHECK-NEXT: Vector element type mismatch of the result and first operand vector!
; CHECK-NEXT: <4 x float> (<4 x i32>, <4 x i32>, i32, i32, i32)* @llvm.matrix.multiply.v4f32.v4i32.v4i32 ; CHECK-NEXT: <4 x float> (<4 x i32>, <4 x i32>, i32, i32, i32)* @llvm.matrix.multiply.v4f32.v4i32.v4i32
; ;
%result.0 = call <4 x i32> @llvm.matrix.multiply.v4i32.v4f32.v4f32(<4 x float> %fvec, <4 x float> %fvec, i32 2, i32 2, i32 2) %result.0 = call <4 x i32> @llvm.matrix.multiply.v4i32.v4f32.v4f32(<4 x float> %fvec, <4 x float> %fvec, i32 2, i32 2, i32 2)
%result.1 = call <4 x float> @llvm.matrix.multiply.v4f32.v4i32.v4f32(<4 x i32> %result.0, <4 x float> %fvec, i32 2, i32 2, i32 2) %result.1 = call <4 x float> @llvm.matrix.multiply.v4f32.v4i32.v4f32(<4 x i32> %result.0, <4 x float> %fvec, i32 2, i32 2, i32 2)
%result.2 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4i32(<4 x float> %fvec, <4 x i32> %ivec, i32 2, i32 2, i32 2) %result.2 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4i32(<4 x float> %fvec, <4 x i32> %ivec, i32 2, i32 2, i32 2)
%result.3 = call <4 x float> @llvm.matrix.multiply.v4f32.v4i32.v4i32(<4 x i32> %ivec, <4 x i32> %ivec, i32 2, i32 2, i32 2) %result.3 = call <4 x float> @llvm.matrix.multiply.v4f32.v4i32.v4i32(<4 x i32> %ivec, <4 x i32> %ivec, i32 2, i32 2, i32 2)
ret <4 x float> %result.3 ret <4 x float> %result.3
} }
declare <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4i32(<4 x i32>*, i64, i1, i32, i32) define <4 x float> @column.major_load_mixed_types(i32* %m, float* %n, i32 %arg) {
declare <4 x i32> @llvm.matrix.column.major.load.v4i32.p0v4f32(<4 x float>*, i64, i1, i32, i32)
define <4 x float> @column.major_load_mixed_types(<4 x i32>* %m, <4 x float>* %n, i32 %arg) {
; ;
; CHECK-NEXT: Vector element type mismatch of the result and first operand vector! ; CHECK-NEXT: Intrinsic has incorrect argument type!
; CHECK-NEXT: <4 x float> (<4 x i32>*, i64, i1, i32, i32)* @llvm.matrix.column.major.load.v4f32.p0v4i32 ; CHECK-NEXT: <4 x float> (i32*, i64, i1, i32, i32)* @llvm.matrix.column.major.load.v4f32.pi32
; CHECK-NEXT: Vector element type mismatch of the result and first operand vector! ; CHECK-NEXT: Intrinsic has incorrect argument type!
; CHECK-NEXT: <4 x i32> (<4 x float>*, i64, i1, i32, i32)* @llvm.matrix.column.major.load.v4i32.p0v4f32 ; CHECK-NEXT: <4 x i32> (float*, i64, i1, i32, i32)* @llvm.matrix.column.major.load.v4i32
; ;
%result.0 = call <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4i32(<4 x i32>* %m, i64 2, i1 false, i32 2, i32 2) %result.0 = call <4 x float> @llvm.matrix.column.major.load.v4f32.pi32(i32* %m, i64 2, i1 false, i32 2, i32 2)
%result.1 = call <4 x i32> @llvm.matrix.column.major.load.v4i32.p0v4f32(<4 x float>* %n, i64 2, i1 false, i32 2, i32 2) %result.1 = call <4 x i32> @llvm.matrix.column.major.load.v4i32(float* %n, i64 2, i1 false, i32 2, i32 2)
ret <4 x float> %result.0 ret <4 x float> %result.0
} }
declare void @llvm.matrix.column.major.store.v4i32.p0v4f32(<4 x i32>, <4 x float>*, i64, i1, i32, i32) define void @column.major_store_mixed_types(float* %m, i32* %n, i64 %arg) {
declare void @llvm.matrix.column.major.store.v4f32.p0v4i32(<4 x float>, <4 x i32>*, i64, i1, i32, i32)
define void @column.major_store_mixed_types(<4 x float>* %m, <4 x i32>* %n, i64 %arg) {
; ;
; CHECK-NEXT: Vector element type mismatch of the result and second operand vector! ; CHECK-NEXT: Intrinsic has incorrect argument type!
; CHECK-NEXT: void (<4 x i32>, <4 x float>*, i64, i1, i32, i32)* @llvm.matrix.column.major.store.v4i32.p0v4f32 ; CHECK-NEXT: void (<4 x i32>, float*, i64, i1, i32, i32)* @llvm.matrix.column.major.store.v4i32.vi32
; CHECK-NEXT: Vector element type mismatch of the result and second operand vector! ; CHECK-NEXT: Intrinsic has incorrect argument type!
; CHECK-NEXT: void (<4 x float>, <4 x i32>*, i64, i1, i32, i32)* @llvm.matrix.column.major.store.v4f32.p0v4i32 ; CHECK-NEXT: void (<4 x float>, i32*, i64, i1, i32, i32)* @llvm.matrix.column.major.store.v4f32.pi32
; ;
call void @llvm.matrix.column.major.store.v4i32.p0v4f32(<4 x i32> zeroinitializer, <4 x float>* %m, i64 2, i1 false, i32 2, i32 2) call void @llvm.matrix.column.major.store.v4i32.vi32(<4 x i32> zeroinitializer, float* %m, i64 2, i1 false, i32 2, i32 2)
call void @llvm.matrix.column.major.store.v4f32.p0v4i32(<4 x float> zeroinitializer, <4 x i32>* %n, i64 2, i1 false, i32 2, i32 2) call void @llvm.matrix.column.major.store.v4f32.pi32(<4 x float> zeroinitializer, i32* %n, i64 2, i1 false, i32 2, i32 2)
ret void ret void
} }
declare void @llvm.matrix.column.major.store.v4f32p0.p0v4f32(<4 x float*>, <4 x float>*, i64, i1, i32, i32)
define void @column.major_store_non_int_float_type(<4 x float>* %m, <4 x float>* %n, i64 %arg) { define void @column.major_store_non_int_float_type(<4 x float>* %m, <4 x float>* %n, i64 %arg) {
; ;
; CHECK-NEXT: Result type must be an integer or floating-point type! ; CHECK-NEXT: Intrinsic has incorrect argument type!
; CHECK-NEXT: void (<4 x float*>, <4 x float>*, i64, i1, i32, i32)* @llvm.matrix.column.major.store.v4p0f32.p0v4f32 ; CHECK-NEXT: void (<4 x float*>, <4 x float>*, i64, i1, i32, i32)* @llvm.matrix.column.major.store.v4f32p0.p0v4f32
; ;
call void @llvm.matrix.column.major.store.v4f32p0.p0v4f32(<4 x float*> zeroinitializer, <4 x float>* %n, i64 2, i1 false, i32 2, i32 2) call void @llvm.matrix.column.major.store.v4f32p0.p0v4f32(<4 x float*> zeroinitializer, <4 x float>* %n, i64 2, i1 false, i32 2, i32 2)
ret void ret void
} }
define <4 x float> @column.major_load_stride_too_small(<4 x float>* %m, i32 %arg) { define <4 x float> @column.major_load_stride_too_small(float* %m, i32 %arg) {
; ;
; CHECK-NEXT: Stride must be greater or equal than the number of rows! ; CHECK-NEXT: Stride must be greater or equal than the number of rows!
; CHECK-NEXT: <4 x float> (<4 x float>*, i64, i1, i32, i32)* @llvm.matrix.column.major.load.v4f32.p0v4f32 ; CHECK-NEXT: <4 x float> (float*, i64, i1, i32, i32)* @llvm.matrix.column.major.load.v4f32
; ;
%result.1 = call <4 x float> @llvm.matrix.column.major.load.v4f32.p0v4f32(<4 x float>* %m, i64 1, i1 false, i32 2, i32 2) %result.1 = call <4 x float> @llvm.matrix.column.major.load.v4f32(float* %m, i64 1, i1 false, i32 2, i32 2)
ret <4 x float> %result.1 ret <4 x float> %result.1
} }
define void @column.major_store_stride_too_small(<4 x float>* %m, i64 %arg) { define void @column.major_store_stride_too_small(float* %m, i64 %arg) {
; ;
; CHECK-NEXT: Stride must be greater or equal than the number of rows! ; CHECK-NEXT: Stride must be greater or equal than the number of rows!
; CHECK-NEXT: void (<4 x float>, <4 x float>*, i64, i1, i32, i32)* @llvm.matrix.column.major.store.v4f32.p0v4f32 ; CHECK-NEXT: void (<4 x float>, float*, i64, i1, i32, i32)* @llvm.matrix.column.major.store.v4f32
; ;
call void @llvm.matrix.column.major.store.v4f32.p0v4f32(<4 x float> zeroinitializer, <4 x float>* %m, i64 1, i1 false, i32 2, i32 2) call void @llvm.matrix.column.major.store.v4f32(<4 x float> zeroinitializer, float* %m, i64 1, i1 false, i32 2, i32 2)
ret void ret void
} }
declare <4 x i32> @llvm.matrix.column.major.load.v4i32(float*, i64, i1, i32, i32)
declare <4 x float> @llvm.matrix.column.major.load.v4f32.pi32(i32*, i64, i1, i32, i32)
declare <4 x float> @llvm.matrix.column.major.load.v4f32(float*, i64, i1, i32, i32)
declare <6 x float> @llvm.matrix.column.major.load.v6f32(float*, i64, i1, i32, i32)
declare void @llvm.matrix.column.major.store.v4f32(<4 x float>, float*, i64, i1, i32, i32)
declare void @llvm.matrix.column.major.store.v6f32(<6 x float>, float*, i64, i1, i32, i32)
declare void @llvm.matrix.column.major.store.v4i32.vi32(<4 x i32>, float*, i64, i1, i32, i32)
declare void @llvm.matrix.column.major.store.v4f32.pi32(<4 x float>, i32*, i64, i1, i32, i32)
declare void @llvm.matrix.column.major.store.v4f32p0.p0v4f32(<4 x float*>, <4 x float>*, i64, i1, i32, i32)
declare <4 x i32> @llvm.matrix.transpose.v4i32.v4f32(<4 x float>, i32, i32)
declare <4 x float> @llvm.matrix.transpose.v4f32(<4 x float>, i32, i32)
declare <4 x float> @llvm.matrix.transpose.v4f32.v4i32(<4 x i32>, i32, i32)
declare <4 x i32> @llvm.matrix.multiply.v4i32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
declare <4 x float> @llvm.matrix.multiply.v4f32.v4i32.v4f32(<4 x i32>, <4 x float>, i32, i32, i32)
declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4i32(<4 x float>, <4 x i32>, i32, i32, i32)
declare <4 x float> @llvm.matrix.multiply.v4f32.v4i32.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32)
declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)