This is an archive of the discontinued LLVM Phabricator instance.

Differential D129083

[AArch64][SME] Update load/store intrinsics to take predicate corresponding to element size.
ClosedPublic

Authored by sdesmalen on Jul 4 2022, 7:27 AM.

Details

Reviewers
kmclaughlin
david-arm
sagarkulkarni19
Commits
rG6106a767b723: [AArch64][SME] Update load/store intrinsics to take predicate corresponding to…
Summary

Instead of using <vscale x 16 x i1> for all the loads/stores, we now use the appropriate
predicate type according to the element size, e.g.

ld1b uses <vscale x 16 x i1>
ld1w uses <vscale x 4 x i1>
ld1q uses <vscale x 1 x i1>

Diff Detail

Event Timeline

sdesmalen created this revision.Jul 4 2022, 7:27 AM
Herald added a project: Restricted Project. · View Herald TranscriptJul 4 2022, 7:27 AM
Herald added a subscriber: kristof.beyls. · View Herald Transcript
sdesmalen requested review of this revision.Jul 4 2022, 7:27 AM
Herald added a project: Restricted Project. · View Herald TranscriptJul 4 2022, 7:27 AM
Herald added a subscriber: llvm-commits. · View Herald Transcript
Harbormaster completed remote builds in B173562: Diff 442098.Jul 4 2022, 7:28 AM
kmclaughlin accepted this revision.Jul 5 2022, 5:27 AM

LGTM!

This revision is now accepted and ready to land.Jul 5 2022, 5:27 AM
sdesmalen updated this revision to Diff 442460.Jul 6 2022, 1:14 AM

Updated tests after rebase.

Harbormaster completed remote builds in B173818: Diff 442460.Jul 6 2022, 2:21 AM
Matt added a subscriber: Matt.Jul 6 2022, 3:25 PM
This revision was landed with ongoing or failed builds.Jul 7 2022, 12:40 AM
Closed by commit rG6106a767b723: [AArch64][SME] Update load/store intrinsics to take predicate corresponding to… (authored by sdesmalen). · Explain Why
This revision was automatically updated to reflect the committed changes.
sdesmalen added a commit: rG6106a767b723: [AArch64][SME] Update load/store intrinsics to take predicate corresponding to….

Revision Contents

PathSize
llvm/
include/
llvm/
IR/
33 lines
test/
CodeGen/
AArch64/
176 lines
176 lines

Diff 442805

llvm/include/llvm/IR/IntrinsicsAArch64.td

Show First 20 Lines • Show All 925 Lines • ▼ Show 20 Lines
defvar data512 = !listsplat(llvm_i64_ty, 8); defvar data512 = !listsplat(llvm_i64_ty, 8);
def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>; def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>;
def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>; def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>;
def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>; def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>; def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
} }
def llvm_nxv1i1_ty : LLVMType<nxv1i1>;
def llvm_nxv2i1_ty : LLVMType<nxv2i1>; def llvm_nxv2i1_ty : LLVMType<nxv2i1>;
def llvm_nxv4i1_ty : LLVMType<nxv4i1>; def llvm_nxv4i1_ty : LLVMType<nxv4i1>;
def llvm_nxv8i1_ty : LLVMType<nxv8i1>; def llvm_nxv8i1_ty : LLVMType<nxv8i1>;
def llvm_nxv16i1_ty : LLVMType<nxv16i1>; def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
def llvm_nxv16i8_ty : LLVMType<nxv16i8>; def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
def llvm_nxv4i32_ty : LLVMType<nxv4i32>; def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
def llvm_nxv2i64_ty : LLVMType<nxv2i64>; def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
def llvm_nxv8f16_ty : LLVMType<nxv8f16>; def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
▲ Show 20 Lines • Show All 1,645 Lines • ▼ Show 20 Lines
// Scalable Matrix Extension (SME) Intrinsics // Scalable Matrix Extension (SME) Intrinsics
let TargetPrefix = "aarch64" in { let TargetPrefix = "aarch64" in {
class SME_Load_Store_Intrinsic<LLVMType pred_ty> class SME_Load_Store_Intrinsic<LLVMType pred_ty>
: DefaultAttrsIntrinsic<[], : DefaultAttrsIntrinsic<[],
[pred_ty, llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty], []>; [pred_ty, llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty], []>;
// Loads // Loads
def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
def int_aarch64_sme_ld1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>;
def int_aarch64_sme_ld1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>;
def int_aarch64_sme_ld1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>;
def int_aarch64_sme_ld1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>;
def int_aarch64_sme_ld1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
def int_aarch64_sme_ld1h_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1h_vert : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>;
def int_aarch64_sme_ld1w_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1w_vert : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>;
def int_aarch64_sme_ld1d_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1d_vert : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>;
def int_aarch64_sme_ld1q_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_ld1q_vert : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>;
// Stores // Stores
def int_aarch64_sme_st1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
def int_aarch64_sme_st1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>;
def int_aarch64_sme_st1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>;
def int_aarch64_sme_st1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>;
def int_aarch64_sme_st1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>;
def int_aarch64_sme_st1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
def int_aarch64_sme_st1h_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1h_vert : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>;
def int_aarch64_sme_st1w_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1w_vert : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>;
def int_aarch64_sme_st1d_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1d_vert : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>;
def int_aarch64_sme_st1q_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; def int_aarch64_sme_st1q_vert : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>;
// Spill + fill // Spill + fill
def int_aarch64_sme_ldr : DefaultAttrsIntrinsic< def int_aarch64_sme_ldr : DefaultAttrsIntrinsic<
[], [llvm_i32_ty, llvm_ptr_ty]>; [], [llvm_i32_ty, llvm_ptr_ty]>;
def int_aarch64_sme_str : DefaultAttrsIntrinsic< def int_aarch64_sme_str : DefaultAttrsIntrinsic<
[], [llvm_i32_ty, llvm_ptr_ty]>; [], [llvm_i32_ty, llvm_ptr_ty]>;
class SME_TileToVector_Intrinsic class SME_TileToVector_Intrinsic
▲ Show 20 Lines • Show All 99 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/sme-intrinsics-loads.ll

Show All 24 Lines
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i8, ptr %ptr, i64 %index %base = getelementptr i8, ptr %ptr, i64 %index
%tileslice = add i32 %sliceidx, 15 %tileslice = add i32 %sliceidx, 15
call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 0) call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 %tileslice) call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 %tileslice)
ret void; ret void;
} }
define void @ld1h(<vscale x 16 x i1> %pg, ptr %ptr, i32 %sliceidx) { define void @ld1h(<vscale x 8 x i1> %pg, ptr %ptr, i32 %sliceidx) {
; CHECK-LABEL: ld1h: ; CHECK-LABEL: ld1h:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, w1 ; CHECK-NEXT: mov w12, w1
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: ld1h {za0h.h[w12, 7]}, p0/z, [x0] ; CHECK-NEXT: ld1h {za0h.h[w12, 7]}, p0/z, [x0]
; CHECK-NEXT: ld1h {za1h.h[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1h {za1h.h[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1h {za0v.h[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1h {za0v.h[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1h {za1v.h[w12, 7]}, p0/z, [x0] ; CHECK-NEXT: ld1h {za1v.h[w12, 7]}, p0/z, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%tileslice = add i32 %sliceidx, 7 %tileslice = add i32 %sliceidx, 7
call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 %tileslice) call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> %pg, ptr %ptr, i64 0, i32 %tileslice)
call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.ld1h.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1h.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 %tileslice) call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> %pg, ptr %ptr, i64 1, i32 %tileslice)
ret void; ret void;
} }
define void @ld1h_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) { define void @ld1h_with_addr_offset(<vscale x 8 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {
; CHECK-LABEL: ld1h_with_addr_offset: ; CHECK-LABEL: ld1h_with_addr_offset:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, w2 ; CHECK-NEXT: mov w12, w2
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: ld1h {za0h.h[w12, 7]}, p0/z, [x0, x1, lsl #1] ; CHECK-NEXT: ld1h {za0h.h[w12, 7]}, p0/z, [x0, x1, lsl #1]
; CHECK-NEXT: ld1h {za1v.h[w13, 0]}, p0/z, [x0, x1, lsl #1] ; CHECK-NEXT: ld1h {za1v.h[w13, 0]}, p0/z, [x0, x1, lsl #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i16, ptr %ptr, i64 %index %base = getelementptr i16, ptr %ptr, i64 %index
%tileslice = add i32 %sliceidx, 7 %tileslice = add i32 %sliceidx, 7
call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 %tileslice) call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> %pg, ptr %base, i64 0, i32 %tileslice)
call void @llvm.aarch64.sme.ld1h.vert(<vscale x 16 x i1> %pg, ptr %base, i64 1, i32 0) call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> %pg, ptr %base, i64 1, i32 0)
ret void; ret void;
} }
define void @ld1w(<vscale x 16 x i1> %pg, ptr %ptr, i32 %sliceidx) { define void @ld1w(<vscale x 4 x i1> %pg, ptr %ptr, i32 %sliceidx) {
; CHECK-LABEL: ld1w: ; CHECK-LABEL: ld1w:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, wzr ; CHECK-NEXT: mov w12, wzr
; CHECK-NEXT: mov w13, w1 ; CHECK-NEXT: mov w13, w1
; CHECK-NEXT: ld1w {za0h.s[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za0h.s[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1w {za1h.s[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za1h.s[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1w {za2h.s[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za2h.s[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1w {za3h.s[w13, 3]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za3h.s[w13, 3]}, p0/z, [x0]
; CHECK-NEXT: ld1w {za0v.s[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za0v.s[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1w {za1v.s[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za1v.s[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1w {za2v.s[w13, 3]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za2v.s[w13, 3]}, p0/z, [x0]
; CHECK-NEXT: ld1w {za3v.s[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za3v.s[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%tileslice = add i32 %sliceidx, 3 %tileslice = add i32 %sliceidx, 3
call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 %tileslice) call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i64 3, i32 %tileslice)
call void @llvm.aarch64.sme.ld1w.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1w.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.ld1w.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 %tileslice) call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i64 2, i32 %tileslice)
call void @llvm.aarch64.sme.ld1w.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i64 3, i32 0)
ret void; ret void;
} }
define void @ld1w_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) { define void @ld1w_with_addr_offset(<vscale x 4 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {
; CHECK-LABEL: ld1w_with_addr_offset: ; CHECK-LABEL: ld1w_with_addr_offset:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, w2 ; CHECK-NEXT: mov w12, w2
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: ld1w {za0h.s[w13, 0]}, p0/z, [x0, x1, lsl #2] ; CHECK-NEXT: ld1w {za0h.s[w13, 0]}, p0/z, [x0, x1, lsl #2]
; CHECK-NEXT: ld1w {za3v.s[w12, 3]}, p0/z, [x0, x1, lsl #2] ; CHECK-NEXT: ld1w {za3v.s[w12, 3]}, p0/z, [x0, x1, lsl #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i32, ptr %ptr, i64 %index %base = getelementptr i32, ptr %ptr, i64 %index
%tileslice = add i32 %sliceidx, 3 %tileslice = add i32 %sliceidx, 3
call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 0) call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %base, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1w.vert(<vscale x 16 x i1> %pg, ptr %base, i64 3, i32 %tileslice) call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %base, i64 3, i32 %tileslice)
ret void; ret void;
} }
define void @ld1d(<vscale x 16 x i1> %pg, ptr %ptr, i32 %sliceidx) { define void @ld1d(<vscale x 2 x i1> %pg, ptr %ptr, i32 %sliceidx) {
; CHECK-LABEL: ld1d: ; CHECK-LABEL: ld1d:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: mov w12, w1 ; CHECK-NEXT: mov w12, w1
; CHECK-NEXT: ld1d {za0h.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za0h.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za1h.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za1h.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za2h.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za2h.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za3h.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za3h.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za4h.d[w12, 1]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za4h.d[w12, 1]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za5h.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za5h.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za6h.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za6h.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za7h.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za7h.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za0v.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za0v.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za1v.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za1v.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za2v.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za2v.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za3v.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za3v.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za4v.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za4v.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za5v.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za5v.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za6v.d[w13, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za6v.d[w13, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1d {za7v.d[w12, 1]}, p0/z, [x0] ; CHECK-NEXT: ld1d {za7v.d[w12, 1]}, p0/z, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%tileslice = add i32 %sliceidx, 1 %tileslice = add i32 %sliceidx, 1
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 3, i32 0)
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 4, i32 %tileslice) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 4, i32 %tileslice)
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 5, i32 0) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 5, i32 0)
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 6, i32 0) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 6, i32 0)
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 7, i32 0) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 7, i32 0)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 3, i32 0)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 4, i32 0) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 4, i32 0)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 5, i32 0) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 5, i32 0)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 6, i32 0) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 6, i32 0)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 7, i32 %tileslice) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 7, i32 %tileslice)
ret void; ret void;
} }
define void @ld1d_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) { define void @ld1d_with_addr_offset(<vscale x 2 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {
; CHECK-LABEL: ld1d_with_addr_offset: ; CHECK-LABEL: ld1d_with_addr_offset:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, w2 ; CHECK-NEXT: mov w12, w2
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: ld1d {za0h.d[w12, 1]}, p0/z, [x0, x1, lsl #3] ; CHECK-NEXT: ld1d {za0h.d[w12, 1]}, p0/z, [x0, x1, lsl #3]
; CHECK-NEXT: ld1d {za7v.d[w13, 0]}, p0/z, [x0, x1, lsl #3] ; CHECK-NEXT: ld1d {za7v.d[w13, 0]}, p0/z, [x0, x1, lsl #3]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i64, ptr %ptr, i64 %index %base = getelementptr i64, ptr %ptr, i64 %index
%tileslice = add i32 %sliceidx, 1 %tileslice = add i32 %sliceidx, 1
call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 %tileslice) call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %base, i64 0, i32 %tileslice)
call void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1> %pg, ptr %base, i64 7, i32 0) call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %base, i64 7, i32 0)
ret void; ret void;
} }
define void @ld1q(<vscale x 16 x i1> %pg, ptr %ptr) { define void @ld1q(<vscale x 1 x i1> %pg, ptr %ptr) {
; CHECK-LABEL: ld1q: ; CHECK-LABEL: ld1q:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, wzr ; CHECK-NEXT: mov w12, wzr
; CHECK-NEXT: ld1q {za0h.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za0h.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za1h.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za1h.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za2h.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za2h.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za3h.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za3h.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za4h.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za4h.q[w12, 0]}, p0/z, [x0]
Show All 20 Lines
; CHECK-NEXT: ld1q {za9v.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za9v.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za10v.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za10v.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za11v.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za11v.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za12v.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za12v.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za13v.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za13v.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za14v.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za14v.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ld1q {za15v.q[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1q {za15v.q[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 3, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 4, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 4, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 5, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 5, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 6, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 6, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 7, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 7, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 8, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 8, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 9, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 9, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 10, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 10, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 11, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 11, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 12, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 12, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 13, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 13, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 14, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 14, i32 0)
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 15, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 15, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 3, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 4, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 4, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 5, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 5, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 6, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 6, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 7, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 7, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 8, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 8, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 9, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 9, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 10, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 10, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 11, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 11, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 12, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 12, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 13, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 13, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 14, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 14, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 15, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 15, i32 0)
ret void; ret void;
} }
define void @ld1q_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index) { define void @ld1q_with_addr_offset(<vscale x 1 x i1> %pg, ptr %ptr, i64 %index) {
; CHECK-LABEL: ld1q_with_addr_offset: ; CHECK-LABEL: ld1q_with_addr_offset:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, wzr ; CHECK-NEXT: mov w12, wzr
; CHECK-NEXT: ld1q {za0h.q[w12, 0]}, p0/z, [x0, x1, lsl #4] ; CHECK-NEXT: ld1q {za0h.q[w12, 0]}, p0/z, [x0, x1, lsl #4]
; CHECK-NEXT: ld1q {za15v.q[w12, 0]}, p0/z, [x0, x1, lsl #4] ; CHECK-NEXT: ld1q {za15v.q[w12, 0]}, p0/z, [x0, x1, lsl #4]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i128, ptr %ptr, i64 %index %base = getelementptr i128, ptr %ptr, i64 %index
call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 0) call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %base, i64 0, i32 0)
call void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1> %pg, ptr %base, i64 15, i32 0) call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %base, i64 15, i32 0)
ret void; ret void;
} }
define void @ldr(ptr %ptr) { define void @ldr(ptr %ptr) {
; CHECK-LABEL: ldr: ; CHECK-LABEL: ldr:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, wzr ; CHECK-NEXT: mov w12, wzr
; CHECK-NEXT: ldr za[w12, 0], [x0] ; CHECK-NEXT: ldr za[w12, 0], [x0]
Show All 38 Lines; CHECK-NEXT: ret
%mulvl = mul i64 %vscale, 256 %mulvl = mul i64 %vscale, 256
%base = getelementptr i8, ptr %ptr, i64 %mulvl %base = getelementptr i8, ptr %ptr, i64 %mulvl
call void @llvm.aarch64.sme.ldr(i32 0, ptr %base) call void @llvm.aarch64.sme.ldr(i32 0, ptr %base)
ret void; ret void;
} }
; Ensure that the tile offset is sunk, given that this is likely to be an 'add' ; Ensure that the tile offset is sunk, given that this is likely to be an 'add'
; that's decomposed into a base + offset in ISel. ; that's decomposed into a base + offset in ISel.
define void @test_ld1_sink_tile0_offset_operand(<vscale x 16 x i1> %pg, ptr %src, i32 %base, i32 %N) { define void @test_ld1_sink_tile0_offset_operand(<vscale x 4 x i1> %pg, ptr %src, i32 %base, i32 %N) {
; CHECK-LABEL: test_ld1_sink_tile0_offset_operand: ; CHECK-LABEL: test_ld1_sink_tile0_offset_operand:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w12, w1 ; CHECK-NEXT: mov w12, w1
; CHECK-NEXT: .LBB14_1: // %for.body ; CHECK-NEXT: .LBB14_1: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ld1w {za0h.s[w12, 0]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za0h.s[w12, 0]}, p0/z, [x0]
; CHECK-NEXT: subs w2, w2, #1 ; CHECK-NEXT: subs w2, w2, #1
; CHECK-NEXT: ld1w {za0h.s[w12, 1]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za0h.s[w12, 1]}, p0/z, [x0]
; CHECK-NEXT: ld1w {za0h.s[w12, 2]}, p0/z, [x0] ; CHECK-NEXT: ld1w {za0h.s[w12, 2]}, p0/z, [x0]
; CHECK-NEXT: b.ne .LBB14_1 ; CHECK-NEXT: b.ne .LBB14_1
; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%add1 = add i32 %base, 1 %add1 = add i32 %base, 1
%add2 = add i32 %base, 2 %add2 = add i32 %base, 2
br label %for.body br label %for.body
for.body: for.body:
%i = phi i32 [ 0, %entry ], [ %inc, %for.body ] %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1> %pg, ptr %src, i64 0, i32 %base) call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i64 0, i32 %base)
call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1> %pg, ptr %src, i64 0, i32 %add1) call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i64 0, i32 %add1)
call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1> %pg, ptr %src, i64 0, i32 %add2) call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i64 0, i32 %add2)
%inc = add nuw nsw i32 %i, 1 %inc = add nuw nsw i32 %i, 1
%exitcond.not = icmp eq i32 %inc, %N %exitcond.not = icmp eq i32 %inc, %N
br i1 %exitcond.not, label %exit, label %for.body br i1 %exitcond.not, label %exit, label %for.body
exit: exit:
ret void ret void
} }
declare void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1h.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1w.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1d.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1q.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1h.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1w.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1d.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ld1q.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.ldr(i32, ptr) declare void @llvm.aarch64.sme.ldr(i32, ptr)
declare i64 @llvm.vscale.i64() declare i64 @llvm.vscale.i64()

llvm/test/CodeGen/AArch64/sme-intrinsics-stores.ll

Show All 24 Lines
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i8, ptr %ptr, i64 %index %base = getelementptr i8, ptr %ptr, i64 %index
%tileslice = add i32 %sliceidx, 15 %tileslice = add i32 %sliceidx, 15
call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 0) call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 0)
call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 %tileslice) call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 %tileslice)
ret void; ret void;
} }
define void @st1h(<vscale x 16 x i1> %pg, ptr %ptr, i32 %sliceidx) { define void @st1h(<vscale x 8 x i1> %pg, ptr %ptr, i32 %sliceidx) {
; CHECK-LABEL: st1h: ; CHECK-LABEL: st1h:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, w1 ; CHECK-NEXT: mov w12, w1
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: st1h {za0h.h[w12, 7]}, p0, [x0] ; CHECK-NEXT: st1h {za0h.h[w12, 7]}, p0, [x0]
; CHECK-NEXT: st1h {za1h.h[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1h {za1h.h[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1h {za0v.h[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1h {za0v.h[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1h {za1v.h[w12, 7]}, p0, [x0] ; CHECK-NEXT: st1h {za1v.h[w12, 7]}, p0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%tileslice = add i32 %sliceidx, 7 %tileslice = add i32 %sliceidx, 7
call void @llvm.aarch64.sme.st1h.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 %tileslice) call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> %pg, ptr %ptr, i64 0, i32 %tileslice)
call void @llvm.aarch64.sme.st1h.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.st1h.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.st1h.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 %tileslice) call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> %pg, ptr %ptr, i64 1, i32 %tileslice)
ret void; ret void;
} }
define void @st1h_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) { define void @st1h_with_addr_offset(<vscale x 8 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {
; CHECK-LABEL: st1h_with_addr_offset: ; CHECK-LABEL: st1h_with_addr_offset:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, w2 ; CHECK-NEXT: mov w12, w2
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: st1h {za0h.h[w12, 7]}, p0, [x0, x1, lsl #1] ; CHECK-NEXT: st1h {za0h.h[w12, 7]}, p0, [x0, x1, lsl #1]
; CHECK-NEXT: st1h {za1v.h[w13, 0]}, p0, [x0, x1, lsl #1] ; CHECK-NEXT: st1h {za1v.h[w13, 0]}, p0, [x0, x1, lsl #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i16, ptr %ptr, i64 %index %base = getelementptr i16, ptr %ptr, i64 %index
%tileslice = add i32 %sliceidx, 7 %tileslice = add i32 %sliceidx, 7
call void @llvm.aarch64.sme.st1h.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 %tileslice) call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> %pg, ptr %base, i64 0, i32 %tileslice)
call void @llvm.aarch64.sme.st1h.vert(<vscale x 16 x i1> %pg, ptr %base, i64 1, i32 0) call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> %pg, ptr %base, i64 1, i32 0)
ret void; ret void;
} }
define void @st1w(<vscale x 16 x i1> %pg, ptr %ptr, i32 %sliceidx) { define void @st1w(<vscale x 4 x i1> %pg, ptr %ptr, i32 %sliceidx) {
; CHECK-LABEL: st1w: ; CHECK-LABEL: st1w:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: mov w12, w1 ; CHECK-NEXT: mov w12, w1
; CHECK-NEXT: st1w {za0h.s[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1w {za0h.s[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1w {za1h.s[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1w {za1h.s[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1w {za2h.s[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1w {za2h.s[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1w {za3h.s[w12, 3]}, p0, [x0] ; CHECK-NEXT: st1w {za3h.s[w12, 3]}, p0, [x0]
; CHECK-NEXT: st1w {za0v.s[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1w {za0v.s[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1w {za1v.s[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1w {za1v.s[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1w {za2v.s[w12, 3]}, p0, [x0] ; CHECK-NEXT: st1w {za2v.s[w12, 3]}, p0, [x0]
; CHECK-NEXT: st1w {za3v.s[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1w {za3v.s[w13, 0]}, p0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%tileslice = add i32 %sliceidx, 3 %tileslice = add i32 %sliceidx, 3
call void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 %tileslice) call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i64 3, i32 %tileslice)
call void @llvm.aarch64.sme.st1w.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.st1w.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.st1w.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 %tileslice) call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i64 2, i32 %tileslice)
call void @llvm.aarch64.sme.st1w.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i64 3, i32 0)
ret void; ret void;
} }
define void @st1w_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) { define void @st1w_with_addr_offset(<vscale x 4 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {
; CHECK-LABEL: st1w_with_addr_offset: ; CHECK-LABEL: st1w_with_addr_offset:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, wzr ; CHECK-NEXT: mov w12, wzr
; CHECK-NEXT: mov w13, w2 ; CHECK-NEXT: mov w13, w2
; CHECK-NEXT: st1w {za0h.s[w12, 0]}, p0, [x0, x1, lsl #2] ; CHECK-NEXT: st1w {za0h.s[w12, 0]}, p0, [x0, x1, lsl #2]
; CHECK-NEXT: st1w {za3v.s[w13, 3]}, p0, [x0, x1, lsl #2] ; CHECK-NEXT: st1w {za3v.s[w13, 3]}, p0, [x0, x1, lsl #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i32, ptr %ptr, i64 %index %base = getelementptr i32, ptr %ptr, i64 %index
%tileslice = add i32 %sliceidx, 3 %tileslice = add i32 %sliceidx, 3
call void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 0) call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> %pg, ptr %base, i64 0, i32 0)
call void @llvm.aarch64.sme.st1w.vert(<vscale x 16 x i1> %pg, ptr %base, i64 3, i32 %tileslice) call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> %pg, ptr %base, i64 3, i32 %tileslice)
ret void; ret void;
} }
define void @st1d(<vscale x 16 x i1> %pg, ptr %ptr, i32 %sliceidx) { define void @st1d(<vscale x 2 x i1> %pg, ptr %ptr, i32 %sliceidx) {
; CHECK-LABEL: st1d: ; CHECK-LABEL: st1d:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: mov w12, w1 ; CHECK-NEXT: mov w12, w1
; CHECK-NEXT: st1d {za0h.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za0h.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za1h.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za1h.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za2h.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za2h.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za3h.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za3h.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za4h.d[w12, 1]}, p0, [x0] ; CHECK-NEXT: st1d {za4h.d[w12, 1]}, p0, [x0]
; CHECK-NEXT: st1d {za5h.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za5h.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za6h.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za6h.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za7h.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za7h.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za0v.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za0v.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za1v.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za1v.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za2v.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za2v.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za3v.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za3v.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za4v.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za4v.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za5v.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za5v.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za6v.d[w13, 0]}, p0, [x0] ; CHECK-NEXT: st1d {za6v.d[w13, 0]}, p0, [x0]
; CHECK-NEXT: st1d {za7v.d[w12, 1]}, p0, [x0] ; CHECK-NEXT: st1d {za7v.d[w12, 1]}, p0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%tileslice = add i32 %sliceidx, 1 %tileslice = add i32 %sliceidx, 1
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 3, i32 0)
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 4, i32 %tileslice) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 4, i32 %tileslice)
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 5, i32 0) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 5, i32 0)
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 6, i32 0) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 6, i32 0)
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 7, i32 0) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i64 7, i32 0)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 3, i32 0)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 4, i32 0) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 4, i32 0)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 5, i32 0) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 5, i32 0)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 6, i32 0) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 6, i32 0)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 7, i32 %tileslice) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i64 7, i32 %tileslice)
ret void; ret void;
} }
define void @st1d_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) { define void @st1d_with_addr_offset(<vscale x 2 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {
; CHECK-LABEL: st1d_with_addr_offset: ; CHECK-LABEL: st1d_with_addr_offset:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, w2 ; CHECK-NEXT: mov w12, w2
; CHECK-NEXT: mov w13, wzr ; CHECK-NEXT: mov w13, wzr
; CHECK-NEXT: st1d {za0h.d[w12, 1]}, p0, [x0, x1, lsl #3] ; CHECK-NEXT: st1d {za0h.d[w12, 1]}, p0, [x0, x1, lsl #3]
; CHECK-NEXT: st1d {za7v.d[w13, 0]}, p0, [x0, x1, lsl #3] ; CHECK-NEXT: st1d {za7v.d[w13, 0]}, p0, [x0, x1, lsl #3]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i64, ptr %ptr, i64 %index %base = getelementptr i64, ptr %ptr, i64 %index
%tileslice = add i32 %sliceidx, 1 %tileslice = add i32 %sliceidx, 1
call void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 %tileslice) call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> %pg, ptr %base, i64 0, i32 %tileslice)
call void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1> %pg, ptr %base, i64 7, i32 0) call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> %pg, ptr %base, i64 7, i32 0)
ret void; ret void;
} }
define void @st1q(<vscale x 16 x i1> %pg, ptr %ptr) { define void @st1q(<vscale x 1 x i1> %pg, ptr %ptr) {
; CHECK-LABEL: st1q: ; CHECK-LABEL: st1q:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, wzr ; CHECK-NEXT: mov w12, wzr
; CHECK-NEXT: st1q {za0h.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za0h.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za1h.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za1h.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za2h.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za2h.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za3h.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za3h.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za4h.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za4h.q[w12, 0]}, p0, [x0]
Show All 20 Lines
; CHECK-NEXT: st1q {za9v.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za9v.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za10v.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za10v.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za11v.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za11v.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za12v.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za12v.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za13v.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za13v.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za14v.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za14v.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: st1q {za15v.q[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1q {za15v.q[w12, 0]}, p0, [x0]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 3, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 4, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 4, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 5, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 5, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 6, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 6, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 7, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 7, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 8, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 8, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 9, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 9, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 10, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 10, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 11, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 11, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 12, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 12, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 13, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 13, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 14, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 14, i32 0)
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i64 15, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i64 15, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 0, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 0, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 1, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 1, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 2, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 2, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 3, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 3, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 4, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 4, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 5, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 5, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 6, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 6, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 7, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 7, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 8, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 8, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 9, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 9, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 10, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 10, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 11, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 11, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 12, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 12, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 13, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 13, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 14, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 14, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %ptr, i64 15, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i64 15, i32 0)
ret void; ret void;
} }
define void @st1q_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index) { define void @st1q_with_addr_offset(<vscale x 1 x i1> %pg, ptr %ptr, i64 %index) {
; CHECK-LABEL: st1q_with_addr_offset: ; CHECK-LABEL: st1q_with_addr_offset:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, wzr ; CHECK-NEXT: mov w12, wzr
; CHECK-NEXT: st1q {za0h.q[w12, 0]}, p0, [x0, x1, lsl #4] ; CHECK-NEXT: st1q {za0h.q[w12, 0]}, p0, [x0, x1, lsl #4]
; CHECK-NEXT: st1q {za15v.q[w12, 0]}, p0, [x0, x1, lsl #4] ; CHECK-NEXT: st1q {za15v.q[w12, 0]}, p0, [x0, x1, lsl #4]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%base = getelementptr i128, ptr %ptr, i64 %index %base = getelementptr i128, ptr %ptr, i64 %index
call void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1> %pg, ptr %base, i64 0, i32 0) call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> %pg, ptr %base, i64 0, i32 0)
call void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1> %pg, ptr %base, i64 15, i32 0) call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> %pg, ptr %base, i64 15, i32 0)
ret void; ret void;
} }
define void @str(ptr %ptr) { define void @str(ptr %ptr) {
; CHECK-LABEL: str: ; CHECK-LABEL: str:
; CHECK: // %bb.0: ; CHECK: // %bb.0:
; CHECK-NEXT: mov w12, wzr ; CHECK-NEXT: mov w12, wzr
; CHECK-NEXT: str za[w12, 0], [x0] ; CHECK-NEXT: str za[w12, 0], [x0]
Show All 38 Lines; CHECK-NEXT: ret
%mulvl = mul i64 %vscale, 256 %mulvl = mul i64 %vscale, 256
%base = getelementptr i8, ptr %ptr, i64 %mulvl %base = getelementptr i8, ptr %ptr, i64 %mulvl
call void @llvm.aarch64.sme.str(i32 0, ptr %base) call void @llvm.aarch64.sme.str(i32 0, ptr %base)
ret void; ret void;
} }
; Ensure that the tile offset is sunk, given that this is likely to be an 'add' ; Ensure that the tile offset is sunk, given that this is likely to be an 'add'
; that's decomposed into a base + offset in ISel. ; that's decomposed into a base + offset in ISel.
define void @test_sink_tile0_offset_operand(<vscale x 16 x i1> %pg, ptr %src, i32 %base, i32 %N) { define void @test_sink_tile0_offset_operand(<vscale x 4 x i1> %pg, ptr %src, i32 %base, i32 %N) {
; CHECK-LABEL: test_sink_tile0_offset_operand: ; CHECK-LABEL: test_sink_tile0_offset_operand:
; CHECK: // %bb.0: // %entry ; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov w12, w1 ; CHECK-NEXT: mov w12, w1
; CHECK-NEXT: .LBB14_1: // %for.body ; CHECK-NEXT: .LBB14_1: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: st1w {za0h.s[w12, 0]}, p0, [x0] ; CHECK-NEXT: st1w {za0h.s[w12, 0]}, p0, [x0]
; CHECK-NEXT: subs w2, w2, #1 ; CHECK-NEXT: subs w2, w2, #1
; CHECK-NEXT: st1w {za0h.s[w12, 1]}, p0, [x0] ; CHECK-NEXT: st1w {za0h.s[w12, 1]}, p0, [x0]
; CHECK-NEXT: st1w {za0h.s[w12, 2]}, p0, [x0] ; CHECK-NEXT: st1w {za0h.s[w12, 2]}, p0, [x0]
; CHECK-NEXT: b.ne .LBB14_1 ; CHECK-NEXT: b.ne .LBB14_1
; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret ; CHECK-NEXT: ret
entry: entry:
%add0 = add i32 %base, 1 %add0 = add i32 %base, 1
%add1 = add i32 %base, 2 %add1 = add i32 %base, 2
br label %for.body br label %for.body
for.body: for.body:
%i = phi i32 [ 0, %entry ], [ %inc, %for.body ] %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1> %pg, ptr %src, i64 0, i32 %base) tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i64 0, i32 %base)
tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1> %pg, ptr %src, i64 0, i32 %add0) tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i64 0, i32 %add0)
tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1> %pg, ptr %src, i64 0, i32 %add1) tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i64 0, i32 %add1)
%inc = add nuw nsw i32 %i, 1 %inc = add nuw nsw i32 %i, 1
%exitcond.not = icmp eq i32 %inc, %N %exitcond.not = icmp eq i32 %inc, %N
br i1 %exitcond.not, label %exit, label %for.body br i1 %exitcond.not, label %exit, label %for.body
exit: exit:
ret void ret void
} }
declare void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1h.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1w.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1d.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1q.horiz(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1h.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1w.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1d.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.st1q.vert(<vscale x 16 x i1>, ptr, i64, i32) declare void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1>, ptr, i64, i32)
declare void @llvm.aarch64.sme.str(i32, ptr) declare void @llvm.aarch64.sme.str(i32, ptr)
declare i64 @llvm.vscale.i64() declare i64 @llvm.vscale.i64()