This is an archive of the discontinued LLVM Phabricator instance.

Differential D71724

[AArch64][SVE] Gather loads: pass 32 bit unpacked offsets as nxv2i32
ClosedPublic

Authored by andwar on Dec 19 2019, 12:55 PM.

Details

Reviewers
sdesmalen
efriedma
Commits
rG404da13e1e94: [AArch64][SVE] Gather loads: pass 32 bit unpacked offsets as nxv2i32
Summary

Currently 32 bit unpacked offsets are passed as nxv2i64. However, as
pointed out in https://reviews.llvm.org/D71074, using nxv2i32 instead
would improve consistency with:

  • how other arguments are treated
  • how scatter stores are implemented

This patch makes sure that 32 bit unpacked offsets are passes as nxv2i32
instead of nxv2i64.

Diff Detail

Event Timeline

andwar created this revision.Dec 19 2019, 12:55 PM
Herald added a reviewer: efriedma. · View Herald TranscriptDec 19 2019, 12:55 PM
Herald added a project: Restricted Project. · View Herald Transcript
Herald added subscribers: llvm-commits, psnobl, rkruppe and 3 others. · View Herald Transcript
Harbormaster completed remote builds in B42793: Diff 234773.Dec 19 2019, 1:03 PM
efriedma accepted this revision.Dec 19 2019, 2:17 PM

LGTM

This revision is now accepted and ready to land.Dec 19 2019, 2:17 PM
Closed by commit rG404da13e1e94: [AArch64][SVE] Gather loads: pass 32 bit unpacked offsets as nxv2i32 (authored by awarzynski). · Explain WhyJan 2 2020, 5:04 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
llvm/
include/
llvm/
IR/
3 lines
lib/
Target/
AArch64/
35 lines
test/
CodeGen/
AArch64/
172 lines
236 lines

Diff 235853

llvm/include/llvm/IR/IntrinsicsAArch64.td

Show First 20 Lines • Show All 1,108 Lines • ▼ Show 20 Lines : Intrinsic<[llvm_anyvector_ty],
LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
], ],
[IntrReadMem, IntrArgMemOnly]>; [IntrReadMem, IntrArgMemOnly]>;
class AdvSIMD_GatherLoad_32bitOffset_Intrinsic class AdvSIMD_GatherLoad_32bitOffset_Intrinsic
: Intrinsic<[llvm_anyvector_ty], : Intrinsic<[llvm_anyvector_ty],
[ [
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
LLVMPointerToElt<0>, llvm_anyvector_ty LLVMPointerToElt<0>,
LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
], ],
[IntrReadMem, IntrArgMemOnly]>; [IntrReadMem, IntrArgMemOnly]>;
class AdvSIMD_GatherLoad_VecTorBase_Intrinsic class AdvSIMD_GatherLoad_VecTorBase_Intrinsic
: Intrinsic<[llvm_anyvector_ty], : Intrinsic<[llvm_anyvector_ty],
[ [
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_anyvector_ty, llvm_anyvector_ty,
▲ Show 20 Lines • Show All 564 Lines • Show Last 20 Lines

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 12,225 Lines • ▼ Show 20 Lines SDValue Ops[] = {N->getOperand(0), // Chain
Base, Base,
Offset, Offset,
InputVT}; InputVT};
return DAG.getNode(Opcode, DL, VTs, Ops); return DAG.getNode(Opcode, DL, VTs, Ops);
} }
static SDValue performLD1GatherCombine(SDNode *N, SelectionDAG &DAG, static SDValue performLD1GatherCombine(SDNode *N, SelectionDAG &DAG,
unsigned Opcode) { unsigned Opcode,
bool OnlyPackedOffsets = true) {
EVT RetVT = N->getValueType(0); EVT RetVT = N->getValueType(0);
assert(RetVT.isScalableVector() && assert(RetVT.isScalableVector() &&
"Gather loads are only possible for SVE vectors"); "Gather loads are only possible for SVE vectors");
SDLoc DL(N); SDLoc DL(N);
MVT RetElVT = RetVT.getVectorElementType().getSimpleVT();
unsigned NumElements = AArch64::SVEBitsPerBlock / RetElVT.getSizeInBits();
EVT MaxVT = llvm::MVT::getScalableVectorVT(RetElVT, NumElements); if (RetVT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
if (RetVT.getSizeInBits().getKnownMinSize() >
MaxVT.getSizeInBits().getKnownMinSize())
return SDValue(); return SDValue();
// Depending on the addressing mode, this is either a pointer or a vector of // Depending on the addressing mode, this is either a pointer or a vector of
// pointers (that fits into one register) // pointers (that fits into one register)
const SDValue Base = N->getOperand(3); const SDValue Base = N->getOperand(3);
// Depending on the addressing mode, this is either a single offset or a // Depending on the addressing mode, this is either a single offset or a
// vector of offsets (that fits into one register) // vector of offsets (that fits into one register)
const SDValue Offset = N->getOperand(4); SDValue Offset = N->getOperand(4);
if (!DAG.getTargetLoweringInfo().isTypeLegal(Base.getValueType()) || auto &TLI = DAG.getTargetLoweringInfo();
!DAG.getTargetLoweringInfo().isTypeLegal(Offset.getValueType())) if (!TLI.isTypeLegal(Base.getValueType()))
return SDValue(); return SDValue();
// Some gather load variants allow unpacked offsets, but only as nxv2i32
// vectors. These are implicitly sign (sxtw) or zero (zxtw) extend to
// nxv2i64. Legalize accordingly.
if (!OnlyPackedOffsets &&
Offset.getValueType().getSimpleVT().SimpleTy == MVT::nxv2i32)
Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0);
// Return value type that is representable in hardware // Return value type that is representable in hardware
EVT HwRetVt = getSVEContainerType(RetVT); EVT HwRetVt = getSVEContainerType(RetVT);
// Keep the original output value type around - this will better inform // Keep the original output value type around - this will better inform
// optimisations (e.g. instruction folding when load is followed by // optimisations (e.g. instruction folding when load is followed by
// zext/sext). This will only be used for ints, so the value for FPs // zext/sext). This will only be used for ints, so the value for FPs
// doesn't matter. // doesn't matter.
SDValue OutVT = DAG.getValueType(RetVT); SDValue OutVT = DAG.getValueType(RetVT);
▲ Show 20 Lines • Show All 167 Lines • ▼ Show 20 Lines case Intrinsic::aarch64_sve_ldnt1:
return performLDNT1Combine(N, DAG); return performLDNT1Combine(N, DAG);
case Intrinsic::aarch64_sve_stnt1: case Intrinsic::aarch64_sve_stnt1:
return performSTNT1Combine(N, DAG); return performSTNT1Combine(N, DAG);
case Intrinsic::aarch64_sve_ld1_gather: case Intrinsic::aarch64_sve_ld1_gather:
return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1); return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1);
case Intrinsic::aarch64_sve_ld1_gather_index: case Intrinsic::aarch64_sve_ld1_gather_index:
return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_SCALED); return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_SCALED);
case Intrinsic::aarch64_sve_ld1_gather_sxtw: case Intrinsic::aarch64_sve_ld1_gather_sxtw:
return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_SXTW); return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_SXTW,
/*OnlyPackedOffsets=*/false);
case Intrinsic::aarch64_sve_ld1_gather_uxtw: case Intrinsic::aarch64_sve_ld1_gather_uxtw:
return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_UXTW); return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_UXTW,
/*OnlyPackedOffsets=*/false);
case Intrinsic::aarch64_sve_ld1_gather_sxtw_index: case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_SXTW_SCALED); return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_SXTW_SCALED,
/*OnlyPackedOffsets=*/false);
case Intrinsic::aarch64_sve_ld1_gather_uxtw_index: case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_UXTW_SCALED); return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_UXTW_SCALED,
/*OnlyPackedOffsets=*/false);
case Intrinsic::aarch64_sve_ld1_gather_imm: case Intrinsic::aarch64_sve_ld1_gather_imm:
return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_IMM); return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_IMM);
case Intrinsic::aarch64_sve_st1_scatter: case Intrinsic::aarch64_sve_st1_scatter:
return performST1ScatterCombine(N, DAG, AArch64ISD::SST1); return performST1ScatterCombine(N, DAG, AArch64ISD::SST1);
case Intrinsic::aarch64_sve_st1_scatter_index: case Intrinsic::aarch64_sve_st1_scatter_index:
return performST1ScatterCombine(N, DAG, AArch64ISD::SST1_SCALED); return performST1ScatterCombine(N, DAG, AArch64ISD::SST1_SCALED);
case Intrinsic::aarch64_sve_st1_scatter_sxtw: case Intrinsic::aarch64_sve_st1_scatter_sxtw:
return performST1ScatterCombine(N, DAG, AArch64ISD::SST1_SXTW, return performST1ScatterCombine(N, DAG, AArch64ISD::SST1_SXTW,
▲ Show 20 Lines • Show All 728 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-scaled-offsets.ll

; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
; ;
; LD1H, LD1W, LD1D: base + 32-bit scaled offset, sign (sxtw) or zero (uxtw) ; LD1H, LD1W, LD1D: base + 32-bit scaled offset, sign (sxtw) or zero (uxtw)
; extended to 64 bits ; extended to 64 bits
; e.g. ld1h z0.d, p0/z, [x0, z0.d, uxtw #1] ; e.g. ld1h z0.d, p0/z, [x0, z0.d, uxtw #1]
; ;
; LD1H ; LD1H
define <vscale x 4 x i32> @gld1h_s_uxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1h_s_uxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1h_s_uxtw_index: ; CHECK-LABEL: gld1h_s_uxtw_index:
; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw #1] ; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> %pg,
i16* %base, i16* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32> %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 4 x i32> @gld1h_s_sxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1h_s_sxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1h_s_sxtw_index: ; CHECK-LABEL: gld1h_s_sxtw_index:
; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1] ; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> %pg,
i16* %base, i16* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32> %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 2 x i64> @gld1h_d_uxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1h_d_uxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1h_d_uxtw_index: ; CHECK-LABEL: gld1h_d_uxtw_index:
; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1] ; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16(<vscale x 2 x i1> %pg,
i16* %base, i16* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1h_d_sxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1h_d_sxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1h_d_sxtw_index: ; CHECK-LABEL: gld1h_d_sxtw_index:
; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1] ; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16(<vscale x 2 x i1> %pg,
i16* %base, i16* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
; LD1W ; LD1W
define <vscale x 4 x i32> @gld1w_s_uxtw_index(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1w_s_uxtw_index(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1w_s_uxtw_index: ; CHECK-LABEL: gld1w_s_uxtw_index:
; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2] ; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> %pg,
i32* %base, i32* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
ret <vscale x 4 x i32> %load ret <vscale x 4 x i32> %load
} }
define <vscale x 4 x i32> @gld1w_s_sxtw_index(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1w_s_sxtw_index(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1w_s_sxtw_index: ; CHECK-LABEL: gld1w_s_sxtw_index:
; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2] ; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> %pg,
i32* %base, i32* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
ret <vscale x 4 x i32> %load ret <vscale x 4 x i32> %load
} }
define <vscale x 2 x i64> @gld1w_d_uxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1w_d_uxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1w_d_uxtw_index: ; CHECK-LABEL: gld1w_d_uxtw_index:
; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] ; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32(<vscale x 2 x i1> %pg,
i32* %base, i32* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1w_d_sxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1w_d_sxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1w_d_sxtw_index: ; CHECK-LABEL: gld1w_d_sxtw_index:
; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw #2] ; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32(<vscale x 2 x i1> %pg,
i32* %base, i32* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 4 x float> @gld1w_s_uxtw_index_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x float> @gld1w_s_uxtw_index_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1w_s_uxtw_index_float: ; CHECK-LABEL: gld1w_s_uxtw_index_float:
; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2] ; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4f32.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4f32(<vscale x 4 x i1> %pg,
float* %base, float* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
ret <vscale x 4 x float> %load ret <vscale x 4 x float> %load
} }
define <vscale x 4 x float> @gld1w_s_sxtw_index_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x float> @gld1w_s_sxtw_index_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1w_s_sxtw_index_float: ; CHECK-LABEL: gld1w_s_sxtw_index_float:
; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2] ; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4f32.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4f32(<vscale x 4 x i1> %pg,
float* %base, float* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
ret <vscale x 4 x float> %load ret <vscale x 4 x float> %load
} }
; LD1D ; LD1D
define <vscale x 2 x i64> @gld1d_s_uxtw_index(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1d_s_uxtw_index(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1d_s_uxtw_index: ; CHECK-LABEL: gld1d_s_uxtw_index:
; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3] ; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i64.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i64(<vscale x 2 x i1> %pg,
i64* %base, i64* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
ret <vscale x 2 x i64> %load ret <vscale x 2 x i64> %load
} }
define <vscale x 2 x i64> @gld1d_sxtw_index(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1d_sxtw_index(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1d_sxtw_index: ; CHECK-LABEL: gld1d_sxtw_index:
; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3] ; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i64.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i64(<vscale x 2 x i1> %pg,
i64* %base, i64* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
ret <vscale x 2 x i64> %load ret <vscale x 2 x i64> %load
} }
define <vscale x 2 x double> @gld1d_uxtw_index_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x double> @gld1d_uxtw_index_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1d_uxtw_index_double: ; CHECK-LABEL: gld1d_uxtw_index_double:
; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3] ; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2f64.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2f64(<vscale x 2 x i1> %pg,
double* %base, double* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
ret <vscale x 2 x double> %load ret <vscale x 2 x double> %load
} }
define <vscale x 2 x double> @gld1d_sxtw_index_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x double> @gld1d_sxtw_index_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1d_sxtw_index_double: ; CHECK-LABEL: gld1d_sxtw_index_double:
; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3] ; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2f64.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2f64(<vscale x 2 x i1> %pg,
double* %base, double* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
ret <vscale x 2 x double> %load ret <vscale x 2 x double> %load
} }
; ;
; LD1SH, LD1SW, LD1SD: base + 32-bit scaled offset, sign (sxtw) or zero (uxtw) ; LD1SH, LD1SW, LD1SD: base + 32-bit scaled offset, sign (sxtw) or zero (uxtw)
; extended to 64 bits ; extended to 64 bits
; e.g. ld1sh z0.d, p0/z, [x0, z0.d, uxtw #1] ; e.g. ld1sh z0.d, p0/z, [x0, z0.d, uxtw #1]
; ;
; LD1SH ; LD1SH
define <vscale x 4 x i32> @gld1sh_s_uxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1sh_s_uxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1sh_s_uxtw_index: ; CHECK-LABEL: gld1sh_s_uxtw_index:
; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw #1] ; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> %pg,
i16* %base, i16* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32> %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 4 x i32> @gld1sh_s_sxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1sh_s_sxtw_index(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1sh_s_sxtw_index: ; CHECK-LABEL: gld1sh_s_sxtw_index:
; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw #1] ; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> %pg,
i16* %base, i16* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32> %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 2 x i64> @gld1sh_d_uxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sh_d_uxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sh_d_uxtw_index: ; CHECK-LABEL: gld1sh_d_uxtw_index:
; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] ; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16(<vscale x 2 x i1> %pg,
i16* %base, i16* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1sh_d_sxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sh_d_sxtw_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sh_d_sxtw_index: ; CHECK-LABEL: gld1sh_d_sxtw_index:
; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1] ; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16(<vscale x 2 x i1> %pg,
i16* %base, i16* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
; LD1SW ; LD1SW
define <vscale x 2 x i64> @gld1sw_d_uxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sw_d_uxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sw_d_uxtw_index: ; CHECK-LABEL: gld1sw_d_uxtw_index:
; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2] ; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32(<vscale x 2 x i1> %pg,
i32* %base, i32* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1sw_d_sxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sw_d_sxtw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sw_d_sxtw_index: ; CHECK-LABEL: gld1sw_d_sxtw_index:
; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2] ; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32(<vscale x 2 x i1> %pg,
i32* %base, i32* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
; LD1H/LD1SH ; LD1H/LD1SH
declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16.nxv4i32(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>) declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16.nxv4i32(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>) declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16.nxv2i64(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>) declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16.nxv2i64(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>) declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
; LD1W/LD1SW ; LD1W/LD1SW
declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>) declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>) declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32.nxv2i64(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>) declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32.nxv2i64(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>) declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4f32.nxv4i32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>) declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4f32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>)
declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4f32.nxv4i32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>) declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4f32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>)
; LD1D ; LD1D
declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i64.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>) declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i64.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>) declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2f64.nxv2i64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>) declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i32>)
declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2f64.nxv2i64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>) declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i32>)

llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-32bit-unscaled-offsets.ll

; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
; ;
; LD1B, LD1W, LD1H, LD1D: base + 32-bit unscaled offset, sign (sxtw) or zero ; LD1B, LD1W, LD1H, LD1D: base + 32-bit unscaled offset, sign (sxtw) or zero
; (uxtw) extended to 64 bits. ; (uxtw) extended to 64 bits.
; e.g. ld1h { z0.d }, p0/z, [x0, z0.d, uxtw] ; e.g. ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]
; ;
; LD1B ; LD1B
define <vscale x 4 x i32> @gld1b_s_uxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1b_s_uxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1b_s_uxtw: ; CHECK-LABEL: gld1b_s_uxtw:
; CHECK: ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] ; CHECK: ld1b { z0.s }, p0/z, [x0, z0.s, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> %pg,
i8* %base, i8* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32> %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 4 x i32> @gld1b_s_sxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1b_s_sxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1b_s_sxtw: ; CHECK-LABEL: gld1b_s_sxtw:
; CHECK: ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] ; CHECK: ld1b { z0.s }, p0/z, [x0, z0.s, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> %pg,
i8* %base, i8* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32> %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 2 x i64> @gld1b_d_uxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1b_d_uxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1b_d_uxtw: ; CHECK-LABEL: gld1b_d_uxtw:
; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, uxtw] ; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8(<vscale x 2 x i1> %pg,
i8* %base, i8* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1b_d_sxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1b_d_sxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1b_d_sxtw: ; CHECK-LABEL: gld1b_d_sxtw:
; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, sxtw] ; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8(<vscale x 2 x i1> %pg,
i8* %base, i8* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
; LD1H ; LD1H
define <vscale x 4 x i32> @gld1h_s_uxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1h_s_uxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1h_s_uxtw: ; CHECK-LABEL: gld1h_s_uxtw:
; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] ; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> %pg,
i16* %base, i16* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32> %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 4 x i32> @gld1h_s_sxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1h_s_sxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1h_s_sxtw: ; CHECK-LABEL: gld1h_s_sxtw:
; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] ; CHECK: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> %pg,
i16* %base, i16* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32> %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 2 x i64> @gld1h_d_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1h_d_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1h_d_uxtw: ; CHECK-LABEL: gld1h_d_uxtw:
; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw] ; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16(<vscale x 2 x i1> %pg,
i16* %base, i16* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1h_d_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1h_d_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1h_d_sxtw: ; CHECK-LABEL: gld1h_d_sxtw:
; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw] ; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16(<vscale x 2 x i1> %pg,
i16* %base, i16* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
; LD1W ; LD1W
define <vscale x 4 x i32> @gld1w_s_uxtw(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1w_s_uxtw(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1w_s_uxtw: ; CHECK-LABEL: gld1w_s_uxtw:
; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] ; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32(<vscale x 4 x i1> %pg,
i32* %base, i32* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
ret <vscale x 4 x i32> %load ret <vscale x 4 x i32> %load
} }
define <vscale x 4 x i32> @gld1w_s_sxtw(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1w_s_sxtw(<vscale x 4 x i1> %pg, i32* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1w_s_sxtw: ; CHECK-LABEL: gld1w_s_sxtw:
; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] ; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32(<vscale x 4 x i1> %pg,
i32* %base, i32* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
ret <vscale x 4 x i32> %load ret <vscale x 4 x i32> %load
} }
define <vscale x 2 x i64> @gld1w_d_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1w_d_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1w_d_uxtw: ; CHECK-LABEL: gld1w_d_uxtw:
; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw] ; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32(<vscale x 2 x i1> %pg,
i32* %base, i32* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1w_d_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1w_d_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1w_d_sxtw: ; CHECK-LABEL: gld1w_d_sxtw:
; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw] ; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32(<vscale x 2 x i1> %pg,
i32* %base, i32* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64> %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 4 x float> @gld1w_s_uxtw_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x float> @gld1w_s_uxtw_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1w_s_uxtw_float: ; CHECK-LABEL: gld1w_s_uxtw_float:
; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] ; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4f32.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4f32(<vscale x 4 x i1> %pg,
float* %base, float* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
ret <vscale x 4 x float> %load ret <vscale x 4 x float> %load
} }
define <vscale x 4 x float> @gld1w_s_sxtw_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x float> @gld1w_s_sxtw_float(<vscale x 4 x i1> %pg, float* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1w_s_sxtw_float: ; CHECK-LABEL: gld1w_s_sxtw_float:
; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] ; CHECK: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4f32.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4f32(<vscale x 4 x i1> %pg,
float* %base, float* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
ret <vscale x 4 x float> %load ret <vscale x 4 x float> %load
} }
; LD1D ; LD1D
define <vscale x 2 x i64> @gld1d_d_uxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1d_d_uxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1d_d_uxtw: ; CHECK-LABEL: gld1d_d_uxtw:
; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw] ; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i64.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i64(<vscale x 2 x i1> %pg,
i64* %base, i64* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
ret <vscale x 2 x i64> %load ret <vscale x 2 x i64> %load
} }
define <vscale x 2 x i64> @gld1d_d_sxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1d_d_sxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1d_d_sxtw: ; CHECK-LABEL: gld1d_d_sxtw:
; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw] ; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i64.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i64(<vscale x 2 x i1> %pg,
i64* %base, i64* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
ret <vscale x 2 x i64> %load ret <vscale x 2 x i64> %load
} }
define <vscale x 2 x double> @gld1d_d_uxtw_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x double> @gld1d_d_uxtw_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1d_d_uxtw_double: ; CHECK-LABEL: gld1d_d_uxtw_double:
; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw] ; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2f64.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2f64(<vscale x 2 x i1> %pg,
double* %base, double* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
ret <vscale x 2 x double> %load ret <vscale x 2 x double> %load
} }
define <vscale x 2 x double> @gld1d_d_sxtw_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x double> @gld1d_d_sxtw_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1d_d_sxtw_double: ; CHECK-LABEL: gld1d_d_sxtw_double:
; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw] ; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2f64.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2f64(<vscale x 2 x i1> %pg,
double* %base, double* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
ret <vscale x 2 x double> %load ret <vscale x 2 x double> %load
} }
; ;
; LD1SB, LD1SW, LD1SH: base + 32-bit unscaled offset, sign (sxtw) or zero ; LD1SB, LD1SW, LD1SH: base + 32-bit unscaled offset, sign (sxtw) or zero
; (uxtw) extended to 64 bits. ; (uxtw) extended to 64 bits.
; e.g. ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw] ; e.g. ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw]
; ;
; LD1SB ; LD1SB
define <vscale x 4 x i32> @gld1sb_s_uxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1sb_s_uxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1sb_s_uxtw: ; CHECK-LABEL: gld1sb_s_uxtw:
; CHECK: ld1sb { z0.s }, p0/z, [x0, z0.s, uxtw] ; CHECK: ld1sb { z0.s }, p0/z, [x0, z0.s, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> %pg,
i8* %base, i8* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32> %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 4 x i32> @gld1sb_s_sxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1sb_s_sxtw(<vscale x 4 x i1> %pg, i8* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1sb_s_sxtw: ; CHECK-LABEL: gld1sb_s_sxtw:
; CHECK: ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] ; CHECK: ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> %pg,
i8* %base, i8* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32> %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 2 x i64> @gld1sb_d_uxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sb_d_uxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sb_d_uxtw: ; CHECK-LABEL: gld1sb_d_uxtw:
; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, uxtw] ; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8(<vscale x 2 x i1> %pg,
i8* %base, i8* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1sb_d_sxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sb_d_sxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sb_d_sxtw: ; CHECK-LABEL: gld1sb_d_sxtw:
; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, sxtw] ; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8(<vscale x 2 x i1> %pg,
i8* %base, i8* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
; LD1SH ; LD1SH
define <vscale x 4 x i32> @gld1sh_s_uxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1sh_s_uxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1sh_s_uxtw: ; CHECK-LABEL: gld1sh_s_uxtw:
; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] ; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> %pg,
i16* %base, i16* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32> %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 4 x i32> @gld1sh_s_sxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) { define <vscale x 4 x i32> @gld1sh_s_sxtw(<vscale x 4 x i1> %pg, i16* %base, <vscale x 4 x i32> %b) {
; CHECK-LABEL: gld1sh_s_sxtw: ; CHECK-LABEL: gld1sh_s_sxtw:
; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] ; CHECK: ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16.nxv4i32(<vscale x 4 x i1> %pg, %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> %pg,
i16* %base, i16* %base,
<vscale x 4 x i32> %b) <vscale x 4 x i32> %b)
%res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32> %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
ret <vscale x 4 x i32> %res ret <vscale x 4 x i32> %res
} }
define <vscale x 2 x i64> @gld1sh_d_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sh_d_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sh_d_uxtw: ; CHECK-LABEL: gld1sh_d_uxtw:
; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw] ; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16(<vscale x 2 x i1> %pg,
i16* %base, i16* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1sh_d_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sh_d_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sh_d_sxtw: ; CHECK-LABEL: gld1sh_d_sxtw:
; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw] ; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16(<vscale x 2 x i1> %pg,
i16* %base, i16* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
; LD1SW ; LD1SW
define <vscale x 2 x i64> @gld1sw_d_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sw_d_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sw_d_uxtw: ; CHECK-LABEL: gld1sw_d_uxtw:
; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw] ; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32(<vscale x 2 x i1> %pg,
i32* %base, i32* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
define <vscale x 2 x i64> @gld1sw_d_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) { define <vscale x 2 x i64> @gld1sw_d_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i32> %b) {
; CHECK-LABEL: gld1sw_d_sxtw: ; CHECK-LABEL: gld1sw_d_sxtw:
; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw] ; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw]
; CHECK-NEXT: ret ; CHECK-NEXT: ret
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32.nxv2i64(<vscale x 2 x i1> %pg, %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32(<vscale x 2 x i1> %pg,
i32* %base, i32* %base,
<vscale x 2 x i64> %b) <vscale x 2 x i32> %b)
%res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64> %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
ret <vscale x 2 x i64> %res ret <vscale x 2 x i64> %res
} }
; LD1B/LD1SB ; LD1B/LD1SB
declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8.nxv4i32(<vscale x 4 x i1>, i8*, <vscale x 4 x i32>) declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1>, i8*, <vscale x 4 x i32>)
declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8.nxv2i64(<vscale x 2 x i1>, i8*, <vscale x 2 x i64>) declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i8(<vscale x 2 x i1>, i8*, <vscale x 2 x i32>)
declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8.nxv4i32(<vscale x 4 x i1>, i8*, <vscale x 4 x i32>) declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1>, i8*, <vscale x 4 x i32>)
declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8.nxv2i64(<vscale x 2 x i1>, i8*, <vscale x 2 x i64>) declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i8(<vscale x 2 x i1>, i8*, <vscale x 2 x i32>)
; LD1H/LD1SH ; LD1H/LD1SH
declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16.nxv4i32(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>) declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16.nxv2i64(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>) declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16.nxv4i32(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>) declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1>, i16*, <vscale x 4 x i32>)
declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16.nxv2i64(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>) declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i32>)
; LD1W/LD1SW ; LD1W/LD1SW
declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>) declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32.nxv2i64(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>) declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>) declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32(<vscale x 4 x i1>, i32*, <vscale x 4 x i32>)
declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32.nxv2i64(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>) declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i32>)
declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4f32.nxv4i32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>) declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4f32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>)
declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4f32.nxv4i32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>) declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4f32(<vscale x 4 x i1>, float*, <vscale x 4 x i32>)
; LD1D ; LD1D
declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i64.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>) declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i64.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>) declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i32>)
declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2f64.nxv2i64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>) declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.sxtw.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i32>)
declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2f64.nxv2i64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>) declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.uxtw.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i32>)