Index: llvm/include/llvm/IR/IntrinsicsAArch64.td =================================================================== --- llvm/include/llvm/IR/IntrinsicsAArch64.td +++ llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1194,4 +1194,15 @@ LLVMPointerToElt<0>, llvm_anyvector_ty], [IntrReadMem, IntrArgMemOnly]>; + +// +// Gather loads: +// - vector base + immediate index +// + +def int_aarch64_sve_ld1_gather_imm : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty, + llvm_i64_ty], + [IntrReadMem, IntrArgMemOnly]>; } Index: llvm/lib/Target/AArch64/AArch64ISelLowering.h =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -203,6 +203,7 @@ GLD1_SXTW, GLD1_UXTW_SCALED, GLD1_SXTW_SCALED, + GLD1_IMM, // NEON Load/Store with post-increment base updates LD2post = ISD::FIRST_TARGET_MEMORY_OPCODE, Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1339,6 +1339,7 @@ case AArch64ISD::GLD1_UXTW: return "AArch64ISD::GLD1_UXTW"; case AArch64ISD::GLD1_SXTW_SCALED: return "AArch64ISD::GLD1_SXTW_SCALED"; case AArch64ISD::GLD1_UXTW_SCALED: return "AArch64ISD::GLD1_UXTW_SCALED"; + case AArch64ISD::GLD1_IMM: return "AArch64ISD::GLD1_IMM"; } return nullptr; } @@ -11930,6 +11931,8 @@ return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_SXTW_SCALED); case Intrinsic::aarch64_sve_ld1_gather_uxtw_index: return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_UXTW_SCALED); + case Intrinsic::aarch64_sve_ld1_gather_imm: + return performLD1GatherCombine(N, DAG, AArch64ISD::GLD1_IMM); default: break; } Index: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -15,12 +15,18 @@ SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1> ]>; +def SDT_AArch64_GLD1_IMM : SDTypeProfile<1, 4, [ + SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisInt<3>, SDTCisVT<4, OtherVT>, + SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1> +]>; + def AArch64ld1_gather : SDNode<"AArch64ISD::GLD1", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>; def AArch64ld1_gather_scaled : SDNode<"AArch64ISD::GLD1_SCALED", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>; def AArch64ld1_gather_uxtw : SDNode<"AArch64ISD::GLD1_UXTW", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>; def AArch64ld1_gather_sxtw : SDNode<"AArch64ISD::GLD1_SXTW", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>; def AArch64ld1_gather_uxtw_scaled : SDNode<"AArch64ISD::GLD1_UXTW_SCALED", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>; def AArch64ld1_gather_sxtw_scaled : SDNode<"AArch64ISD::GLD1_SXTW_SCALED", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>; +def AArch64ld1_gather_imm : SDNode<"AArch64ISD::GLD1_IMM", SDT_AArch64_GLD1_IMM, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>; let Predicates = [HasSVE] in { @@ -434,35 +440,35 @@ defm GLD1W : sve_mem_32b_gld_sv_32_scaled<0b1010, "ld1w", AArch64ld1_gather_sxtw_scaled, AArch64ld1_gather_uxtw_scaled, ZPR32ExtSXTW32, ZPR32ExtUXTW32, nxv4i32>; defm GLDFF1W : sve_mem_32b_gld_sv_32_scaled<0b1011, "ldff1w", null_frag, null_frag, ZPR32ExtSXTW32, ZPR32ExtUXTW32, nxv4i32>; - // Gathers using scaled 32-bit pointers with offset, e.g. + // Gathers using 32-bit pointers with scaled offset, e.g. // ld1h z0.s, p0/z, [z0.s, #16] - defm GLD1SB_S : sve_mem_32b_gld_vi_32_ptrs<0b0000, "ld1sb", imm0_31>; - defm GLDFF1SB_S : sve_mem_32b_gld_vi_32_ptrs<0b0001, "ldff1sb", imm0_31>; - defm GLD1B_S : sve_mem_32b_gld_vi_32_ptrs<0b0010, "ld1b", imm0_31>; - defm GLDFF1B_S : sve_mem_32b_gld_vi_32_ptrs<0b0011, "ldff1b", imm0_31>; - defm GLD1SH_S : sve_mem_32b_gld_vi_32_ptrs<0b0100, "ld1sh", uimm5s2>; - defm GLDFF1SH_S : sve_mem_32b_gld_vi_32_ptrs<0b0101, "ldff1sh", uimm5s2>; - defm GLD1H_S : sve_mem_32b_gld_vi_32_ptrs<0b0110, "ld1h", uimm5s2>; - defm GLDFF1H_S : sve_mem_32b_gld_vi_32_ptrs<0b0111, "ldff1h", uimm5s2>; - defm GLD1W : sve_mem_32b_gld_vi_32_ptrs<0b1010, "ld1w", uimm5s4>; - defm GLDFF1W : sve_mem_32b_gld_vi_32_ptrs<0b1011, "ldff1w", uimm5s4>; - - // Gathers using scaled 64-bit pointers with offset, e.g. + defm GLD1SB_S : sve_mem_32b_gld_vi_32_ptrs<0b0000, "ld1sb", imm0_31, null_frag, nxv4i8>; + defm GLDFF1SB_S : sve_mem_32b_gld_vi_32_ptrs<0b0001, "ldff1sb", imm0_31, null_frag, nxv4i8>; + defm GLD1B_S : sve_mem_32b_gld_vi_32_ptrs<0b0010, "ld1b", imm0_31, AArch64ld1_gather_imm, nxv4i8>; + defm GLDFF1B_S : sve_mem_32b_gld_vi_32_ptrs<0b0011, "ldff1b", imm0_31, null_frag, nxv4i8>; + defm GLD1SH_S : sve_mem_32b_gld_vi_32_ptrs<0b0100, "ld1sh", uimm5s2, null_frag, nxv4i16>; + defm GLDFF1SH_S : sve_mem_32b_gld_vi_32_ptrs<0b0101, "ldff1sh", uimm5s2, null_frag, nxv4i16>; + defm GLD1H_S : sve_mem_32b_gld_vi_32_ptrs<0b0110, "ld1h", uimm5s2, AArch64ld1_gather_imm, nxv4i16>; + defm GLDFF1H_S : sve_mem_32b_gld_vi_32_ptrs<0b0111, "ldff1h", uimm5s2, null_frag, nxv4i16>; + defm GLD1W : sve_mem_32b_gld_vi_32_ptrs<0b1010, "ld1w", uimm5s4, AArch64ld1_gather_imm, nxv4i32>; + defm GLDFF1W : sve_mem_32b_gld_vi_32_ptrs<0b1011, "ldff1w", uimm5s4, null_frag, nxv4i32>; + + // Gathers using 64-bit pointers with scaled offset, e.g. // ld1h z0.d, p0/z, [z0.d, #16] - defm GLD1SB_D : sve_mem_64b_gld_vi_64_ptrs<0b0000, "ld1sb", imm0_31>; - defm GLDFF1SB_D : sve_mem_64b_gld_vi_64_ptrs<0b0001, "ldff1sb", imm0_31>; - defm GLD1B_D : sve_mem_64b_gld_vi_64_ptrs<0b0010, "ld1b", imm0_31>; - defm GLDFF1B_D : sve_mem_64b_gld_vi_64_ptrs<0b0011, "ldff1b", imm0_31>; - defm GLD1SH_D : sve_mem_64b_gld_vi_64_ptrs<0b0100, "ld1sh", uimm5s2>; - defm GLDFF1SH_D : sve_mem_64b_gld_vi_64_ptrs<0b0101, "ldff1sh", uimm5s2>; - defm GLD1H_D : sve_mem_64b_gld_vi_64_ptrs<0b0110, "ld1h", uimm5s2>; - defm GLDFF1H_D : sve_mem_64b_gld_vi_64_ptrs<0b0111, "ldff1h", uimm5s2>; - defm GLD1SW_D : sve_mem_64b_gld_vi_64_ptrs<0b1000, "ld1sw", uimm5s4>; - defm GLDFF1SW_D : sve_mem_64b_gld_vi_64_ptrs<0b1001, "ldff1sw", uimm5s4>; - defm GLD1W_D : sve_mem_64b_gld_vi_64_ptrs<0b1010, "ld1w", uimm5s4>; - defm GLDFF1W_D : sve_mem_64b_gld_vi_64_ptrs<0b1011, "ldff1w", uimm5s4>; - defm GLD1D : sve_mem_64b_gld_vi_64_ptrs<0b1110, "ld1d", uimm5s8>; - defm GLDFF1D : sve_mem_64b_gld_vi_64_ptrs<0b1111, "ldff1d", uimm5s8>; + defm GLD1SB_D : sve_mem_64b_gld_vi_64_ptrs<0b0000, "ld1sb", imm0_31, null_frag, nxv2i8>; + defm GLDFF1SB_D : sve_mem_64b_gld_vi_64_ptrs<0b0001, "ldff1sb", imm0_31, null_frag, nxv2i8>; + defm GLD1B_D : sve_mem_64b_gld_vi_64_ptrs<0b0010, "ld1b", imm0_31, AArch64ld1_gather_imm, nxv2i8>; + defm GLDFF1B_D : sve_mem_64b_gld_vi_64_ptrs<0b0011, "ldff1b", imm0_31, null_frag, nxv2i8>; + defm GLD1SH_D : sve_mem_64b_gld_vi_64_ptrs<0b0100, "ld1sh", uimm5s2, null_frag, nxv2i16>; + defm GLDFF1SH_D : sve_mem_64b_gld_vi_64_ptrs<0b0101, "ldff1sh", uimm5s2, null_frag, nxv2i16>; + defm GLD1H_D : sve_mem_64b_gld_vi_64_ptrs<0b0110, "ld1h", uimm5s4, AArch64ld1_gather_imm, nxv2i16>; + defm GLDFF1H_D : sve_mem_64b_gld_vi_64_ptrs<0b0111, "ldff1h", uimm5s2, null_frag, nxv2i16>; + defm GLD1SW_D : sve_mem_64b_gld_vi_64_ptrs<0b1000, "ld1sw", uimm5s4, null_frag, nxv2i32>; + defm GLDFF1SW_D : sve_mem_64b_gld_vi_64_ptrs<0b1001, "ldff1sw", uimm5s4, null_frag, nxv2i32>; + defm GLD1W_D : sve_mem_64b_gld_vi_64_ptrs<0b1010, "ld1w", uimm5s4, AArch64ld1_gather_imm, nxv2i32>; + defm GLDFF1W_D : sve_mem_64b_gld_vi_64_ptrs<0b1011, "ldff1w", uimm5s4, null_frag, nxv2i32>; + defm GLD1D : sve_mem_64b_gld_vi_64_ptrs<0b1110, "ld1d", uimm5s8, AArch64ld1_gather_imm, nxv2i64>; + defm GLDFF1D : sve_mem_64b_gld_vi_64_ptrs<0b1111, "ldff1d", uimm5s8, null_frag, nxv2i64>; // Gathers using unscaled 64-bit offsets, e.g. // ld1h z0.d, p0/z, [x0, z0.d] Index: llvm/lib/Target/AArch64/SVEInstrFormats.td =================================================================== --- llvm/lib/Target/AArch64/SVEInstrFormats.td +++ llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -5302,7 +5302,8 @@ let Uses = !if(!eq(opc{0}, 1), [FFR], []); } -multiclass sve_mem_32b_gld_vi_32_ptrs opc, string asm, Operand imm_ty> { +multiclass sve_mem_32b_gld_vi_32_ptrs opc, string asm, Operand imm_ty, + SDPatternOperator op, ValueType vt> { def _IMM_REAL : sve_mem_32b_gld_vi; def : InstAlias(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; def : InstAlias(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; + + def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), + (!cast(NAME # _IMM_REAL) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; } class sve_mem_prfm_si msz, string asm> @@ -5632,7 +5636,8 @@ let Uses = !if(!eq(opc{0}, 1), [FFR], []); } -multiclass sve_mem_64b_gld_vi_64_ptrs opc, string asm, Operand imm_ty> { +multiclass sve_mem_64b_gld_vi_64_ptrs opc, string asm, Operand imm_ty, + SDPatternOperator op, ValueType vt> { def _IMM_REAL : sve_mem_64b_gld_vi; def : InstAlias(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; def : InstAlias(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; + + def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), + (!cast(NAME # _IMM_REAL) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; } // bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) Index: llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-vector-base.ll @@ -0,0 +1,139 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s + +; +; LD1B, LD1W, LD1H, LD1D: vector + immediate (index) +; e.g. ld1h { z0.s }, p0/z, [z0.s, #16] +; + +; LD1B +define @gld1b_s_imm( %pg, %base) { +; CHECK-LABEL: gld1b_s_imm: +; CHECK: ld1b { z0.s }, p0/z, [z0.s, #16] +; CHECK-NEXT: mov w8, #255 +; CHECK-NEXT: mov z1.s, w8 +; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv4i8.nxv4i32( %pg, + %base, + i64 16) + %res = zext %load to + ret %res +} + +define @gld1b_d_imm( %pg, %base) { +; CHECK-LABEL: gld1b_d_imm: +; CHECK: ld1b { z0.d }, p0/z, [z0.d, #16] +; CHECK-NEXT: mov w8, #255 +; CHECK-NEXT: mov z1.d, x8 +; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64( %pg, + %base, + i64 16) + %res = zext %load to + ret %res +} + +; LD1H +define @gld1h_s_imm( %pg, %base) { +; CHECK-LABEL: gld1h_s_imm: +; CHECK: ld1h { z0.s }, p0/z, [z0.s, #16] +; CHECK-NEXT: mov w8, #65535 +; CHECK-NEXT: mov z1.s, w8 +; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv4i16.nxv4i32( %pg, + %base, + i64 16) + %res = zext %load to + ret %res +} + +define @gld1h_d_imm( %pg, %base) { +; CHECK-LABEL: gld1h_d_imm: +; CHECK: ld1h { z0.d }, p0/z, [z0.d, #16] +; CHECK-NEXT: mov w8, #65535 +; CHECK-NEXT: mov z1.d, x8 +; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv2i16.nxv2i64( %pg, + %base, + i64 16) + %res = zext %load to + ret %res +} + +; LD1W +define @gld1w_s_imm( %pg, %base) { +; CHECK-LABEL: gld1w_s_imm: +; CHECK: ld1w { z0.s }, p0/z, [z0.s, #16] +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv4i32.nxv4i32( %pg, + %base, + i64 16) + ret %load +} + +define @gld1w_d_imm( %pg, %base) { +; CHECK-LABEL: gld1w_d_imm: +; CHECK: ld1w { z0.d }, p0/z, [z0.d, #16] +; CHECK-NEXT: mov w8, #-1 +; CHECK-NEXT: mov z1.d, x8 +; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv2i32.nxv2i64( %pg, + %base, + i64 16) + %res = zext %load to + ret %res +} + +define @gld1w_s_imm_float( %pg, %base) { +; CHECK-LABEL: gld1w_s_imm_float: +; CHECK: ld1w { z0.s }, p0/z, [z0.s, #16] +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv4f32.nxv4i32( %pg, + %base, + i64 16) + ret %load +} + +; LD1D +define @gld1d_d_imm( %pg, %base) { +; CHECK-LABEL: gld1d_d_imm: +; CHECK: ld1d { z0.d }, p0/z, [z0.d, #16] +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv2i64.nxv2i64( %pg, + %base, + i64 16) + ret %load +} + +define @gld1d_d_imm_double( %pg, %base) { +; CHECK-LABEL: gld1d_d_imm_double: +; CHECK: ld1d { z0.d }, p0/z, [z0.d, #16] +; CHECK-NEXT: ret + %load = call @llvm.aarch64.sve.ld1.gather.imm.nxv2f64.nxv2i64( %pg, + %base, + i64 16) + ret %load +} + +; LD1B +declare @llvm.aarch64.sve.ld1.gather.imm.nxv4i8.nxv4i32(, , i64) +declare @llvm.aarch64.sve.ld1.gather.imm.nxv2i8.nxv2i64(, , i64) + +; LD1H +declare @llvm.aarch64.sve.ld1.gather.imm.nxv4i16.nxv4i32(, , i64) +declare @llvm.aarch64.sve.ld1.gather.imm.nxv2i16.nxv2i64(, , i64) + +; LD1W +declare @llvm.aarch64.sve.ld1.gather.imm.nxv4i32.nxv4i32(, , i64) +declare @llvm.aarch64.sve.ld1.gather.imm.nxv2i32.nxv2i64(, , i64) + +declare @llvm.aarch64.sve.ld1.gather.imm.nxv4f32.nxv4i32(, , i64) + +; LD1D +declare @llvm.aarch64.sve.ld1.gather.imm.nxv2i64.nxv2i64(, , i64) + +declare @llvm.aarch64.sve.ld1.gather.imm.nxv2f64.nxv2i64(, , i64)