diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -1254,6 +1254,27 @@ ], [IntrWriteMem, IntrArgMemOnly]>; + +class SVE_gather_prf_scalar_base_vector_offset_scaled + : Intrinsic<[], + [ + llvm_ptr_ty, // Base address + llvm_anyvector_ty, // Offsets + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate + llvm_i32_ty // Prfop + ], + [IntrInaccessibleMemOrArgMemOnly, NoCapture<0>, ImmArg<3>]>; + +class SVE_gather_prf_vector_base_scalar_offset + : Intrinsic<[], + [ + llvm_anyvector_ty, // Base addresses + llvm_i64_ty, // Scalar offset + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate + llvm_i32_ty // Prfop + ], + [IntrInaccessibleMemOrArgMemOnly, ImmArg<3>]>; + // // Loads // @@ -1270,13 +1291,39 @@ def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic; // -// Prefetch +// Prefetches // def int_aarch64_sve_prf : Intrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty], [IntrArgMemOnly, ImmArg<2>]>; +// Scalar + 32-bit scaled offset vector, zero extend, packed and +// unpacked. +def int_aarch64_sve_gather_prfb_scaled_uxtw : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfh_scaled_uxtw : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfw_scaled_uxtw : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfd_scaled_uxtw : SVE_gather_prf_scalar_base_vector_offset_scaled; + +// Scalar + 32-bit scaled offset vector, sign extend, packed and +// unpacked. +def int_aarch64_sve_gather_prfb_scaled_sxtw : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfw_scaled_sxtw : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfh_scaled_sxtw : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfd_scaled_sxtw : SVE_gather_prf_scalar_base_vector_offset_scaled; + +// Scalar + 64-bit scaled offset vector. +def int_aarch64_sve_gather_prfb_scaled : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfh_scaled : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfw_scaled : SVE_gather_prf_scalar_base_vector_offset_scaled; +def int_aarch64_sve_gather_prfd_scaled : SVE_gather_prf_scalar_base_vector_offset_scaled; + +// Vector + scalar. +def int_aarch64_sve_gather_prfb : SVE_gather_prf_vector_base_scalar_offset; +def int_aarch64_sve_gather_prfh : SVE_gather_prf_vector_base_scalar_offset; +def int_aarch64_sve_gather_prfw : SVE_gather_prf_vector_base_scalar_offset; +def int_aarch64_sve_gather_prfd : SVE_gather_prf_vector_base_scalar_offset; + // // Scalar to vector operations // diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -12911,6 +12911,86 @@ return SDValue(N, 0); } +/// Combine the gather prefetch (scalar + vector addressing mode) when +/// the offset vector is an unpacked 32-bit scalable vector. +static SDValue performSVEPrefetchScalarBaseVecOffsetCombine(SDNode *N, + SelectionDAG &DAG) { + const SDValue Offset = N->getOperand(3); + + // Not an unpacked vector, bail off. + if (Offset.getValueType().getSimpleVT().SimpleTy != MVT::nxv2i32) + return SDValue(); + + // Extend the unpacked offset vector to 64-bit lanes. + const SDLoc DL(N); + SDValue Ops[] = { + N->getOperand(0), // Chain + N->getOperand(1), // Intrinsic ID + N->getOperand(2), // Base + DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset), // Offset + N->getOperand(4), // Pg + N->getOperand(5) // prfop + }; + return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops); +} + +// Check if the value of \p Offset represent a valid immediate for the +// SVE gather prefetch instruction with vector base and immediate +// offset: +// +// PRF , , [.[S|D]{, #}] +// +// where = sizeof() * k, for k = 0, 1, ..., 31. +static bool +isValidOffsetImmediateForSVEGatherPrefetch(SDValue Offset, + unsigned ScalarSizeInBytes) { + ConstantSDNode *OffsetConst = dyn_cast(Offset.getNode()); + // Not a constant, bail out. + if (nullptr == OffsetConst) + return false; + + const unsigned Imm = OffsetConst->getZExtValue(); + + // The immediate is not a multiple of the scalar size. + if (Imm % ScalarSizeInBytes) + return false; + + // The immediate is out of range. + if (Imm / ScalarSizeInBytes > 31) + return false; + + return true; +}; + +/// Combines a node carrying the intrinsic `aarch64_sve_gather_prf` +/// into a node that uses +/// `aarch64_sve_gather_prf_scaled_uxtw`. This transformation is +/// needed when the scalar offset passed to +/// `aarch64_sve_gather_prf` is not a valid immediate for the sve +/// gather prefetch instruction with vector plus immediate addressing +/// mode. +static SDValue performSVEPrefetchVecBaseImmOffCombine( + SDNode *N, SelectionDAG &DAG, unsigned NewIID, unsigned ScalarSizeInBytes) { + // No need to combine the node if the immediate is valid... + if (isValidOffsetImmediateForSVEGatherPrefetch(N->getOperand(3), + ScalarSizeInBytes)) + return SDValue(); + + // ...otherwise, remap `aarch64_sve_gather_prf` to + // `aarch64_sve_gather_prf_scaled_uxtw` by swapping base with the + // offset (operand 2 and 3 of N, respectively). + SDLoc DL(N); + SDValue Ops[] = { + N->getOperand(0), // Chain + DAG.getConstant(NewIID, DL, MVT::i64), // Intrinsic ID + N->getOperand(3), // Base + N->getOperand(2), // Offset + N->getOperand(4), // Pg + N->getOperand(5) // prfop + }; + return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops); +} + SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; @@ -12975,6 +13055,31 @@ case ISD::INTRINSIC_VOID: case ISD::INTRINSIC_W_CHAIN: switch (cast(N->getOperand(1))->getZExtValue()) { + case Intrinsic::aarch64_sve_gather_prfb: + return performSVEPrefetchVecBaseImmOffCombine( + N, DAG, Intrinsic::aarch64_sve_gather_prfb_scaled_uxtw, + 1 /*=ScalarSizeInBytes*/); + case Intrinsic::aarch64_sve_gather_prfh: + return performSVEPrefetchVecBaseImmOffCombine( + N, DAG, Intrinsic::aarch64_sve_gather_prfh_scaled_uxtw, + 2 /*=ScalarSizeInBytes*/); + case Intrinsic::aarch64_sve_gather_prfw: + return performSVEPrefetchVecBaseImmOffCombine( + N, DAG, Intrinsic::aarch64_sve_gather_prfw_scaled_uxtw, + 4 /*=ScalarSizeInBytes*/); + case Intrinsic::aarch64_sve_gather_prfd: + return performSVEPrefetchVecBaseImmOffCombine( + N, DAG, Intrinsic::aarch64_sve_gather_prfd_scaled_uxtw, + 8 /*=ScalarSizeInBytes*/); + case Intrinsic::aarch64_sve_gather_prfb_scaled_uxtw: + case Intrinsic::aarch64_sve_gather_prfb_scaled_sxtw: + case Intrinsic::aarch64_sve_gather_prfh_scaled_uxtw: + case Intrinsic::aarch64_sve_gather_prfh_scaled_sxtw: + case Intrinsic::aarch64_sve_gather_prfw_scaled_uxtw: + case Intrinsic::aarch64_sve_gather_prfw_scaled_sxtw: + case Intrinsic::aarch64_sve_gather_prfd_scaled_uxtw: + case Intrinsic::aarch64_sve_gather_prfd_scaled_sxtw: + return performSVEPrefetchScalarBaseVecOffsetCombine(N, DAG); case Intrinsic::aarch64_neon_ld2: case Intrinsic::aarch64_neon_ld3: case Intrinsic::aarch64_neon_ld4: diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -880,37 +880,37 @@ // Gather prefetch using scaled 32-bit offsets, e.g. // prfh pldl1keep, p0, [x0, z0.s, uxtw #1] - defm PRFB_S : sve_mem_32b_prfm_sv_scaled<0b00, "prfb", ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only>; - defm PRFH_S : sve_mem_32b_prfm_sv_scaled<0b01, "prfh", ZPR32ExtSXTW16, ZPR32ExtUXTW16>; - defm PRFW_S : sve_mem_32b_prfm_sv_scaled<0b10, "prfw", ZPR32ExtSXTW32, ZPR32ExtUXTW32>; - defm PRFD_S : sve_mem_32b_prfm_sv_scaled<0b11, "prfd", ZPR32ExtSXTW64, ZPR32ExtUXTW64>; + defm PRFB_S : sve_mem_32b_prfm_sv_scaled<0b00, "prfb", ZPR32ExtSXTW8Only, ZPR32ExtUXTW8Only, int_aarch64_sve_gather_prfb_scaled_sxtw, int_aarch64_sve_gather_prfb_scaled_uxtw>; + defm PRFH_S : sve_mem_32b_prfm_sv_scaled<0b01, "prfh", ZPR32ExtSXTW16, ZPR32ExtUXTW16, int_aarch64_sve_gather_prfh_scaled_sxtw, int_aarch64_sve_gather_prfh_scaled_uxtw>; + defm PRFW_S : sve_mem_32b_prfm_sv_scaled<0b10, "prfw", ZPR32ExtSXTW32, ZPR32ExtUXTW32, int_aarch64_sve_gather_prfw_scaled_sxtw, int_aarch64_sve_gather_prfw_scaled_uxtw>; + defm PRFD_S : sve_mem_32b_prfm_sv_scaled<0b11, "prfd", ZPR32ExtSXTW64, ZPR32ExtUXTW64, int_aarch64_sve_gather_prfd_scaled_sxtw, int_aarch64_sve_gather_prfd_scaled_uxtw>; // Gather prefetch using unpacked, scaled 32-bit offsets, e.g. // prfh pldl1keep, p0, [x0, z0.d, uxtw #1] - defm PRFB_D : sve_mem_64b_prfm_sv_ext_scaled<0b00, "prfb", ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only>; - defm PRFH_D : sve_mem_64b_prfm_sv_ext_scaled<0b01, "prfh", ZPR64ExtSXTW16, ZPR64ExtUXTW16>; - defm PRFW_D : sve_mem_64b_prfm_sv_ext_scaled<0b10, "prfw", ZPR64ExtSXTW32, ZPR64ExtUXTW32>; - defm PRFD_D : sve_mem_64b_prfm_sv_ext_scaled<0b11, "prfd", ZPR64ExtSXTW64, ZPR64ExtUXTW64>; + defm PRFB_D : sve_mem_64b_prfm_sv_ext_scaled<0b00, "prfb", ZPR64ExtSXTW8Only, ZPR64ExtUXTW8Only, int_aarch64_sve_gather_prfb_scaled_sxtw, int_aarch64_sve_gather_prfb_scaled_uxtw>; + defm PRFH_D : sve_mem_64b_prfm_sv_ext_scaled<0b01, "prfh", ZPR64ExtSXTW16, ZPR64ExtUXTW16, int_aarch64_sve_gather_prfh_scaled_sxtw, int_aarch64_sve_gather_prfh_scaled_uxtw>; + defm PRFW_D : sve_mem_64b_prfm_sv_ext_scaled<0b10, "prfw", ZPR64ExtSXTW32, ZPR64ExtUXTW32, int_aarch64_sve_gather_prfw_scaled_sxtw, int_aarch64_sve_gather_prfw_scaled_uxtw>; + defm PRFD_D : sve_mem_64b_prfm_sv_ext_scaled<0b11, "prfd", ZPR64ExtSXTW64, ZPR64ExtUXTW64, int_aarch64_sve_gather_prfd_scaled_sxtw, int_aarch64_sve_gather_prfd_scaled_uxtw>; // Gather prefetch using scaled 64-bit offsets, e.g. // prfh pldl1keep, p0, [x0, z0.d, lsl #1] - defm PRFB_D_SCALED : sve_mem_64b_prfm_sv_lsl_scaled<0b00, "prfb", ZPR64ExtLSL8>; - defm PRFH_D_SCALED : sve_mem_64b_prfm_sv_lsl_scaled<0b01, "prfh", ZPR64ExtLSL16>; - defm PRFW_D_SCALED : sve_mem_64b_prfm_sv_lsl_scaled<0b10, "prfw", ZPR64ExtLSL32>; - defm PRFD_D_SCALED : sve_mem_64b_prfm_sv_lsl_scaled<0b11, "prfd", ZPR64ExtLSL64>; + defm PRFB_D_SCALED : sve_mem_64b_prfm_sv_lsl_scaled<0b00, "prfb", ZPR64ExtLSL8, int_aarch64_sve_gather_prfb_scaled>; + defm PRFH_D_SCALED : sve_mem_64b_prfm_sv_lsl_scaled<0b01, "prfh", ZPR64ExtLSL16, int_aarch64_sve_gather_prfh_scaled>; + defm PRFW_D_SCALED : sve_mem_64b_prfm_sv_lsl_scaled<0b10, "prfw", ZPR64ExtLSL32, int_aarch64_sve_gather_prfw_scaled>; + defm PRFD_D_SCALED : sve_mem_64b_prfm_sv_lsl_scaled<0b11, "prfd", ZPR64ExtLSL64, int_aarch64_sve_gather_prfd_scaled>; // Gather prefetch using 32/64-bit pointers with offset, e.g. // prfh pldl1keep, p0, [z0.s, #16] // prfh pldl1keep, p0, [z0.d, #16] - defm PRFB_S_PZI : sve_mem_32b_prfm_vi<0b00, "prfb", imm0_31>; - defm PRFH_S_PZI : sve_mem_32b_prfm_vi<0b01, "prfh", uimm5s2>; - defm PRFW_S_PZI : sve_mem_32b_prfm_vi<0b10, "prfw", uimm5s4>; - defm PRFD_S_PZI : sve_mem_32b_prfm_vi<0b11, "prfd", uimm5s8>; - - defm PRFB_D_PZI : sve_mem_64b_prfm_vi<0b00, "prfb", imm0_31>; - defm PRFH_D_PZI : sve_mem_64b_prfm_vi<0b01, "prfh", uimm5s2>; - defm PRFW_D_PZI : sve_mem_64b_prfm_vi<0b10, "prfw", uimm5s4>; - defm PRFD_D_PZI : sve_mem_64b_prfm_vi<0b11, "prfd", uimm5s8>; + defm PRFB_S_PZI : sve_mem_32b_prfm_vi<0b00, "prfb", imm0_31, int_aarch64_sve_gather_prfb>; + defm PRFH_S_PZI : sve_mem_32b_prfm_vi<0b01, "prfh", uimm5s2, int_aarch64_sve_gather_prfh>; + defm PRFW_S_PZI : sve_mem_32b_prfm_vi<0b10, "prfw", uimm5s4, int_aarch64_sve_gather_prfw>; + defm PRFD_S_PZI : sve_mem_32b_prfm_vi<0b11, "prfd", uimm5s8, int_aarch64_sve_gather_prfd>; + + defm PRFB_D_PZI : sve_mem_64b_prfm_vi<0b00, "prfb", imm0_31, int_aarch64_sve_gather_prfb>; + defm PRFH_D_PZI : sve_mem_64b_prfm_vi<0b01, "prfh", uimm5s2, int_aarch64_sve_gather_prfh>; + defm PRFW_D_PZI : sve_mem_64b_prfm_vi<0b10, "prfw", uimm5s4, int_aarch64_sve_gather_prfw>; + defm PRFD_D_PZI : sve_mem_64b_prfm_vi<0b11, "prfd", uimm5s8, int_aarch64_sve_gather_prfd>; defm ADR_SXTW_ZZZ_D : sve_int_bin_cons_misc_0_a_sxtw<0b00, "adr">; defm ADR_UXTW_ZZZ_D : sve_int_bin_cons_misc_0_a_uxtw<0b01, "adr">; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -6445,9 +6445,17 @@ multiclass sve_mem_32b_prfm_sv_scaled msz, string asm, RegisterOperand sxtw_opnd, - RegisterOperand uxtw_opnd> { + RegisterOperand uxtw_opnd, + PatFrag op_sxtw, + PatFrag op_uxtw> { def _UXTW_SCALED : sve_mem_32b_prfm_sv; def _SXTW_SCALED : sve_mem_32b_prfm_sv; + + def : Pat<(op_uxtw (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (nxv4i1 PPR3bAny:$Pg), (i32 sve_prfop:$prfop)), + (!cast(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; + + def : Pat<(op_sxtw (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (nxv4i1 PPR3bAny:$Pg), (i32 sve_prfop:$prfop)), + (!cast(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; } class sve_mem_32b_prfm_vi msz, string asm, Operand imm_ty> @@ -6470,9 +6478,12 @@ let Inst{3-0} = prfop; } -multiclass sve_mem_32b_prfm_vi msz, string asm, Operand imm_ty> { +multiclass sve_mem_32b_prfm_vi msz, string asm, Operand imm_ty, SDPatternOperator op> { def NAME : sve_mem_32b_prfm_vi; + def : Pat<(op (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (nxv4i1 PPR_3b:$Pg), (i32 sve_prfop:$prfop)), + (!cast(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; + def : InstAlias(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; } @@ -6788,14 +6799,27 @@ multiclass sve_mem_64b_prfm_sv_ext_scaled msz, string asm, RegisterOperand sxtw_opnd, - RegisterOperand uxtw_opnd> { + RegisterOperand uxtw_opnd, + PatFrag op_sxtw, + PatFrag op_uxtw> { def _UXTW_SCALED : sve_mem_64b_prfm_sv; def _SXTW_SCALED : sve_mem_64b_prfm_sv; + + def : Pat<(op_uxtw (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (nxv2i1 PPR3bAny:$Pg), (i32 sve_prfop:$prfop)), + (!cast(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; + + def : Pat<(op_sxtw (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (nxv2i1 PPR3bAny:$Pg), (i32 sve_prfop:$prfop)), + (!cast(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; + } multiclass sve_mem_64b_prfm_sv_lsl_scaled msz, string asm, - RegisterOperand zprext> { + RegisterOperand zprext, PatFrag frag> { def NAME : sve_mem_64b_prfm_sv; + + def : Pat<(frag (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (nxv2i1 PPR3bAny:$Pg), (i32 sve_prfop:$prfop)), + (!cast(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; + } @@ -6821,9 +6845,12 @@ let hasSideEffects = 1; } -multiclass sve_mem_64b_prfm_vi msz, string asm, Operand imm_ty> { +multiclass sve_mem_64b_prfm_vi msz, string asm, Operand imm_ty, SDPatternOperator op> { def NAME : sve_mem_64b_prfm_vi; + def : Pat<(op (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (nxv2i1 PPR_3b:$Pg), (i32 sve_prfop:$prfop)), + (!cast(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; + def : InstAlias(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; } diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-scaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-scaled-offset.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-scaled-offset.ll @@ -0,0 +1,200 @@ +; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve --asm-verbose=false < %s | FileCheck %s + +; PRFB , , [, .S, ] -> 32-bit scaled offset +define void @llvm_aarch64_sve_gather_prfb_scaled_uxtw_nx4vi32(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_scaled_uxtw_nx4vi32: +; CHECK: prfb pldl1strm, p0, [x0, z0.s, uxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.scaled.uxtw.nx4vi32(i8* %base, %offset, %Pg, i32 1) + ret void + } + +define void @llvm_aarch64_sve_gather_prfb_scaled_sxtw_nx4vi32(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_scaled_sxtw_nx4vi32: +; CHECK: prfb pldl1strm, p0, [x0, z0.s, sxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.scaled.sxtw.nx4vi32(i8* %base, %offset, %Pg, i32 1) + ret void + } + +; PRFB , , [, .D, ] -> 32-bit unpacked scaled offset + +define void @llvm_aarch64_sve_gather_prfb_scaled_uxtw_nx2vi64(i8* %base, %offset, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_scaled_uxtw_nx2vi64: +; CHECK-NEXT: prfb pldl1strm, p0, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.scaled.uxtw.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +define void @llvm_aarch64_sve_gather_prfb_scaled_sxtw_nx2vi64(i8* %base, %offset, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_scaled_sxtw_nx2vi64: +; CHECK-NEXT: prfb pldl1strm, p0, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.scaled.sxtw.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } +; PRFB , , [, .D] -> 64-bit scaled offset +define void @llvm_aarch64_sve_gather_prfb_scaled_nx2vi64(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_scaled_nx2vi64: +; CHECK: prfb pldl1strm, p0, [x0, z0.d] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.scaled.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; PRFH , , [, .S, ] -> 32-bit scaled offset +define void @llvm_aarch64_sve_gather_prfh_scaled_uxtw_nx4vi32(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_scaled_uxtw_nx4vi32: +; CHECK: prfh pldl1strm, p0, [x0, z0.s, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.scaled.uxtw.nx4vi32(i8* %base, %offset, %Pg, i32 1) + ret void + } + +define void @llvm_aarch64_sve_gather_prfh_scaled_sxtw_nx4vi32(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_scaled_sxtw_nx4vi32: +; CHECK: prfh pldl1strm, p0, [x0, z0.s, sxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.scaled.sxtw.nx4vi32(i8* %base, %offset, %Pg, i32 1) + ret void + } + +; PRFH , , [, .D, #1] -> 32-bit unpacked scaled offset +define void @llvm_aarch64_sve_gather_prfh_scaled_uxtw_nx2vi64(i8* %base, %offset, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_scaled_uxtw_nx2vi64: +; CHECK-NEXT: prfh pldl1strm, p0, [x0, z0.d, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.scaled.uxtw.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +define void @llvm_aarch64_sve_gather_prfh_scaled_sxtw_nx2vi64(i8* %base, %offset, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_scaled_sxtw_nx2vi64: +; CHECK-NEXT: prfh pldl1strm, p0, [x0, z0.d, sxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.scaled.sxtw.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +; PRFH , , [, .D] -> 64-bit scaled offset +define void @llvm_aarch64_sve_gather_prfh_scaled_nx2vi64(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_scaled_nx2vi64: +; CHECK: prfh pldl1strm, p0, [x0, z0.d, lsl #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.scaled.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; PRFW , , [, .S, ] -> 32-bit scaled offset +define void @llvm_aarch64_sve_gather_prfw_scaled_uxtw_nx4vi32(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_scaled_uxtw_nx4vi32: +; CHECK: prfw pldl1strm, p0, [x0, z0.s, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.scaled.uxtw.nx4vi32(i8* %base, %offset, %Pg, i32 1) + ret void + } + +define void @llvm_aarch64_sve_gather_prfw_scaled_sxtw_nx4vi32(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_scaled_sxtw_nx4vi32: +; CHECK: prfw pldl1strm, p0, [x0, z0.s, sxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.scaled.sxtw.nx4vi32(i8* %base, %offset, %Pg, i32 1) + ret void + } + +; PRFW , , [, .D, #2] -> 32-bit unpacked scaled offset +define void @llvm_aarch64_sve_gather_prfw_scaled_uxtw_nx2vi64(i8* %base, %offset, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_scaled_uxtw_nx2vi64: +; CHECK-NEXT: prfw pldl1strm, p0, [x0, z0.d, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.scaled.uxtw.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +define void @llvm_aarch64_sve_gather_prfw_scaled_sxtw_nx2vi64(i8* %base, %offset, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_scaled_sxtw_nx2vi64: +; CHECK-NEXT: prfw pldl1strm, p0, [x0, z0.d, sxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.scaled.sxtw.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +; PRFW , , [, .D] -> 64-bit scaled offset +define void @llvm_aarch64_sve_gather_prfw_scaled_nx2vi64(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_scaled_nx2vi64: +; CHECK: prfw pldl1strm, p0, [x0, z0.d, lsl #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.scaled.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; PRFD , , [, .S, ] -> 32-bit scaled offset +define void @llvm_aarch64_sve_gather_prfd_scaled_uxtw_nx4vi32(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_scaled_uxtw_nx4vi32: +; CHECK: prfd pldl1strm, p0, [x0, z0.s, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.scaled.uxtw.nx4vi32(i8* %base, %offset, %Pg, i32 1) + ret void + } + +define void @llvm_aarch64_sve_gather_prfd_scaled_sxtw_nx4vi32(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_scaled_sxtw_nx4vi32: +; CHECK: prfd pldl1strm, p0, [x0, z0.s, sxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.scaled.sxtw.nx4vi32(i8* %base, %offset, %Pg, i32 1) + ret void + } + +; PRFD , , [, .D, #3] -> 32-bit unpacked scaled offset +define void @llvm_aarch64_sve_gather_prfd_scaled_uxtw_nx2vi64(i8* %base, %offset, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_scaled_uxtw_nx2vi64: +; CHECK-NEXT: prfd pldl1strm, p0, [x0, z0.d, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.scaled.uxtw.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +define void @llvm_aarch64_sve_gather_prfd_scaled_sxtw_nx2vi64(i8* %base, %offset, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_scaled_sxtw_nx2vi64: +; CHECK-NEXT: prfd pldl1strm, p0, [x0, z0.d, sxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.scaled.sxtw.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +; PRFD , , [, .D] -> 64-bit scaled offset +define void @llvm_aarch64_sve_gather_prfd_scaled_nx2vi64(i8* %base, %offset, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_scaled_nx2vi64: +; CHECK: prfd pldl1strm, p0, [x0, z0.d, lsl #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.scaled.nx2vi64(i8* %base, %offset, %Pg, i32 1) + ret void + } + +declare void @llvm.aarch64.sve.gather.prfb.scaled.uxtw.nx4vi32(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfb.scaled.sxtw.nx4vi32(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfb.scaled.uxtw.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfb.scaled.sxtw.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfb.scaled.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.scaled.uxtw.nx4vi32(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.scaled.sxtw.nx4vi32(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.scaled.uxtw.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.scaled.sxtw.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.scaled.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.scaled.uxtw.nx4vi32(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.scaled.sxtw.nx4vi32(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.scaled.uxtw.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.scaled.sxtw.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.scaled.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.scaled.uxtw.nx4vi32(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.scaled.sxtw.nx4vi32(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.scaled.uxtw.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.scaled.sxtw.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.scaled.nx2vi64(i8* %base, %offset, %Pg, i32 %prfop) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-vect-base-imm-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-vect-base-imm-offset.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-vect-base-imm-offset.ll @@ -0,0 +1,82 @@ +; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve --asm-verbose=false < %s | FileCheck %s + +; PRFB , , [.S{, #}] -> 32-bit element +define void @llvm_aarch64_sve_gather_prfb_nx4vi32( %bases, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_nx4vi32: +; CHECK: prfb pldl1strm, p0, [z0.s, #7] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.nx4vi32( %bases, i64 7, %Pg, i32 1) + ret void +} + +; PRFB , , [.D{, #}] -> 64-bit element +define void @llvm_aarch64_sve_gather_prfb_nx2vi64( %bases, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_nx2vi64: +; CHECK: prfb pldl1strm, p0, [z0.d, #7] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.nx2vi64( %bases, i64 7, %Pg, i32 1) + ret void +} + +; PRFH , , [.S{, #}] -> 32-bit element +define void @llvm_aarch64_sve_gather_prfh_nx4vi32( %bases, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx4vi32: +; CHECK: prfh pldl1strm, p0, [z0.s, #6] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx4vi32( %bases, i64 6, %Pg, i32 1) + ret void +} + +; PRFH , , [.D{, #}] -> 64-bit element +define void @llvm_aarch64_sve_gather_prfh_nx2vi64( %bases, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx2vi64: +; CHECK: prfh pldl1strm, p0, [z0.d, #6] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx2vi64( %bases, i64 6, %Pg, i32 1) + ret void +} + +; PRFW , , [.S{, #}] -> 32-bit element +define void @llvm_aarch64_sve_gather_prfw_nx4vi32( %bases, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx4vi32: +; CHECK: prfw pldl1strm, p0, [z0.s, #12] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx4vi32( %bases, i64 12, %Pg, i32 1) + ret void +} + +; PRFW , , [.D{, #}] -> 64-bit element +define void @llvm_aarch64_sve_gather_prfw_nx2vi64( %bases, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx2vi64: +; CHECK: prfw pldl1strm, p0, [z0.d, #12] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx2vi64( %bases, i64 12, %Pg, i32 1) + ret void +} + +; PRFD , , [.S{, #}] -> 32-bit element +define void @llvm_aarch64_sve_gather_prfd_nx4vi32( %bases, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx4vi32: +; CHECK: prfd pldl1strm, p0, [z0.s, #16] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx4vi32( %bases, i64 16, %Pg, i32 1) + ret void +} + +; PRFD , , [.D{, #}] -> 64-bit element +define void @llvm_aarch64_sve_gather_prfd_nx2vi64( %bases, %Pg) { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx2vi64: +; CHECK: prfd pldl1strm, p0, [z0.d, #16] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx2vi64( %bases, i64 16, %Pg, i32 1) + ret void +} + +declare void @llvm.aarch64.sve.gather.prfb.nx4vi32( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfb.nx2vi64( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.nx4vi32( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.nx2vi64( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.nx4vi32( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.nx2vi64( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.nx4vi32( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.nx2vi64( %bases, i64 %imm, %Pg, i32 %prfop) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-vect-base-invalid-imm-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-vect-base-invalid-imm-offset.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-prefetches-vect-base-invalid-imm-offset.ll @@ -0,0 +1,286 @@ +; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve --asm-verbose=false < %s | FileCheck %s + +; PRFB , , [.S{, #}] -> 32-bit element, imm = 0, 1, ..., 31 +define void @llvm_aarch64_sve_gather_prfb_nx4vi32_runtime_offset( %bases, i64 %imm, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_nx4vi32_runtime_offset: +; CHECK-NEXT: prfb pldl1strm, p0, [x0, z0.s, uxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.nx4vi32( %bases, i64 %imm, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfb_nx4vi32_invalid_immediate_offset_upper_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_nx4vi32_invalid_immediate_offset_upper_bound: +; CHECK-NEXT: mov w[[N:[0-9]+]], #32 +; CHECK-NEXT: prfb pldl1strm, p0, [x[[N]], z0.s, uxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.nx4vi32( %bases, i64 32, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfb_nx4vi32_invalid_immediate_offset_lower_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_nx4vi32_invalid_immediate_offset_lower_bound: +; CHECK-NEXT: mov x[[N:[0-9]+]], #-1 +; CHECK-NEXT: prfb pldl1strm, p0, [x[[N:[0-9]+]], z0.s, uxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.nx4vi32( %bases, i64 -1, %Pg, i32 1) + ret void +} + +; PRFB , , [.D{, #}] -> 64-bit element, imm = 0, 1, ..., 31 +define void @llvm_aarch64_sve_gather_prfb_nx2vi64_runtime_offset( %bases, i64 %imm, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_nx2vi64_runtime_offset: +; CHECK-NEXT: prfb pldl1strm, p0, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.nx2vi64( %bases, i64 %imm, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfb_nx2vi64_invalid_immediate_offset_upper_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_nx2vi64_invalid_immediate_offset_upper_bound: +; CHECK-NEXT: mov w[[N:[0-9]+]], #32 +; CHECK-NEXT: prfb pldl1strm, p0, [x[[N]], z0.d, uxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.nx2vi64( %bases, i64 32, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfb_nx2vi64_invalid_immediate_offset_lower_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfb_nx2vi64_invalid_immediate_offset_lower_bound: +; CHECK-NEXT: mov x[[N:[0-9]+]], #-1 +; CHECK-NEXT: prfb pldl1strm, p0, [x[[N:[0-9]+]], z0.d, uxtw] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfb.nx2vi64( %bases, i64 -1, %Pg, i32 1) + ret void +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; PRFH , , [.S{, #}] -> 32-bit element, imm = 0, 2, ..., 62 +define void @llvm_aarch64_sve_gather_prfh_nx4vi32_runtime_offset( %bases, i64 %imm, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx4vi32_runtime_offset: +; CHECK-NEXT: prfh pldl1strm, p0, [x0, z0.s, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx4vi32( %bases, i64 %imm, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfh_nx4vi32_invalid_immediate_offset_upper_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx4vi32_invalid_immediate_offset_upper_bound: +; CHECK-NEXT: mov w[[N:[0-9]+]], #63 +; CHECK-NEXT: prfh pldl1strm, p0, [x[[N]], z0.s, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx4vi32( %bases, i64 63, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfh_nx4vi32_invalid_immediate_offset_lower_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx4vi32_invalid_immediate_offset_lower_bound: +; CHECK-NEXT: mov x[[N:[0-9]+]], #-1 +; CHECK-NEXT: prfh pldl1strm, p0, [x[[N:[0-9]+]], z0.s, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx4vi32( %bases, i64 -1, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfh_nx4vi32_invalid_immediate_offset_inbound_not_multiple_of_2( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx4vi32_invalid_immediate_offset_inbound_not_multiple_of_2: +; CHECK-NEXT: mov w[[N:[0-9]+]], #33 +; CHECK-NEXT: prfh pldl1strm, p0, [x[[N:[0-9]+]], z0.s, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx4vi32( %bases, i64 33, %Pg, i32 1) + ret void +} + +; PRFH , , [.D{, #}] -> 64-bit element, imm = 0, 2, ..., 62 +define void @llvm_aarch64_sve_gather_prfh_nx2vi64_runtime_offset( %bases, i64 %imm, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx2vi64_runtime_offset: +; CHECK-NEXT: prfh pldl1strm, p0, [x0, z0.d, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx2vi64( %bases, i64 %imm, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfh_nx2vi64_invalid_immediate_offset_upper_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx2vi64_invalid_immediate_offset_upper_bound: +; CHECK-NEXT: mov w[[N:[0-9]+]], #63 +; CHECK-NEXT: prfh pldl1strm, p0, [x[[N]], z0.d, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx2vi64( %bases, i64 63, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfh_nx2vi64_invalid_immediate_offset_lower_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx2vi64_invalid_immediate_offset_lower_bound: +; CHECK-NEXT: mov x[[N:[0-9]+]], #-1 +; CHECK-NEXT: prfh pldl1strm, p0, [x[[N:[0-9]+]], z0.d, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx2vi64( %bases, i64 -1, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfh_nx2vi64_invalid_immediate_offset_inbound_not_multiple_of_2( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfh_nx2vi64_invalid_immediate_offset_inbound_not_multiple_of_2: +; CHECK-NEXT: mov w[[N:[0-9]+]], #33 +; CHECK-NEXT: prfh pldl1strm, p0, [x[[N:[0-9]+]], z0.d, uxtw #1] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfh.nx2vi64( %bases, i64 33, %Pg, i32 1) + ret void +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; PRFW , , [.S{, #}] -> 32-bit element, imm = 0, 4, ..., 124 +define void @llvm_aarch64_sve_gather_prfw_nx4vi32_runtime_offset( %bases, i64 %imm, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx4vi32_runtime_offset: +; CHECK-NEXT: prfw pldl1strm, p0, [x0, z0.s, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx4vi32( %bases, i64 %imm, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfw_nx4vi32_invalid_immediate_offset_upper_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx4vi32_invalid_immediate_offset_upper_bound: +; CHECK-NEXT: mov w[[N:[0-9]+]], #125 +; CHECK-NEXT: prfw pldl1strm, p0, [x[[N]], z0.s, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx4vi32( %bases, i64 125, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfw_nx4vi32_invalid_immediate_offset_lower_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx4vi32_invalid_immediate_offset_lower_bound: +; CHECK-NEXT: mov x[[N:[0-9]+]], #-1 +; CHECK-NEXT: prfw pldl1strm, p0, [x[[N:[0-9]+]], z0.s, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx4vi32( %bases, i64 -1, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfw_nx4vi32_invalid_immediate_offset_inbound_not_multiple_of_4( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx4vi32_invalid_immediate_offset_inbound_not_multiple_of_4: +; CHECK-NEXT: mov w[[N:[0-9]+]], #33 +; CHECK-NEXT: prfw pldl1strm, p0, [x[[N:[0-9]+]], z0.s, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx4vi32( %bases, i64 33, %Pg, i32 1) + ret void +} + +; PRFW , , [.D{, #}] -> 64-bit element, imm = 0, 4, ..., 124 +define void @llvm_aarch64_sve_gather_prfw_nx2vi64_runtime_offset( %bases, i64 %imm, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx2vi64_runtime_offset: +; CHECK-NEXT: prfw pldl1strm, p0, [x0, z0.d, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx2vi64( %bases, i64 %imm, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfw_nx2vi64_invalid_immediate_offset_upper_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx2vi64_invalid_immediate_offset_upper_bound: +; CHECK-NEXT: mov w[[N:[0-9]+]], #125 +; CHECK-NEXT: prfw pldl1strm, p0, [x[[N]], z0.d, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx2vi64( %bases, i64 125, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfw_nx2vi64_invalid_immediate_offset_lower_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx2vi64_invalid_immediate_offset_lower_bound: +; CHECK-NEXT: mov x[[N:[0-9]+]], #-1 +; CHECK-NEXT: prfw pldl1strm, p0, [x[[N:[0-9]+]], z0.d, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx2vi64( %bases, i64 -1, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfw_nx2vi64_invalid_immediate_offset_inbound_not_multiple_of_4( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfw_nx2vi64_invalid_immediate_offset_inbound_not_multiple_of_4: +; CHECK-NEXT: mov w[[N:[0-9]+]], #33 +; CHECK-NEXT: prfw pldl1strm, p0, [x[[N:[0-9]+]], z0.d, uxtw #2] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfw.nx2vi64( %bases, i64 33, %Pg, i32 1) + ret void +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; PRFD , , [.S{, #}] -> 32-bit element, imm = 0, 8, ..., 248 +define void @llvm_aarch64_sve_gather_prfd_nx4vi32_runtime_offset( %bases, i64 %imm, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx4vi32_runtime_offset: +; CHECK-NEXT: prfd pldl1strm, p0, [x0, z0.s, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx4vi32( %bases, i64 %imm, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfd_nx4vi32_invalid_immediate_offset_upper_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx4vi32_invalid_immediate_offset_upper_bound: +; CHECK-NEXT: mov w[[N:[0-9]+]], #125 +; CHECK-NEXT: prfd pldl1strm, p0, [x[[N]], z0.s, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx4vi32( %bases, i64 125, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfd_nx4vi32_invalid_immediate_offset_lower_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx4vi32_invalid_immediate_offset_lower_bound: +; CHECK-NEXT: mov x[[N:[0-9]+]], #-1 +; CHECK-NEXT: prfd pldl1strm, p0, [x[[N:[0-9]+]], z0.s, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx4vi32( %bases, i64 -1, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfd_nx4vi32_invalid_immediate_offset_inbound_not_multiple_of_8( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx4vi32_invalid_immediate_offset_inbound_not_multiple_of_8: +; CHECK-NEXT: mov w[[N:[0-9]+]], #33 +; CHECK-NEXT: prfd pldl1strm, p0, [x[[N:[0-9]+]], z0.s, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx4vi32( %bases, i64 33, %Pg, i32 1) + ret void +} + +; PRFD , , [.D{, #}] -> 64-bit element, imm = 0, 4, ..., 248 +define void @llvm_aarch64_sve_gather_prfd_nx2vi64_runtime_offset( %bases, i64 %imm, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx2vi64_runtime_offset: +; CHECK-NEXT: prfd pldl1strm, p0, [x0, z0.d, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx2vi64( %bases, i64 %imm, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfd_nx2vi64_invalid_immediate_offset_upper_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx2vi64_invalid_immediate_offset_upper_bound: +; CHECK-NEXT: mov w[[N:[0-9]+]], #125 +; CHECK-NEXT: prfd pldl1strm, p0, [x[[N]], z0.d, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx2vi64( %bases, i64 125, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfd_nx2vi64_invalid_immediate_offset_lower_bound( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx2vi64_invalid_immediate_offset_lower_bound: +; CHECK-NEXT: mov x[[N:[0-9]+]], #-1 +; CHECK-NEXT: prfd pldl1strm, p0, [x[[N:[0-9]+]], z0.d, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx2vi64( %bases, i64 -1, %Pg, i32 1) + ret void +} + +define void @llvm_aarch64_sve_gather_prfd_nx2vi64_invalid_immediate_offset_inbound_not_multiple_of_8( %bases, %Pg) nounwind { +; CHECK-LABEL: llvm_aarch64_sve_gather_prfd_nx2vi64_invalid_immediate_offset_inbound_not_multiple_of_8: +; CHECK-NEXT: mov w[[N:[0-9]+]], #33 +; CHECK-NEXT: prfd pldl1strm, p0, [x[[N:[0-9]+]], z0.d, uxtw #3] +; CHECK-NEXT: ret + call void @llvm.aarch64.sve.gather.prfd.nx2vi64( %bases, i64 33, %Pg, i32 1) + ret void +} + +declare void @llvm.aarch64.sve.gather.prfb.nx4vi32( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfb.nx2vi64( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.nx4vi32( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfh.nx2vi64( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.nx4vi32( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfw.nx2vi64( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.nx4vi32( %bases, i64 %imm, %Pg, i32 %prfop) +declare void @llvm.aarch64.sve.gather.prfd.nx2vi64( %bases, i64 %imm, %Pg, i32 %prfop)