[AArch64][SVE] Add SVE intrinsic for LD1RQ

[kmclaughlin]

Summary

Adds the following intrinsic for contiguous load & replicate:

• @llvm.aarch64.sve.ld1rq

The LD1RQ intrinsic only needs the SImmS16XForm added by this
patch. The others (SImmS2XForm, SImmS3XForm & SImmS4XForm)
were added for consistency.

[andwar]

Cheers for working on this @kmclaughlin !

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1421

[nit] In AArch64ISelLowering.h you added LD1RQ as the last load instruction, here it's the first load instruction.

11687

[Nit] I think that this method could be simplified and made more explicit:

static SDValue performLD1RQCombine(SDNode *N, SelectionDAG &DAG) {
  SDLoc DL(N);
  EVT VT = N->getValueType(0);

  EVT LoadVT = VT;
  if (VT.isFloatingPoint())
    LoadVT = VT.changeTypeToInteger();

  SDValue Ops[] = {N->getOperand(0), N->getOperand(2), N->getOperand(3)};
  SDValue Load = DAG.getNode(AArch64ISD::LD1RQ, DL, {LoadVT, MVT::Other}, Ops);

  if (VT.isFloatingPoint()) {
    SDValue LoadChain = SDValue(Load.getNode(), 1);
    Load = DAG.getMergeValues(
        {DAG.getNode(ISD::BITCAST, DL, VT, Load), LoadChain}, DL);
  }

  return Load;
}

This way:

• there's only 1 return statement
• you are being explicit about preserving the chain when generating the bit cast
• Load replaces a bit non-descriptive L

This is a matter of style, so please use what feels best.

llvm/lib/Target/AArch64/AArch64InstrFormats.td

495

IIUC, only this definition is need for LD1RQ. Could you add a note in the commit message to highlight that SImmS2XForm, SImmS3XForm and SImmS4XForm are added for consistency? Alternatively, only keep that one that's needed.

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

16

This comment should read Non-faulting & first-faulting loads - node definitions. That's my fault,, sorry!

23

The (undocumented) style here is:

// <Load type 1> - node definitions
//
<Node definitions for type 1>

// <Load type 2> - node definitions
//
<Node definitions for type 2>

Could you move LD1RQ to a dedicated block and add a comment? Ta!

[andwar]

Btw, could you also add some negative tests? (e.g. out-of-range immediate)

[kmclaughlin]

Simplified performLD1RQCombine method & added negative tests where the immediate is out of range.

[kmclaughlin]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

11687

Done, these changes do make it neater :)

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

11699

It seems like the LoadChain is lost if VT is of integer type.

[kmclaughlin]

Ensure LoadChain is always preserved in performLD1RQCombine

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

11700

I'd expect this to then use Load.getValue(0) ?

[kmclaughlin]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

11700

I think this will have the same effect, as Load just returns a single value

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

11700

SDValue LoadChain = SDValue(Load.getNode(), 1); suggests that Load has two return values, the result of the load, and the Chain.

[kmclaughlin]

• Use Load.getValue(0) when creating a bitcast in performLD1RQCombine

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

11700

I think you're right - I've changed this to use the result of the load as suggested

[sdesmalen]

LGTM!

llvm/include/llvm/IR/IntrinsicsAArch64.td

 //
 
 def int_aarch64_sve_ld1   : AdvSIMD_1Vec_PredLoad_Intrinsic;
 
 def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
 def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
 def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
 
+def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
+
 //
 // Stores
 //
 
 def int_aarch64_sve_st1  : AdvSIMD_1Vec_PredStore_Intrinsic;
 def int_aarch64_sve_st2  : AdvSIMD_2Vec_PredStore_Intrinsic;
 def int_aarch64_sve_st3  : AdvSIMD_3Vec_PredStore_Intrinsic;
 def int_aarch64_sve_st4  : AdvSIMD_4Vec_PredStore_Intrinsic;

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   REINTERPRET_CAST,
 
   LD1,
   LD1S,
   LDNF1,
   LDNF1S,
   LDFF1,
   LDFF1S,
+  LD1RQ,
 
   // Unsigned gather loads.
   GLD1,
   GLD1_SCALED,
   GLD1_UXTW,
   GLD1_SXTW,
   GLD1_UXTW_SCALED,
   GLD1_SXTW_SCALED,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case AArch64ISD::STZ2G:             return "AArch64ISD::STZ2G";
   case AArch64ISD::SUNPKHI:           return "AArch64ISD::SUNPKHI";
   case AArch64ISD::SUNPKLO:           return "AArch64ISD::SUNPKLO";
   case AArch64ISD::UUNPKHI:           return "AArch64ISD::UUNPKHI";
   case AArch64ISD::UUNPKLO:           return "AArch64ISD::UUNPKLO";
   case AArch64ISD::INSR:              return "AArch64ISD::INSR";
   case AArch64ISD::PTEST:             return "AArch64ISD::PTEST";
   case AArch64ISD::PTRUE:             return "AArch64ISD::PTRUE";
   case AArch64ISD::LD1:               return "AArch64ISD::LD1";
   case AArch64ISD::LD1S:              return "AArch64ISD::LD1S";
   case AArch64ISD::LDNF1:             return "AArch64ISD::LDNF1";
   case AArch64ISD::LDNF1S:            return "AArch64ISD::LDNF1S";
   case AArch64ISD::LDFF1:             return "AArch64ISD::LDFF1";
   case AArch64ISD::LDFF1S:            return "AArch64ISD::LDFF1S";
+  case AArch64ISD::LD1RQ:             return "AArch64ISD::LD1RQ";
   case AArch64ISD::GLD1:              return "AArch64ISD::GLD1";
   case AArch64ISD::GLD1_SCALED:       return "AArch64ISD::GLD1_SCALED";
   case AArch64ISD::GLD1_SXTW:         return "AArch64ISD::GLD1_SXTW";
   case AArch64ISD::GLD1_UXTW:         return "AArch64ISD::GLD1_UXTW";
   case AArch64ISD::GLD1_SXTW_SCALED:  return "AArch64ISD::GLD1_SXTW_SCALED";
   case AArch64ISD::GLD1_UXTW_SCALED:  return "AArch64ISD::GLD1_UXTW_SCALED";
   case AArch64ISD::GLD1_IMM:          return "AArch64ISD::GLD1_IMM";
   case AArch64ISD::GLD1S:             return "AArch64ISD::GLD1S";
static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
    if (VT.isFloatingPoint()) {
      SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) };
      return DAG.getMergeValues(Ops, DL);
    }
 
   return L;
 }
 
+static SDValue performLD1RQCombine(SDNode *N, SelectionDAG &DAG) {
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+
+  EVT LoadVT = VT;
+  if (VT.isFloatingPoint())
+    LoadVT = VT.changeTypeToInteger();
+
+  SDValue Ops[] = {N->getOperand(0), N->getOperand(2), N->getOperand(3)};
+  SDValue Load = DAG.getNode(AArch64ISD::LD1RQ, DL, {LoadVT, MVT::Other}, Ops);
+  SDValue LoadChain = SDValue(Load.getNode(), 1);
+
+  if (VT.isFloatingPoint())
+    Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0));
+
+  return DAG.getMergeValues({ Load, LoadChain }, DL);
+}
+
 static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
   SDLoc DL(N);
   SDValue Data = N->getOperand(2);
   EVT DataVT = Data.getValueType();
   EVT HwSrcVt = getSVEContainerType(DataVT);
   SDValue InputVT = DAG.getValueType(DataVT);
 
   if (DataVT.isFloatingPoint())
static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
 
   auto *MINode = cast<MemIntrinsicSDNode>(N);
   return DAG.getMaskedStore(MINode->getChain(), DL, Data, MINode->getOperand(4),
                             DAG.getUNDEF(PtrTy), MINode->getOperand(3),
                             MINode->getMemoryVT(), MINode->getMemOperand(),
                             ISD::UNINDEXED, false, false);
 }
 
 /// Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.  The
 /// load store optimizer pass will merge them to store pair stores.  This should
 /// be better than a movi to create the vector zero followed by a vector store
 /// if the zero constant is not re-used, since one instructions and one register
 /// live range will be removed.
 ///
 /// For example, the final generated code should be:
 ///
 ///   stp xzr, xzr, [x0]
 ///
 /// instead of:
 ///
 ///   movi v0.2d, #0
 ///   str q0, [x0]
 ///
 static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
   SDValue StVal = St.getValue();
   EVT VT = StVal.getValueType();
 
   // It is beneficial to scalarize a zero splat store for 2 or 3 i64 elements or
   // 2, 3 or 4 i32 elements.
   int NumVecElts = VT.getVectorNumElements();
case ISD::INTRINSIC_W_CHAIN:
     case Intrinsic::aarch64_neon_st1x3:
     case Intrinsic::aarch64_neon_st1x4:
     case Intrinsic::aarch64_neon_st2lane:
     case Intrinsic::aarch64_neon_st3lane:
     case Intrinsic::aarch64_neon_st4lane:
       return performNEONPostLDSTCombine(N, DCI, DAG);
     case Intrinsic::aarch64_sve_ldnt1:
       return performLDNT1Combine(N, DAG);
+    case Intrinsic::aarch64_sve_ld1rq:
+      return performLD1RQCombine(N, DAG);
     case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
     case Intrinsic::aarch64_sve_ldnt1_gather:
       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
     case Intrinsic::aarch64_sve_ldnt1_gather_index:
       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_INDEX);
     case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);

llvm/lib/Target/AArch64/AArch64InstrFormats.td

[{ return Imm >= 0 && Imm < (64*8) && ((Imm % 8) == 0); }]> {
   let ParserMatchClass = UImm6s8Operand;
 }
 def uimm6s16 : Operand<i64>, ImmLeaf<i64,
 [{ return Imm >= 0 && Imm < (64*16) && ((Imm % 16) == 0); }]> {
   let PrintMethod = "printImmScale<16>";
   let ParserMatchClass = UImm6s16Operand;
 }
 
+def SImmS2XForm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue() / 2, SDLoc(N), MVT::i64);
+}]>;
+def SImmS3XForm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue() / 3, SDLoc(N), MVT::i64);
+}]>;
+def SImmS4XForm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue() / 4, SDLoc(N), MVT::i64);
+}]>;
+def SImmS16XForm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue() / 16, SDLoc(N), MVT::i64);
+}]>;
+
 // simm6sN predicate - True if the immediate is a multiple of N in the range
 // [-32 * N, 31 * N].
 def SImm6s1Operand : SImmScaledMemoryIndexed<6, 1>;
 def simm6s1 : Operand<i64>, ImmLeaf<i64, [{ return Imm >= -32 && Imm < 32; }]> {
   let ParserMatchClass = SImm6s1Operand;
   let DecoderMethod = "DecodeSImm<6>";
 }
 
 // simm4sN predicate - True if the immediate is a multiple of N in the range
 // [ -8* N, 7 * N].
 def SImm4s1Operand  : SImmScaledMemoryIndexed<4, 1>;
 def SImm4s2Operand  : SImmScaledMemoryIndexed<4, 2>;
 def SImm4s3Operand  : SImmScaledMemoryIndexed<4, 3>;
 def SImm4s4Operand  : SImmScaledMemoryIndexed<4, 4>;
 def SImm4s16Operand : SImmScaledMemoryIndexed<4, 16>;
 
 def simm4s1 : Operand<i64>, ImmLeaf<i64,
 [{ return Imm >=-8  && Imm <= 7; }]> {
   let ParserMatchClass = SImm4s1Operand;
   let DecoderMethod = "DecodeSImm<4>";
 }
 
 def simm4s2 : Operand<i64>, ImmLeaf<i64,
-[{ return Imm >=-16  && Imm <= 14 && (Imm % 2) == 0x0; }]> {
+[{ return Imm >=-16  && Imm <= 14 && (Imm % 2) == 0x0; }], SImmS2XForm> {
   let PrintMethod = "printImmScale<2>";
   let ParserMatchClass = SImm4s2Operand;
   let DecoderMethod = "DecodeSImm<4>";
 }
 
 def simm4s3 : Operand<i64>, ImmLeaf<i64,
-[{ return Imm >=-24  && Imm <= 21 && (Imm % 3) == 0x0; }]> {
+[{ return Imm >=-24  && Imm <= 21 && (Imm % 3) == 0x0; }], SImmS3XForm> {
   let PrintMethod = "printImmScale<3>";
   let ParserMatchClass = SImm4s3Operand;
   let DecoderMethod = "DecodeSImm<4>";
 }
 
 def simm4s4 : Operand<i64>, ImmLeaf<i64,
-[{ return Imm >=-32  && Imm <= 28 && (Imm % 4) == 0x0; }]> {
+[{ return Imm >=-32  && Imm <= 28 && (Imm % 4) == 0x0; }], SImmS4XForm> {
   let PrintMethod = "printImmScale<4>";
   let ParserMatchClass = SImm4s4Operand;
   let DecoderMethod = "DecodeSImm<4>";
 }
 def simm4s16 : Operand<i64>, ImmLeaf<i64,
-[{ return Imm >=-128  && Imm <= 112 && (Imm % 16) == 0x0; }]> {
+[{ return Imm >=-128  && Imm <= 112 && (Imm % 16) == 0x0; }], SImmS16XForm> {
   let PrintMethod = "printImmScale<16>";
   let ParserMatchClass = SImm4s16Operand;
   let DecoderMethod = "DecodeSImm<4>";
 }
 
 def Imm1_8Operand : AsmImmRange<1, 8>;
 def Imm1_16Operand : AsmImmRange<1, 16>;
 def Imm1_32Operand : AsmImmRange<1, 32>;

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 //=- AArch64SVEInstrInfo.td -  AArch64 SVE Instructions -*- tablegen -*-----=//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // AArch64 Scalable Vector Extension (SVE) Instruction definitions.
 //
 //===----------------------------------------------------------------------===//
 
 def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>;
 def SVELShiftImm64 : ComplexPattern<i32, 1, "SelectSVEShiftImm64<0, 64>", []>;
 
 // Contiguous loads - node definitions
 //
 def SDT_AArch64_LD1 : SDTypeProfile<1, 3, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>,
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
 def AArch64ld1  : SDNode<"AArch64ISD::LD1",    SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
 def AArch64ld1s : SDNode<"AArch64ISD::LD1S",   SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
 
 // Non-faulting & first-faulting loads - node definitions
 //
 def AArch64ldnf1 : SDNode<"AArch64ISD::LDNF1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 def AArch64ldff1 : SDNode<"AArch64ISD::LDFF1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 
 def AArch64ldnf1s : SDNode<"AArch64ISD::LDNF1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 def AArch64ldff1s : SDNode<"AArch64ISD::LDFF1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 
+// Contiguous load and replicate - node definitions
+//
+
+def SDT_AArch64_LD1RQ : SDTypeProfile<1, 2, [
+  SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>,
+  SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
+]>;
+
+def AArch64ld1rq  : SDNode<"AArch64ISD::LD1RQ",  SDT_AArch64_LD1RQ, [SDNPHasChain, SDNPMayLoad]>;
+
 // Gather loads - node definitions
 //
 def SDT_AArch64_GATHER_SV : SDTypeProfile<1, 4, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisVT<4, OtherVT>,
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
 def SDT_AArch64_GATHER_VS : SDTypeProfile<1, 4, [
def : Pat<(AArch64ptest (nxv16i1 PPR:$pg), (nxv16i1 PPR:$src)),
             (PTEST_PP PPR:$pg, PPR:$src)>;
   def : Pat<(AArch64ptest (nxv8i1 PPR:$pg), (nxv8i1 PPR:$src)),
             (PTEST_PP PPR:$pg, PPR:$src)>;
   def : Pat<(AArch64ptest (nxv4i1 PPR:$pg), (nxv4i1 PPR:$src)),
             (PTEST_PP PPR:$pg, PPR:$src)>;
   def : Pat<(AArch64ptest (nxv2i1 PPR:$pg), (nxv2i1 PPR:$src)),
             (PTEST_PP PPR:$pg, PPR:$src)>;
 
+  // LD1R of 128-bit masked data
+  def : Pat<(nxv16i8 (AArch64ld1rq PPR:$gp, GPR64:$base)),
+            (LD1RQ_B_IMM $gp, $base, (i64 0))>;
+  def : Pat<(nxv8i16 (AArch64ld1rq PPR:$gp, GPR64:$base)),
+            (LD1RQ_H_IMM $gp, $base, (i64 0))>;
+  def : Pat<(nxv4i32 (AArch64ld1rq PPR:$gp, GPR64:$base)),
+            (LD1RQ_W_IMM $gp, $base, (i64 0))>;
+  def : Pat<(nxv2i64 (AArch64ld1rq PPR:$gp, GPR64:$base)),
+            (LD1RQ_D_IMM $gp, $base, (i64 0))>;
+
+  def : Pat<(nxv16i8 (AArch64ld1rq PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
+            (LD1RQ_B_IMM $gp, $base, simm4s16:$imm)>;
+  def : Pat<(nxv8i16 (AArch64ld1rq PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
+            (LD1RQ_H_IMM $gp, $base, simm4s16:$imm)>;
+  def : Pat<(nxv4i32 (AArch64ld1rq PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
+            (LD1RQ_W_IMM $gp, $base, simm4s16:$imm)>;
+  def : Pat<(nxv2i64 (AArch64ld1rq PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
+            (LD1RQ_D_IMM $gp, $base, simm4s16:$imm)>;
+
   def : Pat<(sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (SXTW_ZPmZ_D (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
   def : Pat<(sext_inreg (nxv2i64 ZPR:$Zs), nxv2i16), (SXTH_ZPmZ_D (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
   def : Pat<(sext_inreg (nxv2i64 ZPR:$Zs), nxv2i8),  (SXTB_ZPmZ_D (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
   def : Pat<(sext_inreg (nxv4i32 ZPR:$Zs), nxv4i16), (SXTH_ZPmZ_S (IMPLICIT_DEF), (PTRUE_S 31), ZPR:$Zs)>;
   def : Pat<(sext_inreg (nxv4i32 ZPR:$Zs), nxv4i8),  (SXTB_ZPmZ_S (IMPLICIT_DEF), (PTRUE_S 31), ZPR:$Zs)>;
   def : Pat<(sext_inreg (nxv8i16 ZPR:$Zs), nxv8i8),  (SXTB_ZPmZ_H (IMPLICIT_DEF), (PTRUE_H 31), ZPR:$Zs)>;
 
   // General case that we ideally never want to match.

llvm/test/CodeGen/AArch64/sve-intrinsics-loads.ll

 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
 ;
+; LD1RQB
+;
+
+define <vscale x 16 x i8> @ld1rqb_i8(<vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1rqb_i8:
+; CHECK: ld1rqb { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, i8* %addr)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @ld1rqb_i8_imm(<vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1rqb_i8_imm:
+; CHECK: ld1rqb { z0.b }, p0/z, [x0, #16]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds i8, i8* %addr, i8 16
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, i8* %ptr)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @ld1rqb_i8_imm_lower_bound(<vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1rqb_i8_imm_lower_bound:
+; CHECK: ld1rqb { z0.b }, p0/z, [x0, #-128]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds i8, i8* %addr, i8 -128
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, i8* %ptr)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @ld1rqb_i8_imm_upper_bound(<vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1rqb_i8_imm_upper_bound:
+; CHECK: ld1rqb { z0.b }, p0/z, [x0, #112]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds i8, i8* %addr, i8 112
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, i8* %ptr)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @ld1rqb_i8_imm_out_of_lower_bound(<vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1rqb_i8_imm_out_of_lower_bound:
+; CHECK: sub x8, x0, #129
+; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x8]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds i8, i8* %addr, i64 -129
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, i8* %ptr)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @ld1rqb_i8_imm_out_of_upper_bound(<vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1rqb_i8_imm_out_of_upper_bound:
+; CHECK: add x8, x0, #113
+; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x8]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds i8, i8* %addr, i64 113
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, i8* %ptr)
+  ret <vscale x 16 x i8> %res
+}
+
+;
+; LD1RQH
+;
+
+define <vscale x 8 x i16> @ld1rqh_i16(<vscale x 8 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1rqh_i16:
+; CHECK: ld1rqh { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1rq.nxv8i16(<vscale x 8 x i1> %pred, i16* %addr)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x half> @ld1rqh_f16(<vscale x 8 x i1> %pred, half* %addr) {
+; CHECK-LABEL: ld1rqh_f16:
+; CHECK: ld1rqh { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1rq.nxv8f16(<vscale x 8 x i1> %pred, half* %addr)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x i16> @ld1rqh_i16_imm(<vscale x 8 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1rqh_i16_imm:
+; CHECK: ld1rqh { z0.h }, p0/z, [x0, #-64]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds i16, i16* %addr, i16 -32
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1rq.nxv8i16(<vscale x 8 x i1> %pred, i16* %ptr)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x half> @ld1rqh_f16_imm(<vscale x 8 x i1> %pred, half* %addr) {
+; CHECK-LABEL: ld1rqh_f16_imm:
+; CHECK: ld1rqh { z0.h }, p0/z, [x0, #-16]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds half, half* %addr, i16 -8
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1rq.nxv8f16(<vscale x 8 x i1> %pred, half* %ptr)
+  ret <vscale x 8 x half> %res
+}
+
+;
+; LD1RQW
+;
+
+define <vscale x 4 x i32> @ld1rqw_i32(<vscale x 4 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: ld1rqw_i32:
+; CHECK: ld1rqw { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1rq.nxv4i32(<vscale x 4 x i1> %pred, i32* %addr)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x float> @ld1rqw_f32(<vscale x 4 x i1> %pred, float* %addr) {
+; CHECK-LABEL: ld1rqw_f32:
+; CHECK: ld1rqw { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1rq.nxv4f32(<vscale x 4 x i1> %pred, float* %addr)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x i32> @ld1rqw_i32_imm(<vscale x 4 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: ld1rqw_i32_imm:
+; CHECK: ld1rqw { z0.s }, p0/z, [x0, #112]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds i32, i32* %addr, i32 28
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1rq.nxv4i32(<vscale x 4 x i1> %pred, i32* %ptr)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x float> @ld1rqw_f32_imm(<vscale x 4 x i1> %pred, float* %addr) {
+; CHECK-LABEL: ld1rqw_f32_imm:
+; CHECK: ld1rqw { z0.s }, p0/z, [x0, #32]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds float, float* %addr, i32 8
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1rq.nxv4f32(<vscale x 4 x i1> %pred, float* %ptr)
+  ret <vscale x 4 x float> %res
+}
+
+;
+; LD1RQD
+;
+
+define <vscale x 2 x i64> @ld1rqd_i64(<vscale x 2 x i1> %pred, i64* %addr) {
+; CHECK-LABEL: ld1rqd_i64:
+; CHECK: ld1rqd { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1rq.nxv2i64(<vscale x 2 x i1> %pred, i64* %addr)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x double> @ld1rqd_f64(<vscale x 2 x i1> %pred, double* %addr) {
+; CHECK-LABEL: ld1rqd_f64:
+; CHECK: ld1rqd { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1rq.nxv2f64(<vscale x 2 x i1> %pred, double* %addr)
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x i64> @ld1rqd_i64_imm(<vscale x 2 x i1> %pred, i64* %addr) {
+; CHECK-LABEL: ld1rqd_i64_imm:
+; CHECK: ld1rqd { z0.d }, p0/z, [x0, #64]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds i64, i64* %addr, i64 8
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1rq.nxv2i64(<vscale x 2 x i1> %pred, i64* %ptr)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x double> @ld1rqd_f64_imm(<vscale x 2 x i1> %pred, double* %addr) {
+; CHECK-LABEL: ld1rqd_f64_imm:
+; CHECK: ld1rqd { z0.d }, p0/z, [x0, #-128]
+; CHECK-NEXT: ret
+  %ptr = getelementptr inbounds double, double* %addr, i64 -16
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1rq.nxv2f64(<vscale x 2 x i1> %pred, double* %ptr)
+  ret <vscale x 2 x double> %res
+}
+
+;
 ; LDNT1B
 ;
 
 define <vscale x 16 x i8> @ldnt1b_i8(<vscale x 16 x i1> %pred, i8* %addr) {
 ; CHECK-LABEL: ldnt1b_i8:
 ; CHECK: ldnt1b { z0.b }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> %pred,

 ; CHECK-LABEL: ldnt1d_f64:
 ; CHECK: ldnt1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT: ret
   %res = call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> %pred,
                                                                     double* %addr)
   ret <vscale x 2 x double> %res
 }
 
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1>, i8*)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1rq.nxv8i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1rq.nxv4i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1rq.nxv2i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ld1rq.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1rq.nxv4f32(<vscale x 4 x i1>, float*)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1rq.nxv2f64(<vscale x 2 x i1>, double*)
+
 declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, i8*)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, i16*)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, i32*)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, i64*)
 declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, half*)
 declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, float*)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, double*)