[AArch64][SVE] Remove LD1/ST1 dependency on llvm.masked.load/store

[kmclaughlin]

Summary

The SVE masked load and store intrinsics introduced in D76688 rely on
common llvm.masked.load/store nodes. This patch creates new ISD nodes
for LD1(S) & ST1 to remove this dependency.

Additionally, this adds support for sign & zero extending
loads and truncating stores.

[efriedma]

Practically, I'm not sure you're really getting much benefit out of this; there's very little common code that touches MLOAD/MSTORE nodes anyway. But, sure, LGTM.

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
 
   if (isa<MemIntrinsicSDNode>(Root))
     return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
 
   const unsigned Opcode = Root->getOpcode();
   // For custom ISD nodes, we have to look at them individually to extract the
   // type of the data moved to/from memory.
   switch (Opcode) {
+  case AArch64ISD::LD1:
+  case AArch64ISD::LD1S:
   case AArch64ISD::LDNF1:
   case AArch64ISD::LDNF1S:
     return cast<VTSDNode>(Root->getOperand(3))->getVT();
+  case AArch64ISD::ST1:
+    return cast<VTSDNode>(Root->getOperand(4))->getVT();
   default:
     break;
   }
 
   if (Opcode != ISD::INTRINSIC_VOID)
     return EVT();
 
   const unsigned IntNo =

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   PTEST,
   PTRUE,
 
   DUP_PRED,
   INDEX_VECTOR,
 
   REINTERPRET_CAST,
 
+  LD1,
+  LD1S,
   LDNF1,
   LDNF1S,
   LDFF1,
   LDFF1S,
 
   // Unsigned gather loads.
   GLD1,
   GLD1_SCALED,
enum NodeType : unsigned {
   GLDFF1S_SXTW_SCALED,
   GLDFF1S_IMM,
 
   // Non-temporal gather loads
   GLDNT1,
   GLDNT1_INDEX,
   GLDNT1S,
 
+  ST1,
+
   // Scatter store
   SST1,
   SST1_SCALED,
   SST1_UXTW,
   SST1_SXTW,
   SST1_UXTW_SCALED,
   SST1_SXTW_SCALED,
   SST1_IMM,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case AArch64ISD::STZ2G:             return "AArch64ISD::STZ2G";
   case AArch64ISD::SUNPKHI:           return "AArch64ISD::SUNPKHI";
   case AArch64ISD::SUNPKLO:           return "AArch64ISD::SUNPKLO";
   case AArch64ISD::UUNPKHI:           return "AArch64ISD::UUNPKHI";
   case AArch64ISD::UUNPKLO:           return "AArch64ISD::UUNPKLO";
   case AArch64ISD::INSR:              return "AArch64ISD::INSR";
   case AArch64ISD::PTEST:             return "AArch64ISD::PTEST";
   case AArch64ISD::PTRUE:             return "AArch64ISD::PTRUE";
+  case AArch64ISD::LD1:               return "AArch64ISD::LD1";
+  case AArch64ISD::LD1S:              return "AArch64ISD::LD1S";
   case AArch64ISD::LDNF1:             return "AArch64ISD::LDNF1";
   case AArch64ISD::LDNF1S:            return "AArch64ISD::LDNF1S";
   case AArch64ISD::LDFF1:             return "AArch64ISD::LDFF1";
   case AArch64ISD::LDFF1S:            return "AArch64ISD::LDFF1S";
   case AArch64ISD::GLD1:              return "AArch64ISD::GLD1";
   case AArch64ISD::GLD1_SCALED:       return "AArch64ISD::GLD1_SCALED";
   case AArch64ISD::GLD1_SXTW:         return "AArch64ISD::GLD1_SXTW";
   case AArch64ISD::GLD1_UXTW:         return "AArch64ISD::GLD1_UXTW";
const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case AArch64ISD::GLDFF1S_UXTW_SCALED:
     return "AArch64ISD::GLDFF1S_UXTW_SCALED";
   case AArch64ISD::GLDFF1S_IMM:       return "AArch64ISD::GLDFF1S_IMM";
 
   case AArch64ISD::GLDNT1:            return "AArch64ISD::GLDNT1";
   case AArch64ISD::GLDNT1_INDEX:      return "AArch64ISD::GLDNT1_INDEX";
   case AArch64ISD::GLDNT1S:           return "AArch64ISD::GLDNT1S";
 
+  case AArch64ISD::ST1:               return "AArch64ISD::ST1";
+
   case AArch64ISD::SST1:              return "AArch64ISD::SST1";
   case AArch64ISD::SST1_SCALED:       return "AArch64ISD::SST1_SCALED";
   case AArch64ISD::SST1_SXTW:         return "AArch64ISD::SST1_SXTW";
   case AArch64ISD::SST1_UXTW:         return "AArch64ISD::SST1_UXTW";
   case AArch64ISD::SST1_SXTW_SCALED:  return "AArch64ISD::SST1_SXTW_SCALED";
   case AArch64ISD::SST1_UXTW_SCALED:  return "AArch64ISD::SST1_UXTW_SCALED";
   case AArch64ISD::SST1_IMM:          return "AArch64ISD::SST1_IMM";
 
bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   case Intrinsic::aarch64_stxp:
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::i128;
     Info.ptrVal = I.getArgOperand(2);
     Info.offset = 0;
     Info.align = Align(16);
     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
     return true;
-  case Intrinsic::aarch64_sve_ld1:
   case Intrinsic::aarch64_sve_ldnt1: {
     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(I.getType());
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
     Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
     Info.flags = MachineMemOperand::MOLoad;
     if (Intrinsic == Intrinsic::aarch64_sve_ldnt1)
       Info.flags |= MachineMemOperand::MONonTemporal;
     return true;
   }
-  case Intrinsic::aarch64_sve_st1:
   case Intrinsic::aarch64_sve_stnt1: {
     PointerType *PtrTy = cast<PointerType>(I.getArgOperand(2)->getType());
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(I.getOperand(0)->getType());
     Info.ptrVal = I.getArgOperand(2);
     Info.offset = 0;
     Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
     Info.flags = MachineMemOperand::MOStore;
static SDValue performSVEAndCombine(SDNode *N,
   if (!Src.hasOneUse())
     return SDValue();
 
   EVT MemVT;
 
   // SVE load instructions perform an implicit zero-extend, which makes them
   // perfect candidates for combining.
   switch (Src->getOpcode()) {
+  case AArch64ISD::LD1:
   case AArch64ISD::LDNF1:
   case AArch64ISD::LDFF1:
     MemVT = cast<VTSDNode>(Src->getOperand(3))->getVT();
     break;
   case AArch64ISD::GLD1:
   case AArch64ISD::GLD1_SCALED:
   case AArch64ISD::GLD1_SXTW:
   case AArch64ISD::GLD1_SXTW_SCALED:
static MVT getSVEContainerType(EVT ContentTy) {
   case MVT::nxv8i16:
   case MVT::nxv8f16:
     return MVT::nxv8i16;
   case MVT::nxv16i8:
     return MVT::nxv16i8;
   }
 }
 
-static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG) {
+static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+
+  if (VT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
+    return SDValue();
+
+  EVT ContainerVT = VT;
+  if (ContainerVT.isInteger())
+    ContainerVT = getSVEContainerType(ContainerVT);
+
+  SDVTList VTs = DAG.getVTList(ContainerVT, MVT::Other);
+  SDValue Ops[] = { N->getOperand(0), // Chain
+                    N->getOperand(2), // Pg
+                    N->getOperand(3), // Base
+                    DAG.getValueType(VT) };
+
+  SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
+  SDValue LoadChain = SDValue(Load.getNode(), 1);
+
+  if (ContainerVT.isInteger() && (VT != ContainerVT))
+    Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0));
+
+  return DAG.getMergeValues({ Load, LoadChain }, DL);
+}
+
+static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
   EVT PtrTy = N->getOperand(3).getValueType();
 
   EVT LoadVT = VT;
   if (VT.isFloatingPoint())
     LoadVT = VT.changeTypeToInteger();
 
if (VT.isFloatingPoint()) {
      return DAG.getMergeValues(Ops, DL);
    }
 
   return L;
 }
 
 static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
   SDLoc DL(N);
+  SDValue Data = N->getOperand(2);
+  EVT DataVT = Data.getValueType();
+  EVT HwSrcVt = getSVEContainerType(DataVT);
+  SDValue InputVT = DAG.getValueType(DataVT);
+
+  if (DataVT.isFloatingPoint())
+    InputVT = DAG.getValueType(HwSrcVt);
+
+  SDValue SrcNew;
+  if (Data.getValueType().isFloatingPoint())
+    SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Data);
+  else
+    SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Data);
+
+  SDValue Ops[] = { N->getOperand(0), // Chain
+                    SrcNew,
+                    N->getOperand(4), // Base
+                    N->getOperand(3), // Pg
+                    InputVT
+                  };
+
+  return DAG.getNode(AArch64ISD::ST1, DL, N->getValueType(0), Ops);
+}
+
+static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
+  SDLoc DL(N);
 
   SDValue Data = N->getOperand(2);
   EVT DataVT = Data.getValueType();
   EVT PtrTy = N->getOperand(4).getValueType();
 
   if (DataVT.isFloatingPoint())
     Data = DAG.getNode(ISD::BITCAST, DL, DataVT.changeTypeToInteger(), Data);
 
   auto *MINode = cast<MemIntrinsicSDNode>(N);
   return DAG.getMaskedStore(MINode->getChain(), DL, Data, MINode->getOperand(4),
                             DAG.getUNDEF(PtrTy), MINode->getOperand(3),
                             MINode->getMemoryVT(), MINode->getMemOperand(),
                             ISD::UNINDEXED, false, false);
 }
 
-static SDValue performLDNF1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
-  SDLoc DL(N);
-  EVT VT = N->getValueType(0);
-
-  if (VT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
-    return SDValue();
-
-  EVT ContainerVT = VT;
-  if (ContainerVT.isInteger())
-    ContainerVT = getSVEContainerType(ContainerVT);
-
-  SDVTList VTs = DAG.getVTList(ContainerVT, MVT::Other);
-  SDValue Ops[] = { N->getOperand(0), // Chain
-                    N->getOperand(2), // Pg
-                    N->getOperand(3), // Base
-                    DAG.getValueType(VT) };
-
-  SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
-  SDValue LoadChain = SDValue(Load.getNode(), 1);
-
-  if (ContainerVT.isInteger() && (VT != ContainerVT))
-    Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0));
-
-  return DAG.getMergeValues({ Load, LoadChain }, DL);
-}
-
 /// Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.  The
 /// load store optimizer pass will merge them to store pair stores.  This should
 /// be better than a movi to create the vector zero followed by a vector store
 /// if the zero constant is not re-used, since one instructions and one register
 /// live range will be removed.
 ///
 /// For example, the final generated code should be:
 ///
performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
   SDValue Src = N->getOperand(0);
   unsigned Opc = Src->getOpcode();
 
   // SVE load nodes (e.g. AArch64ISD::GLD1) are straightforward candidates
   // for DAG Combine with SIGN_EXTEND_INREG. Bail out for all other nodes.
   unsigned NewOpc;
   unsigned MemVTOpNum = 4;
   switch (Opc) {
+  case AArch64ISD::LD1:
+    NewOpc = AArch64ISD::LD1S;
+    MemVTOpNum = 3;
+    break;
   case AArch64ISD::LDNF1:
     NewOpc = AArch64ISD::LDNF1S;
     MemVTOpNum = 3;
     break;
   case AArch64ISD::LDFF1:
     NewOpc = AArch64ISD::LDFF1S;
     MemVTOpNum = 3;
     break;
case ISD::INTRINSIC_W_CHAIN:
     case Intrinsic::aarch64_neon_st4:
     case Intrinsic::aarch64_neon_st1x2:
     case Intrinsic::aarch64_neon_st1x3:
     case Intrinsic::aarch64_neon_st1x4:
     case Intrinsic::aarch64_neon_st2lane:
     case Intrinsic::aarch64_neon_st3lane:
     case Intrinsic::aarch64_neon_st4lane:
       return performNEONPostLDSTCombine(N, DCI, DAG);
-    case Intrinsic::aarch64_sve_ld1:
     case Intrinsic::aarch64_sve_ldnt1:
-      return performLD1Combine(N, DAG);
+      return performLDNT1Combine(N, DAG);
     case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
     case Intrinsic::aarch64_sve_ldnt1_gather:
       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
     case Intrinsic::aarch64_sve_ldnt1_gather_index:
       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_INDEX);
     case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
       return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
+    case Intrinsic::aarch64_sve_ld1:
+      return performLD1Combine(N, DAG, AArch64ISD::LD1);
     case Intrinsic::aarch64_sve_ldnf1:
-      return performLDNF1Combine(N, DAG, AArch64ISD::LDNF1);
+      return performLD1Combine(N, DAG, AArch64ISD::LDNF1);
     case Intrinsic::aarch64_sve_ldff1:
-      return performLDNF1Combine(N, DAG, AArch64ISD::LDFF1);
+      return performLD1Combine(N, DAG, AArch64ISD::LDFF1);
     case Intrinsic::aarch64_sve_st1:
-    case Intrinsic::aarch64_sve_stnt1:
       return performST1Combine(N, DAG);
+    case Intrinsic::aarch64_sve_stnt1:
+      return performSTNT1Combine(N, DAG);
     case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1);
     case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1);
     case Intrinsic::aarch64_sve_stnt1_scatter:
       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1);
     case Intrinsic::aarch64_sve_stnt1_scatter_index:
       return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1_INDEX);

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 //=- AArch64SVEInstrInfo.td -  AArch64 SVE Instructions -*- tablegen -*-----=//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // AArch64 Scalable Vector Extension (SVE) Instruction definitions.
 //
 //===----------------------------------------------------------------------===//
 
 def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>;
 def SVELShiftImm64 : ComplexPattern<i32, 1, "SelectSVEShiftImm64<0, 64>", []>;
 
-// Non-faulting & first-faulting loads - node definitions
+// Contiguous loads - node definitions
 //
-def SDT_AArch64_LDNF1 : SDTypeProfile<1, 3, [
+def SDT_AArch64_LD1 : SDTypeProfile<1, 3, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>,
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
-def AArch64ldnf1 : SDNode<"AArch64ISD::LDNF1", SDT_AArch64_LDNF1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1 : SDNode<"AArch64ISD::LDFF1", SDT_AArch64_LDNF1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ld1  : SDNode<"AArch64ISD::LD1",    SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
+def AArch64ld1s : SDNode<"AArch64ISD::LD1S",   SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
+
+// Non-faulting & first-faulting loads - node definitions
+//
+def AArch64ldnf1 : SDNode<"AArch64ISD::LDNF1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1 : SDNode<"AArch64ISD::LDFF1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 
-def AArch64ldnf1s : SDNode<"AArch64ISD::LDNF1S", SDT_AArch64_LDNF1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s : SDNode<"AArch64ISD::LDFF1S", SDT_AArch64_LDNF1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldnf1s : SDNode<"AArch64ISD::LDNF1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s : SDNode<"AArch64ISD::LDFF1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 
 // Gather loads - node definitions
 //
 def SDT_AArch64_GATHER_SV : SDTypeProfile<1, 4, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisVT<4, OtherVT>,
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 

 def AArch64ldff1s_gather_sxtw        : SDNode<"AArch64ISD::GLDFF1S_SXTW",        SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 def AArch64ldff1s_gather_uxtw_scaled : SDNode<"AArch64ISD::GLDFF1S_UXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 def AArch64ldff1s_gather_sxtw_scaled : SDNode<"AArch64ISD::GLDFF1S_SXTW_SCALED", SDT_AArch64_GATHER_SV, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 def AArch64ldff1s_gather_imm         : SDNode<"AArch64ISD::GLDFF1S_IMM",         SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
 
 def AArch64ldnt1_gather  : SDNode<"AArch64ISD::GLDNT1",  SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
 def AArch64ldnt1s_gather : SDNode<"AArch64ISD::GLDNT1S", SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
 
+// Contiguous stores - node definitions
+//
+def SDT_AArch64_ST1 : SDTypeProfile<0, 4, [
+  SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisVec<2>,
+  SDTCVecEltisVT<2,i1>, SDTCisSameNumEltsAs<0,2>
+]>;
+
+def AArch64st1 : SDNode<"AArch64ISD::ST1", SDT_AArch64_ST1, [SDNPHasChain, SDNPMayStore]>;
+
 // Scatter stores - node definitions
 //
 def SDT_AArch64_SCATTER_SV : SDTypeProfile<0, 5, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisVT<4, OtherVT>,
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
 def SDT_AArch64_SCATTER_VS : SDTypeProfile<0, 5, [
def _default : Pat<(Ty (load GPR64:$base)),
                   (Load GPR64:$base, (i64 0))>;
   }
 
   defm Pat_Load_P16 : unpred_load_predicate<nxv16i1, LDR_PXI>;
   defm Pat_Load_P8  : unpred_load_predicate<nxv8i1, LDR_PXI>;
   defm Pat_Load_P4  : unpred_load_predicate<nxv4i1, LDR_PXI>;
   defm Pat_Load_P2  : unpred_load_predicate<nxv2i1, LDR_PXI>;
 
-  multiclass ldnf1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT> {
+  multiclass ld1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT> {
     // scalar + immediate (mul vl)
     let AddedComplexity = 1 in {
       def : Pat<(Ty (Load  (PredTy PPR:$gp), (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), MemVT)),
                 (I PPR:$gp, GPR64sp:$base, simm4s1:$offset)>;
     }
 
     // base
     def : Pat<(Ty (Load  (PredTy PPR:$gp), GPR64:$base, MemVT)),
               (I PPR:$gp, GPR64sp:$base, (i64 0))>;
   }
 
+  // 2-element contiguous loads
+  defm : ld1<LD1B_D_IMM,  nxv2i64, AArch64ld1,  nxv2i1, nxv2i8>;
+  defm : ld1<LD1SB_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i8>;
+  defm : ld1<LD1H_D_IMM,  nxv2i64, AArch64ld1,  nxv2i1, nxv2i16>;
+  defm : ld1<LD1SH_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i16>;
+  defm : ld1<LD1W_D_IMM,  nxv2i64, AArch64ld1,  nxv2i1, nxv2i32>;
+  defm : ld1<LD1SW_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i32>;
+  defm : ld1<LD1D_IMM,    nxv2i64, AArch64ld1,  nxv2i1, nxv2i64>;
+  defm : ld1<LD1D_IMM,    nxv2f64, AArch64ld1,  nxv2i1, nxv2f64>;
+
+  // 4-element contiguous loads
+  defm : ld1<LD1B_S_IMM,  nxv4i32, AArch64ld1,  nxv4i1, nxv4i8>;
+  defm : ld1<LD1SB_S_IMM, nxv4i32, AArch64ld1s, nxv4i1, nxv4i8>;
+  defm : ld1<LD1H_S_IMM,  nxv4i32, AArch64ld1,  nxv4i1, nxv4i16>;
+  defm : ld1<LD1SH_S_IMM, nxv4i32, AArch64ld1s, nxv4i1, nxv4i16>;
+  defm : ld1<LD1W_IMM,    nxv4i32, AArch64ld1,  nxv4i1, nxv4i32>;
+  defm : ld1<LD1W_IMM,    nxv4f32, AArch64ld1,  nxv4i1, nxv4f32>;
+
+  // 8-element contiguous loads
+  defm : ld1<LD1B_H_IMM,  nxv8i16, AArch64ld1,  nxv8i1, nxv8i8>;
+  defm : ld1<LD1SB_H_IMM, nxv8i16, AArch64ld1s, nxv8i1, nxv8i8>;
+  defm : ld1<LD1H_IMM,    nxv8i16, AArch64ld1,  nxv8i1, nxv8i16>;
+  defm : ld1<LD1H_IMM,    nxv8f16, AArch64ld1,  nxv8i1, nxv8f16>;
+
+  // 16-element contiguous loads
+  defm : ld1<LD1B_IMM,    nxv16i8, AArch64ld1, nxv16i1, nxv16i8>;
+
+
   // 2-element contiguous non-faulting loads
-  defm : ldnf1<LDNF1B_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i8>;
-  defm : ldnf1<LDNF1SB_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i8>;
-  defm : ldnf1<LDNF1H_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i16>;
-  defm : ldnf1<LDNF1SH_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i16>;
-  defm : ldnf1<LDNF1W_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i32>;
-  defm : ldnf1<LDNF1SW_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i32>;
-  defm : ldnf1<LDNF1D_IMM,    nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i64>;
-  defm : ldnf1<LDNF1D_IMM,    nxv2f64, AArch64ldnf1,  nxv2i1, nxv2f64>;
+  defm : ld1<LDNF1B_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i8>;
+  defm : ld1<LDNF1SB_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i8>;
+  defm : ld1<LDNF1H_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i16>;
+  defm : ld1<LDNF1SH_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i16>;
+  defm : ld1<LDNF1W_D_IMM,  nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i32>;
+  defm : ld1<LDNF1SW_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i32>;
+  defm : ld1<LDNF1D_IMM,    nxv2i64, AArch64ldnf1,  nxv2i1, nxv2i64>;
+  defm : ld1<LDNF1D_IMM,    nxv2f64, AArch64ldnf1,  nxv2i1, nxv2f64>;
 
   // 4-element contiguous non-faulting loads
-  defm : ldnf1<LDNF1B_S_IMM,  nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i8>;
-  defm : ldnf1<LDNF1SB_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i8>;
-  defm : ldnf1<LDNF1H_S_IMM,  nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i16>;
-  defm : ldnf1<LDNF1SH_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i16>;
-  defm : ldnf1<LDNF1W_IMM,    nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i32>;
-  defm : ldnf1<LDNF1W_IMM,    nxv4f32, AArch64ldnf1,  nxv4i1, nxv4f32>;
+  defm : ld1<LDNF1B_S_IMM,  nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i8>;
+  defm : ld1<LDNF1SB_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i8>;
+  defm : ld1<LDNF1H_S_IMM,  nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i16>;
+  defm : ld1<LDNF1SH_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i16>;
+  defm : ld1<LDNF1W_IMM,    nxv4i32, AArch64ldnf1,  nxv4i1, nxv4i32>;
+  defm : ld1<LDNF1W_IMM,    nxv4f32, AArch64ldnf1,  nxv4i1, nxv4f32>;
 
   // 8-element contiguous non-faulting loads
-  defm : ldnf1<LDNF1B_H_IMM,  nxv8i16, AArch64ldnf1,  nxv8i1, nxv8i8>;
-  defm : ldnf1<LDNF1SB_H_IMM, nxv8i16, AArch64ldnf1s, nxv8i1, nxv8i8>;
-  defm : ldnf1<LDNF1H_IMM,    nxv8i16, AArch64ldnf1,  nxv8i1, nxv8i16>;
-  defm : ldnf1<LDNF1H_IMM,    nxv8f16, AArch64ldnf1,  nxv8i1, nxv8f16>;
+  defm : ld1<LDNF1B_H_IMM,  nxv8i16, AArch64ldnf1,  nxv8i1, nxv8i8>;
+  defm : ld1<LDNF1SB_H_IMM, nxv8i16, AArch64ldnf1s, nxv8i1, nxv8i8>;
+  defm : ld1<LDNF1H_IMM,    nxv8i16, AArch64ldnf1,  nxv8i1, nxv8i16>;
+  defm : ld1<LDNF1H_IMM,    nxv8f16, AArch64ldnf1,  nxv8i1, nxv8f16>;
 
   // 16-element contiguous non-faulting loads
-  defm : ldnf1<LDNF1B_IMM,    nxv16i8, AArch64ldnf1, nxv16i1, nxv16i8>;
+  defm : ld1<LDNF1B_IMM,    nxv16i8, AArch64ldnf1, nxv16i1, nxv16i8>;
 
   multiclass ldff1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT, ComplexPattern AddrCP> {
     // reg + reg
     let AddedComplexity = 1 in {
       def : Pat<(Ty (Load  (PredTy PPR:$gp), (AddrCP GPR64:$base, GPR64:$offset), MemVT)),
                 (I PPR:$gp, GPR64sp:$base, GPR64:$offset)>;
     }
 
multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instruction RegImmInst, Instruction RegRegInst, int scale, ComplexPattern AddrCP> {
   // 8-element contiguous first faulting loads
   defm : ldff1<LDFF1B_H,  nxv8i16, AArch64ldff1,  nxv8i1, nxv8i8,  am_sve_regreg_lsl0>;
   defm : ldff1<LDFF1SB_H, nxv8i16, AArch64ldff1s, nxv8i1, nxv8i8,  am_sve_regreg_lsl0>;
   defm : ldff1<LDFF1H,    nxv8i16, AArch64ldff1,  nxv8i1, nxv8i16, am_sve_regreg_lsl1>;
   defm : ldff1<LDFF1H,    nxv8f16, AArch64ldff1,  nxv8i1, nxv8f16, am_sve_regreg_lsl1>;
 
   // 16-element contiguous first faulting loads
   defm : ldff1<LDFF1B, nxv16i8, AArch64ldff1, nxv16i1, nxv16i8, am_sve_regreg_lsl0>;
+
+  multiclass st1<Instruction I, ValueType Ty, SDPatternOperator Store, ValueType PredTy, ValueType MemVT> {
+    // scalar + immediate (mul vl)
+    let AddedComplexity = 1 in {
+      def : Pat<(Store (Ty ZPR:$vec), (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), (PredTy PPR:$gp), MemVT),
+                (I ZPR:$vec, PPR:$gp, GPR64sp:$base, simm4s1:$offset)>;
+    }
+
+    // base
+    def : Pat<(Store (Ty ZPR:$vec), GPR64:$base, (PredTy PPR:$gp), MemVT),
+              (I ZPR:$vec, PPR:$gp, GPR64:$base, (i64 0))>;
+  }
+
+  // 2-element contiguous store
+  defm : st1<ST1B_D_IMM, nxv2i64, AArch64st1, nxv2i1, nxv2i8>;
+  defm : st1<ST1H_D_IMM, nxv2i64, AArch64st1, nxv2i1, nxv2i16>;
+  defm : st1<ST1W_D_IMM, nxv2i64, AArch64st1, nxv2i1, nxv2i32>;
+  defm : st1<ST1D_IMM,   nxv2i64, AArch64st1, nxv2i1, nxv2i64>;
+
+  // 4-element contiguous store
+  defm : st1<ST1B_S_IMM, nxv4i32, AArch64st1, nxv4i1, nxv4i8>;
+  defm : st1<ST1H_S_IMM, nxv4i32, AArch64st1, nxv4i1, nxv4i16>;
+  defm : st1<ST1W_IMM,   nxv4i32, AArch64st1, nxv4i1, nxv4i32>;
+
+  // 8-element contiguous store
+  defm : st1<ST1B_H_IMM, nxv8i16, AArch64st1, nxv8i1, nxv8i8>;
+  defm : st1<ST1H_IMM,   nxv8i16, AArch64st1, nxv8i1, nxv8i16>;
+
+  // 16-element contiguous store
+  defm : st1<ST1B_IMM,   nxv16i8, AArch64st1, nxv16i1, nxv16i8>;
+
 }
 
 let Predicates = [HasSVE2] in {
   // SVE2 integer multiply-add (indexed)
   defm MLA_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b0, "mla", int_aarch64_sve_mla_lane>;
   defm MLS_ZZZI : sve2_int_mla_by_indexed_elem<0b01, 0b1, "mls", int_aarch64_sve_mls_lane>;
 
   // SVE2 saturating multiply-add high (indexed)

llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; LD1B
+;
+
+define <vscale x 16 x i8> @ld1b_i8(<vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1b_i8:
+; CHECK: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pred, i8* %addr)
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @ld1b_h(<vscale x 8 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1b_h:
+; CHECK: ld1b { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1> %pred, i8* %addr)
+  %res = zext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @ld1sb_h(<vscale x 8 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1sb_h:
+; CHECK: ld1sb { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1> %pred, i8* %addr)
+  %res = sext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @ld1b_s(<vscale x 4 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1b_s:
+; CHECK: ld1b { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1> %pred, i8* %addr)
+  %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ld1sb_s(<vscale x 4 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1sb_s:
+; CHECK: ld1sb { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1> %pred, i8* %addr)
+  %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @ld1b_d(<vscale x 2 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1b_d:
+; CHECK: ld1b { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1> %pred, i8* %addr)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sb_d(<vscale x 2 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1sb_d:
+; CHECK: ld1sb { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1> %pred, i8* %addr)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @ld1b_upper_bound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_upper_bound:
+; CHECK: ld1b { z0.b }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_inbound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_inbound:
+; CHECK: ld1b { z0.b }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 4 x i32> @ld1b_s_inbound(<vscale x 4 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_s_inbound:
+; CHECK: ld1b { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 4 x i8>*
+  %base = getelementptr <vscale x 4 x i8>, <vscale x 4 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 4 x i8>* %base to i8*
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1> %pg, i8* %base_scalar)
+  %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ld1sb_s_inbound(<vscale x 4 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1sb_s_inbound:
+; CHECK: ld1sb { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 4 x i8>*
+  %base = getelementptr <vscale x 4 x i8>, <vscale x 4 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 4 x i8>* %base to i8*
+  %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1> %pg, i8* %base_scalar)
+  %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 16 x i8> @ld1b_lower_bound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_lower_bound:
+; CHECK: ld1b { z0.b }, p0/z, [x0, #-8, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 -8
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_out_of_upper_bound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_out_of_upper_bound:
+; CHECK:       rdvl    x[[OFFSET:[0-9]+]], #8
+; CHECK-NEXT:  add     x[[BASE:[0-9]+]], x0, x[[OFFSET]]
+; CHECK-NEXT:  ld1b { z0.b }, p0/z, [x[[BASE]]]
+; CHECK-NEXT:  ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 8
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_out_of_lower_bound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_out_of_lower_bound:
+; CHECK:       rdvl    x[[OFFSET:[0-9]+]], #-9
+; CHECK-NEXT:  add     x[[BASE:[0-9]+]], x0, x[[OFFSET]]
+; CHECK-NEXT:  ld1b { z0.b }, p0/z, [x[[BASE]]]
+; CHECK-NEXT:  ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 -9
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret <vscale x 16 x i8> %load
+}
+
+;
+; LD1H
+;
+
+define <vscale x 8 x i16> @ld1h_i16(<vscale x 8 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1h_i16:
+; CHECK: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> %pred, i16* %addr)
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x half> @ld1h_f16(<vscale x 8 x i1> %pred, half* %addr) {
+; CHECK-LABEL: ld1h_f16:
+; CHECK: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1> %pred, half* %addr)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x i32> @ld1h_s(<vscale x 4 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1h_s:
+; CHECK: ld1h { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1> %pred, i16* %addr)
+  %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ld1sh_s(<vscale x 4 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1sh_s:
+; CHECK: ld1sh { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1> %pred, i16* %addr)
+  %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @ld1h_d(<vscale x 2 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1h_d:
+; CHECK: ld1h { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1> %pred, i16* %addr)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sh_d(<vscale x 2 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1sh_d:
+; CHECK: ld1sh { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1> %pred, i16* %addr)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x i16> @ld1b_h_inbound(<vscale x 8 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_h_inbound:
+; CHECK: ld1b { z0.h }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 8 x i8>*
+  %base = getelementptr <vscale x 8 x i8>, <vscale x 8 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 8 x i8>* %base to i8*
+  %load = call <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1> %pg, i8* %base_scalar)
+  %res = zext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @ld1sb_h_inbound(<vscale x 8 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1sb_h_inbound:
+; CHECK: ld1sb { z0.h }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 8 x i8>*
+  %base = getelementptr <vscale x 8 x i8>, <vscale x 8 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 8 x i8>* %base to i8*
+  %load = call <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1> %pg, i8* %base_scalar)
+  %res = sext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @ld1h_inbound(<vscale x 8 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1h_inbound:
+; CHECK: ld1h { z0.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i16* %a to <vscale x 8 x i16>*
+  %base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 8 x i16>* %base to i16*
+  %load = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> %pg, i16* %base_scalar)
+  ret <vscale x 8 x i16> %load
+}
+
+define <vscale x 4 x i32> @ld1h_s_inbound(<vscale x 4 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1h_s_inbound:
+; CHECK: ld1h { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i16* %a to <vscale x 4 x i16>*
+  %base = getelementptr <vscale x 4 x i16>, <vscale x 4 x i16>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 4 x i16>* %base to i16*
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1> %pg, i16* %base_scalar)
+  %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ld1sh_s_inbound(<vscale x 4 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1sh_s_inbound:
+; CHECK: ld1sh { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i16* %a to <vscale x 4 x i16>*
+  %base = getelementptr <vscale x 4 x i16>, <vscale x 4 x i16>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 4 x i16>* %base to i16*
+  %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1> %pg, i16* %base_scalar)
+  %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @ld1b_d_inbound(<vscale x 2 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_d_inbound:
+; CHECK: ld1b { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 2 x i8>*
+  %base = getelementptr <vscale x 2 x i8>, <vscale x 2 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 2 x i8>* %base to i8*
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1> %pg, i8* %base_scalar)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sb_d_inbound(<vscale x 2 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1sb_d_inbound:
+; CHECK: ld1sb { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 2 x i8>*
+  %base = getelementptr <vscale x 2 x i8>, <vscale x 2 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 2 x i8>* %base to i8*
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1> %pg, i8* %base_scalar)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1h_d_inbound(<vscale x 2 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1h_d_inbound:
+; CHECK: ld1h { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i16* %a to <vscale x 2 x i16>*
+  %base = getelementptr <vscale x 2 x i16>, <vscale x 2 x i16>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 2 x i16>* %base to i16*
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1> %pg, i16* %base_scalar)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sh_d_inbound(<vscale x 2 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1sh_d_inbound:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i16* %a to <vscale x 2 x i16>*
+  %base = getelementptr <vscale x 2 x i16>, <vscale x 2 x i16>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 2 x i16>* %base to i16*
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1> %pg, i16* %base_scalar)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x half> @ld1h_f16_inbound(<vscale x 8 x i1> %pg, half* %a) {
+; CHECK-LABEL: ld1h_f16_inbound:
+; CHECK: ld1h { z0.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast half* %a to <vscale x 8 x half>*
+  %base = getelementptr <vscale x 8 x half>, <vscale x 8 x half>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 8 x half>* %base to half*
+  %load = call <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1> %pg, half* %base_scalar)
+  ret <vscale x 8 x half> %load
+}
+
+;
+; LD1W
+;
+
+define <vscale x 4 x i32> @ld1w_i32(<vscale x 4 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: ld1w_i32:
+; CHECK: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %pred, i32* %addr)
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x float> @ld1w_f32(<vscale x 4 x i1> %pred, float* %addr) {
+; CHECK-LABEL: ld1w_f32:
+; CHECK: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1> %pred, float* %addr)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x i64> @ld1w_d(<vscale x 2 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: ld1w_d:
+; CHECK: ld1w { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %pred, i32* %addr)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sw_d(<vscale x 2 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: ld1sw_d:
+; CHECK: ld1sw { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %pred, i32* %addr)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @ld1w_inbound(<vscale x 4 x i1> %pg, i32* %a) {
+; CHECK-LABEL: ld1w_inbound:
+; CHECK: ld1w { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i32* %a to <vscale x 4 x i32>*
+  %base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 4 x i32>* %base to i32*
+  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %pg, i32* %base_scalar)
+  ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 4 x float> @ld1w_f32_inbound(<vscale x 4 x i1> %pg, float* %a) {
+; CHECK-LABEL: ld1w_f32_inbound:
+; CHECK: ld1w { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast float* %a to <vscale x 4 x float>*
+  %base = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 4 x float>* %base to float*
+  %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1> %pg, float* %base_scalar)
+  ret <vscale x 4 x float> %load
+}
+
+;
+; LD1D
+;
+
+define <vscale x 2 x i64> @ld1d_i64(<vscale x 2 x i1> %pred, i64* %addr) {
+; CHECK-LABEL: ld1d_i64:
+; CHECK: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1> %pred,
+                                                               i64* %addr)
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x double> @ld1d_f64(<vscale x 2 x i1> %pred, double* %addr) {
+; CHECK-LABEL: ld1d_f64:
+; CHECK: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1> %pred,
+                                                                  double* %addr)
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x i64> @ld1d_inbound(<vscale x 2 x i1> %pg, i64* %a) {
+; CHECK-LABEL: ld1d_inbound:
+; CHECK: ld1d { z0.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i64* %a to <vscale x 2 x i64>*
+  %base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 2 x i64>* %base to i64*
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1> %pg, i64* %base_scalar)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x i64> @ld1w_d_inbound(<vscale x 2 x i1> %pg, i32* %a) {
+; CHECK-LABEL: ld1w_d_inbound:
+; CHECK: ld1w { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i32* %a to <vscale x 2 x i32>*
+  %base = getelementptr <vscale x 2 x i32>, <vscale x 2 x i32>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 2 x i32>* %base to i32*
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %pg, i32* %base_scalar)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sw_d_inbound(<vscale x 2 x i1> %pg, i32* %a) {
+; CHECK-LABEL: ld1sw_d_inbound:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i32* %a to <vscale x 2 x i32>*
+  %base = getelementptr <vscale x 2 x i32>, <vscale x 2 x i32>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 2 x i32>* %base to i32*
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %pg, i32* %base_scalar)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x double> @ld1d_f64_inbound(<vscale x 2 x i1> %pg, double* %a) {
+; CHECK-LABEL: ld1d_f64_inbound:
+; CHECK: ld1d { z0.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast double* %a to <vscale x 2 x double>*
+  %base = getelementptr <vscale x 2 x double>, <vscale x 2 x double>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 2 x double>* %base to double*
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1> %pg, double* %base_scalar)
+  ret <vscale x 2 x double> %load
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*)
+
+declare <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1>, i8*)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1>, half*)
+
+declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1>, i8*)
+declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1>, i16*)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1>, float*)
+
+declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1>, i8*)
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1>, i16*)
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1>, i32*)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1>, double*)

llvm/test/CodeGen/AArch64/sve-intrinsics-ldst1.ll

-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
-
-;
-; LD1B
-;
-
-define <vscale x 16 x i8> @ld1b_i8(<vscale x 16 x i1> %pred, i8* %addr) {
-; CHECK-LABEL: ld1b_i8:
-; CHECK: ld1b { z0.b }, p0/z, [x0]
-; CHECK-NEXT: ret
-  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pred,
-                                                               i8* %addr)
-  ret <vscale x 16 x i8> %res
-}
-
-;
-; LD1H
-;
-
-define <vscale x 8 x i16> @ld1h_i16(<vscale x 8 x i1> %pred, i16* %addr) {
-; CHECK-LABEL: ld1h_i16:
-; CHECK: ld1h { z0.h }, p0/z, [x0]
-; CHECK-NEXT: ret
-  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> %pred,
-                                                               i16* %addr)
-  ret <vscale x 8 x i16> %res
-}
-
-define <vscale x 8 x half> @ld1h_f16(<vscale x 8 x i1> %pred, half* %addr) {
-; CHECK-LABEL: ld1h_f16:
-; CHECK: ld1h { z0.h }, p0/z, [x0]
-; CHECK-NEXT: ret
-  %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1> %pred,
-                                                                half* %addr)
-  ret <vscale x 8 x half> %res
-}
-
-;
-; LD1W
-;
-
-define <vscale x 4 x i32> @ld1w_i32(<vscale x 4 x i1> %pred, i32* %addr) {
-; CHECK-LABEL: ld1w_i32:
-; CHECK: ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT: ret
-  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %pred,
-                                                               i32* %addr)
-  ret <vscale x 4 x i32> %res
-}
-
-define <vscale x 4 x float> @ld1w_f32(<vscale x 4 x i1> %pred, float* %addr) {
-; CHECK-LABEL: ld1w_f32:
-; CHECK: ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT: ret
-  %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1> %pred,
-                                                                 float* %addr)
-  ret <vscale x 4 x float> %res
-}
-
-;
-; LD1D
-;
-
-define <vscale x 2 x i64> @ld1d_i64(<vscale x 2 x i1> %pred, i64* %addr) {
-; CHECK-LABEL: ld1d_i64:
-; CHECK: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: ret
-  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1> %pred,
-                                                               i64* %addr)
-  ret <vscale x 2 x i64> %res
-}
-
-define <vscale x 2 x double> @ld1d_f64(<vscale x 2 x i1> %pred, double* %addr) {
-; CHECK-LABEL: ld1d_f64:
-; CHECK: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: ret
-  %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1> %pred,
-                                                                  double* %addr)
-  ret <vscale x 2 x double> %res
-}
-
-;
-; ST1B
-;
-
-define void @st1b_i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pred, i8* %addr) {
-; CHECK-LABEL: st1b_i8:
-; CHECK: st1b { z0.b }, p0, [x0]
-; CHECK-NEXT: ret
-  call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data,
-                                          <vscale x 16 x i1> %pred,
-                                          i8* %addr)
-  ret void
-}
-
-;
-; ST1H
-;
-
-define void @st1h_i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, i16* %addr) {
-; CHECK-LABEL: st1h_i16:
-; CHECK: st1h { z0.h }, p0, [x0]
-; CHECK-NEXT: ret
-  call void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16> %data,
-                                          <vscale x 8 x i1> %pred,
-                                          i16* %addr)
-  ret void
-}
-
-define void @st1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, half* %addr) {
-; CHECK-LABEL: st1h_f16:
-; CHECK: st1h { z0.h }, p0, [x0]
-; CHECK-NEXT: ret
-  call void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half> %data,
-                                          <vscale x 8 x i1> %pred,
-                                          half* %addr)
-  ret void
-}
-
-;
-; ST1W
-;
-
-define void @st1w_i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i32* %addr) {
-; CHECK-LABEL: st1w_i32:
-; CHECK: st1w { z0.s }, p0, [x0]
-; CHECK-NEXT: ret
-  call void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32> %data,
-                                          <vscale x 4 x i1> %pred,
-                                          i32* %addr)
-  ret void
-}
-
-define void @st1w_f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pred, float* %addr) {
-; CHECK-LABEL: st1w_f32:
-; CHECK: st1w { z0.s }, p0, [x0]
-; CHECK-NEXT: ret
-  call void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float> %data,
-                                          <vscale x 4 x i1> %pred,
-                                          float* %addr)
-  ret void
-}
-
-;
-; ST1D
-;
-
-define void @st1d_i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i64* %addr) {
-; CHECK-LABEL: st1d_i64:
-; CHECK: st1d { z0.d }, p0, [x0]
-; CHECK-NEXT: ret
-  call void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64> %data,
-                                          <vscale x 2 x i1> %pred,
-                                          i64* %addr)
-  ret void
-}
-
-define void @st1d_f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pred, double* %addr) {
-; CHECK-LABEL: st1d_f64:
-; CHECK: st1d { z0.d }, p0, [x0]
-; CHECK-NEXT: ret
-  call void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double> %data,
-                                          <vscale x 2 x i1> %pred,
-                                          double* %addr)
-  ret void
-}
-
-declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*)
-declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1>, i16*)
-declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1>, i32*)
-declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1>, i64*)
-declare <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1>, half*)
-declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1>, float*)
-declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1>, double*)
-
-declare void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
-declare void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
-declare void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
-declare void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
-declare void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
-declare void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
-declare void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)

llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ST1B
+;
+
+define void @st1b_i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: st1b_i8:
+; CHECK: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data,
+                                          <vscale x 16 x i1> %pred,
+                                          i8* %addr)
+  ret void
+}
+
+define void @st1b_h(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: st1b_h:
+; CHECK: st1b { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 8 x i16> %data to <vscale x 8 x i8>
+  call void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8> %trunc,
+                                         <vscale x 8 x i1> %pred,
+                                         i8* %addr)
+  ret void
+}
+
+define void @st1b_s(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: st1b_s:
+; CHECK: st1b { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i8>
+  call void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8> %trunc,
+                                         <vscale x 4 x i1> %pred,
+                                         i8* %addr)
+  ret void
+}
+
+define void @st1b_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: st1b_d:
+; CHECK: st1b { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i8>
+  call void @llvm.aarch64.sve.st1.nxv2i8(<vscale x 2 x i8> %trunc,
+                                         <vscale x 2 x i1> %pred,
+                                         i8* %addr)
+  ret void
+}
+
+define void @st1b_upper_bound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_upper_bound:
+; CHECK: st1b { z0.b }, p0, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret void
+}
+
+define void @st1b_inbound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_inbound:
+; CHECK: st1b { z0.b }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret void
+}
+
+define void @st1b_lower_bound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_lower_bound:
+; CHECK: st1b { z0.b }, p0, [x0, #-8, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 -8
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret void
+}
+
+define void @st1b_out_of_upper_bound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_out_of_upper_bound:
+; CHECK: rdvl x[[OFFSET:[0-9]+]], #8
+; CHECK: add x[[BASE:[0-9]+]], x0, x[[OFFSET]]
+; CHECK: st1b { z0.b }, p0, [x[[BASE]]]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 8
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret void
+}
+
+define void @st1b_out_of_lower_bound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_out_of_lower_bound:
+; CHECK: rdvl x[[OFFSET:[0-9]+]], #-9
+; CHECK: add x[[BASE:[0-9]+]], x0, x[[OFFSET]]
+; CHECK: st1b { z0.b }, p0, [x[[BASE]]]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+  %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 -9
+  %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+  call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+  ret void
+}
+
+define void @st1b_s_inbound(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_s_inbound:
+; CHECK: st1b { z0.s }, p0, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 4 x i8>*
+  %base = getelementptr <vscale x 4 x i8>, <vscale x 4 x i8>* %base_scalable, i64 7
+  %base_scalar = bitcast <vscale x 4 x i8>* %base to i8*
+  %trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i8>
+  call void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8> %trunc, <vscale x 4 x i1> %pg, i8* %base_scalar)
+  ret void
+}
+
+define void @st1b_h_inbound(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_h_inbound:
+; CHECK: st1b { z0.h }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 8 x i8>*
+  %base = getelementptr <vscale x 8 x i8>, <vscale x 8 x i8>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 8 x i8>* %base to i8*
+  %trunc = trunc <vscale x 8 x i16> %data to <vscale x 8 x i8>
+  call void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8> %trunc, <vscale x 8 x i1> %pg, i8* %base_scalar)
+  ret void
+}
+
+define void @st1b_d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_d_inbound:
+; CHECK: st1b { z0.d }, p0, [x0, #-7, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i8* %a to <vscale x 2 x i8>*
+  %base = getelementptr <vscale x 2 x i8>, <vscale x 2 x i8>* %base_scalable, i64 -7
+  %base_scalar = bitcast <vscale x 2 x i8>* %base to i8*
+  %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i8>
+  call void @llvm.aarch64.sve.st1.nxv2i8(<vscale x 2 x i8> %trunc, <vscale x 2 x i1> %pg, i8* %base_scalar)
+  ret void
+}
+
+;
+; ST1H
+;
+
+define void @st1h_i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: st1h_i16:
+; CHECK: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16> %data,
+                                          <vscale x 8 x i1> %pred,
+                                          i16* %addr)
+  ret void
+}
+
+define void @st1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, half* %addr) {
+; CHECK-LABEL: st1h_f16:
+; CHECK: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half> %data,
+                                          <vscale x 8 x i1> %pred,
+                                          half* %addr)
+  ret void
+}
+
+define void @st1h_s(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: st1h_s:
+; CHECK: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+  call void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16> %trunc,
+                                         <vscale x 4 x i1> %pred,
+                                         i16* %addr)
+  ret void
+}
+
+define void @st1h_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: st1h_d:
+; CHECK: st1h { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16> %trunc,
+                                         <vscale x 2 x i1> %pred,
+                                         i16* %addr)
+  ret void
+}
+
+define void @st1h_inbound(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pg, i16* %a) {
+; CHECK-LABEL: st1h_inbound:
+; CHECK: st1h { z0.h }, p0, [x0, #-1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i16* %a to <vscale x 8 x i16>*
+  %base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %base_scalable, i64 -1
+  %base_scalar = bitcast <vscale x 8 x i16>* %base to i16*
+  call void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pg, i16* %base_scalar)
+  ret void
+}
+
+define void @st1h_f16_inbound(<vscale x 8 x half> %data, <vscale x 8 x i1> %pg, half* %a) {
+; CHECK-LABEL: st1h_f16_inbound:
+; CHECK: st1h { z0.h }, p0, [x0, #-5, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast half* %a to <vscale x 8 x half>*
+  %base = getelementptr <vscale x 8 x half>, <vscale x 8 x half>* %base_scalable, i64 -5
+  %base_scalar = bitcast <vscale x 8 x half>* %base to half*
+  call void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pg, half* %base_scalar)
+  ret void
+}
+
+define void @st1h_s_inbound(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i16* %a) {
+; CHECK-LABEL: st1h_s_inbound:
+; CHECK: st1h { z0.s }, p0, [x0, #2, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i16* %a to <vscale x 4 x i16>*
+  %base = getelementptr <vscale x 4 x i16>, <vscale x 4 x i16>* %base_scalable, i64 2
+  %base_scalar = bitcast <vscale x 4 x i16>* %base to i16*
+  %trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+  call void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16> %trunc, <vscale x 4 x i1> %pg, i16* %base_scalar)
+  ret void
+}
+
+define void @st1h_d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i16* %a) {
+; CHECK-LABEL: st1h_d_inbound:
+; CHECK: st1h { z0.d }, p0, [x0, #-4, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i16* %a to <vscale x 2 x i16>*
+  %base = getelementptr <vscale x 2 x i16>, <vscale x 2 x i16>* %base_scalable, i64 -4
+  %base_scalar = bitcast <vscale x 2 x i16>* %base to i16*
+  %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+  call void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16> %trunc, <vscale x 2 x i1> %pg, i16* %base_scalar)
+  ret void
+}
+
+;
+; ST1W
+;
+
+define void @st1w_i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: st1w_i32:
+; CHECK: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32> %data,
+                                          <vscale x 4 x i1> %pred,
+                                          i32* %addr)
+  ret void
+}
+
+define void @st1w_f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pred, float* %addr) {
+; CHECK-LABEL: st1w_f32:
+; CHECK: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float> %data,
+                                          <vscale x 4 x i1> %pred,
+                                          float* %addr)
+  ret void
+}
+
+define void @st1w_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: st1w_d:
+; CHECK: st1w { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32> %trunc,
+                                         <vscale x 2 x i1> %pred,
+                                         i32* %addr)
+  ret void
+}
+
+define void @st1w_inbound(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i32* %a) {
+; CHECK-LABEL: st1w_inbound:
+; CHECK: st1w { z0.s }, p0, [x0, #6, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i32* %a to <vscale x 4 x i32>*
+  %base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %base_scalable, i64 6
+  %base_scalar = bitcast <vscale x 4 x i32>* %base to i32*
+  call void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i32* %base_scalar)
+  ret void
+}
+
+define void @st1w_f32_inbound(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, float* %a) {
+; CHECK-LABEL: st1w_f32_inbound:
+; CHECK: st1w { z0.s }, p0, [x0, #-1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast float* %a to <vscale x 4 x float>*
+  %base = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %base_scalable, i64 -1
+  %base_scalar = bitcast <vscale x 4 x float>* %base to float*
+  call void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, float* %base_scalar)
+  ret void
+}
+
+define void @st1w_d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i32* %a) {
+; CHECK-LABEL: st1w_d_inbound:
+; CHECK: st1w { z0.d }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i32* %a to <vscale x 2 x i32>*
+  %base = getelementptr <vscale x 2 x i32>, <vscale x 2 x i32>* %base_scalable, i64 1
+  %base_scalar = bitcast <vscale x 2 x i32>* %base to i32*
+  %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+  call void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32> %trunc, <vscale x 2 x i1> %pg, i32* %base_scalar)
+  ret void
+}
+
+;
+; ST1D
+;
+
+define void @st1d_i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i64* %addr) {
+; CHECK-LABEL: st1d_i64:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64> %data,
+                                          <vscale x 2 x i1> %pred,
+                                          i64* %addr)
+  ret void
+}
+
+define void @st1d_f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pred, double* %addr) {
+; CHECK-LABEL: st1d_f64:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double> %data,
+                                          <vscale x 2 x i1> %pred,
+                                          double* %addr)
+  ret void
+}
+
+define void @st1d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %a) {
+; CHECK-LABEL: st1d_inbound:
+; CHECK: st1d { z0.d }, p0, [x0, #5, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast i64* %a to <vscale x 2 x i64>*
+  %base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %base_scalable, i64 5
+  %base_scalar = bitcast <vscale x 2 x i64>* %base to i64*
+  call void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %base_scalar)
+  ret void
+}
+
+define void @st1d_f64_inbound(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %a) {
+; CHECK-LABEL: st1d_f64_inbound:
+; CHECK: st1d { z0.d }, p0, [x0, #-8, mul vl]
+; CHECK-NEXT: ret
+  %base_scalable = bitcast double* %a to <vscale x 2 x double>*
+  %base = getelementptr <vscale x 2 x double>, <vscale x 2 x double>* %base_scalable, i64 -8
+  %base_scalar = bitcast <vscale x 2 x double>* %base to double*
+  call void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %base_scalar)
+  ret void
+}
+
+declare void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
+
+declare void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i1>, i8*)
+declare void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
+declare void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+
+declare void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i1>, i8*)
+declare void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*)
+declare void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
+declare void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
+
+declare void @llvm.aarch64.sve.st1.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i1>, i8*)
+declare void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i16*)
+declare void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*)
+declare void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
+declare void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)