[SVE] Remove custom lowering of scalable vector MGATHER & MSCATTER operations.

[paulwalker-arm]

I can't quite decide if this is better or worse than the current implementation as much of the target specific gather/scatter code remains. My hope is there's value in lowering the intrinsics to stock ISD::MGATHER/ISD::MCATTER operations early, thus forcing all normal gather/scatter operations down the same path.

[sdesmalen]

Nice cleanup.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

552–557

I don't think there's anything SVE specific about these nodes or the corresponding PatFrags. Can you make this a generic node in include/llvm/Target/TargetSelectionDAG.td and possibly move the PatFrags as well? (same question for gather)

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64InstrInfo.td

552–557

It's the SDTCVecEltisVT<1, i1> part that make theses target specific because not all targets have predicate vectors and thus can represent the mask as an ordinary integer vector. That said, looking at masked_ld I can see they just omit element parsing for that parameter so I can do likewise.

[paulwalker-arm]

Restructured PatFrags to maximise code reuse.

[sdesmalen]

Thanks for making these PatFrags largely generic!

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
        AArch64ISD::GLD1_SCALED_MERGE_ZERO},
       {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ true),
        AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO},
   };
   auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
   return AddrModes.find(Key)->second;
 }
 
-unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
-  std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
-      {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
-       AArch64ISD::SST1_PRED},
-      {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ true),
-       AArch64ISD::SST1_UXTW_PRED},
-      {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ false),
-       AArch64ISD::SST1_PRED},
-      {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ true),
-       AArch64ISD::SST1_SXTW_PRED},
-      {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ false),
-       AArch64ISD::SST1_SCALED_PRED},
-      {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ true),
-       AArch64ISD::SST1_UXTW_SCALED_PRED},
-      {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ false),
-       AArch64ISD::SST1_SCALED_PRED},
-      {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ true),
-       AArch64ISD::SST1_SXTW_SCALED_PRED},
-  };
-  auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
-  return AddrModes.find(Key)->second;
-}
-
 unsigned getSignExtendedGatherOpcode(unsigned Opcode) {
   switch (Opcode) {
   default:
     llvm_unreachable("unimplemented opcode");
     return Opcode;
   case AArch64ISD::GLD1_MERGE_ZERO:
     return AArch64ISD::GLD1S_MERGE_ZERO;
   case AArch64ISD::GLD1_IMM_MERGE_ZERO:
     return AArch64ISD::GLD1S_IMM_MERGE_ZERO;
   case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
     return AArch64ISD::GLD1S_UXTW_MERGE_ZERO;
   case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
     return AArch64ISD::GLD1S_SXTW_MERGE_ZERO;
   case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
     return AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
   case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
     return AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO;
   case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
     return AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO;
   }
 }
 
-bool getGatherScatterIndexIsExtended(SDValue Index) {
-  // Ignore non-pointer sized indices.
-  if (Index.getValueType() != MVT::nxv2i64)
-    return false;
-
-  unsigned Opcode = Index.getOpcode();
-  if (Opcode == ISD::SIGN_EXTEND_INREG)
-    return cast<VTSDNode>(Index.getOperand(1))->getVT() == MVT::nxv2i32;
-
-  if (Opcode == ISD::AND) {
-    SDValue Splat = Index.getOperand(1);
-    if (Splat.getOpcode() != ISD::SPLAT_VECTOR)
-      return false;
-    ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Splat.getOperand(0));
-    if (!Mask || Mask->getZExtValue() != 0xFFFFFFFF)
-      return false;
-    return true;
-  }
-
-  return false;
-}
-
-// If the base pointer of a masked gather or scatter is constant, we
-// may be able to swap BasePtr & Index and use the vector + immediate addressing
-// mode, e.g.
-// VECTOR + IMMEDIATE:
-//    getelementptr nullptr, <vscale x N x T> (splat(#x)) + %indices)
-// -> getelementptr #x, <vscale x N x T> %indices
-void selectGatherScatterAddrMode(SDValue &BasePtr, SDValue &Index,
-                                 bool IsScaled, EVT MemVT, unsigned &Opcode,
-                                 bool IsGather, SelectionDAG &DAG) {
-  ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(BasePtr);
-  if (!Offset || IsScaled)
-    return;
-
-  uint64_t OffsetVal = Offset->getZExtValue();
-  unsigned ScalarSizeInBytes = MemVT.getScalarSizeInBits() / 8;
-
-  if (OffsetVal % ScalarSizeInBytes || OffsetVal / ScalarSizeInBytes > 31)
-    return;
-
-  // Immediate is in range
-  Opcode =
-      IsGather ? AArch64ISD::GLD1_IMM_MERGE_ZERO : AArch64ISD::SST1_IMM_PRED;
-  std::swap(BasePtr, Index);
-}
-
 SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
                                             SelectionDAG &DAG) const {
   MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(Op);
 
   SDLoc DL(Op);
   SDValue Chain = MGT->getChain();
   SDValue PassThru = MGT->getPassThru();
   SDValue Mask = MGT->getMask();
if (VT.isFixedLengthVector()) {
     SDValue Result = convertFromScalableVector(DAG, PromotedVT, Load);
     Result = DAG.getNode(ISD::TRUNCATE, DL, DataVT, Result);
     if (VT.isFloatingPoint())
       Result = DAG.getNode(ISD::BITCAST, DL, VT, Result);
 
     return DAG.getMergeValues({Result, Load.getValue(1)}, DL);
   }
 
-  bool IdxNeedsExtend =
-      getGatherScatterIndexIsExtended(Index) ||
-      Index.getSimpleValueType().getVectorElementType() == MVT::i32;
-
-  EVT IndexVT = Index.getSimpleValueType();
-  SDValue InputVT = DAG.getValueType(MemVT);
-
-  // Handle FP data by using an integer gather and casting the result.
-  if (VT.isFloatingPoint())
-    InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
-
-  SDVTList VTs = DAG.getVTList(IndexVT, MVT::Other);
-
-  if (getGatherScatterIndexIsExtended(Index))
-    Index = Index.getOperand(0);
-
-  unsigned Opcode = getGatherVecOpcode(IsScaled, IsSigned, IdxNeedsExtend);
-  selectGatherScatterAddrMode(BasePtr, Index, IsScaled, MemVT, Opcode,
-                              /*isGather=*/true, DAG);
-
-  if (ExtType == ISD::SEXTLOAD)
-    Opcode = getSignExtendedGatherOpcode(Opcode);
-
-  SDValue Ops[] = {Chain, Mask, BasePtr, Index, InputVT};
-  SDValue Result = DAG.getNode(Opcode, DL, VTs, Ops);
-  Chain = Result.getValue(1);
-
-  if (VT.isFloatingPoint())
-    Result = getSVESafeBitCast(VT, Result, DAG);
-
-  return DAG.getMergeValues({Result, Chain}, DL);
+  // Everything else is legal.
+  return Op;
 }
 
 SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op,
                                              SelectionDAG &DAG) const {
   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Op);
 
   SDLoc DL(Op);
   SDValue Chain = MSC->getChain();
if (VT.isFixedLengthVector()) {
     StoreVal = convertToScalableVector(DAG, ContainerVT, StoreVal);
 
     // Emit equivalent scalable vector scatter.
     SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
     return DAG.getMaskedScatter(MSC->getVTList(), MemVT, DL, Ops,
                                 MSC->getMemOperand(), IndexType, Truncating);
   }
 
-  bool NeedsExtend =
-      getGatherScatterIndexIsExtended(Index) ||
-      Index.getSimpleValueType().getVectorElementType() == MVT::i32;
-
-  SDVTList VTs = DAG.getVTList(MVT::Other);
-  SDValue InputVT = DAG.getValueType(MemVT);
-
-  if (VT.isFloatingPoint()) {
-    // Handle FP data by casting the data so an integer scatter can be used.
-    EVT StoreValVT = getPackedSVEVectorVT(VT.getVectorElementCount());
-    StoreVal = getSVESafeBitCast(StoreValVT, StoreVal, DAG);
-    InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
-  }
-
-  if (getGatherScatterIndexIsExtended(Index))
-    Index = Index.getOperand(0);
-
-  unsigned Opcode = getScatterVecOpcode(IsScaled, IsSigned, NeedsExtend);
-  selectGatherScatterAddrMode(BasePtr, Index, IsScaled, MemVT, Opcode,
-                              /*isGather=*/false, DAG);
-
-  SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, InputVT};
-  return DAG.getNode(Opcode, DL, VTs, Ops);
+  // Everything else is legal.
+  return Op;
 }
 
 SDValue AArch64TargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
   SDLoc DL(Op);
   MaskedLoadSDNode *LoadNode = cast<MaskedLoadSDNode>(Op);
   assert(LoadNode && "Expected custom lowering of a masked load node");
   EVT VT = Op->getValueType(0);
 

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 def non_temporal_store :
   PatFrag<(ops node:$val, node:$ptr, node:$pred),
           (masked_st node:$val, node:$ptr, undef, node:$pred), [{
   return !cast<MaskedStoreSDNode>(N)->isTruncatingStore() &&
          cast<MaskedStoreSDNode>(N)->isUnindexed() &&
          cast<MaskedStoreSDNode>(N)->isNonTemporal();
 }]>;
 
+def SDT_SVEMaskedGather : SDTypeProfile<1, 4, [
+  SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVec<2>, SDTCisPtrTy<3>, SDTCisVec<4>,
+  SDTCVecEltisVT<2, i1>, SDTCisSameNumEltsAs<0, 2>, SDTCisSameNumEltsAs<0, 4>
+]>;
+def sve_masked_gather : SDNode<"ISD::MGATHER", SDT_SVEMaskedGather,
+                               [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+
+// masked gather (signed scaled offsets).
+def masked_gather_signed_scaled :
+  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+          (sve_masked_gather node:$def, node:$pred, node:$ptr, node:$idx),[{
+  auto MGN = cast<MaskedGatherSDNode>(N);
+  bool Signed =
+    MGN->isIndexSigned() || MGN->getIndex().getValueType() == MVT::nxv2i64;
+  return Signed && MGN->isIndexScaled();
+}]>;
+// masked gather (signed unscaled offsets).
+def masked_gather_signed_unscaled :
+  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+          (sve_masked_gather node:$def, node:$pred, node:$ptr, node:$idx),[{
+  auto MGN = cast<MaskedGatherSDNode>(N);
+  bool Signed =
+    MGN->isIndexSigned() || MGN->getIndex().getValueType() == MVT::nxv2i64;
+  return Signed && !MGN->isIndexScaled();
+}]>;
+// masked gather (unsigned scaled offsets).
+def masked_gather_unsigned_scaled :
+  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+          (sve_masked_gather node:$def, node:$pred, node:$ptr, node:$idx),[{
+  auto MGN = cast<MaskedGatherSDNode>(N);
+  bool Signed =
+    MGN->isIndexSigned() || MGN->getIndex().getValueType() == MVT::nxv2i64;
+  return !Signed && MGN->isIndexScaled();
+}]>;
+// masked gather (unsigned unscaled offsets).
+def masked_gather_unsigned_unscaled :
+  PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+          (sve_masked_gather node:$def, node:$pred, node:$ptr, node:$idx),[{
+  auto MGN = cast<MaskedGatherSDNode>(N);
+  bool Signed =
+    MGN->isIndexSigned() || MGN->getIndex().getValueType() == MVT::nxv2i64;
+  return !Signed && !MGN->isIndexScaled();
+}]>;
+
+multiclass masked_gather<PatFrag GatherOp> {
+  def nonext_#NAME :
+    PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+            (GatherOp node:$def, node:$pred, node:$ptr, node:$idx), [{
+    return cast<MaskedGatherSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
+  }]>;
+
+  // Any/Zero extending masked gather fragments.
+  def azext_#NAME#_i8 :
+    PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+            (GatherOp node:$def, node:$pred, node:$ptr, node:$idx), [{
+    auto MGN = cast<MaskedGatherSDNode>(N);
+    return (MGN->getExtensionType() == ISD::EXTLOAD ||
+            MGN->getExtensionType() == ISD::ZEXTLOAD) &&
+           MGN->getMemoryVT().getScalarType() == MVT::i8;
+  }]>;
+  def azext_#NAME#_i16 :
+    PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+            (GatherOp node:$def, node:$pred, node:$ptr, node:$idx), [{
+    auto MGN = cast<MaskedGatherSDNode>(N);
+    return (MGN->getExtensionType() == ISD::EXTLOAD ||
+            MGN->getExtensionType() == ISD::ZEXTLOAD) &&
+           MGN->getMemoryVT().getScalarType() == MVT::i16;
+  }]>;
+  def azext_#NAME#_i32 :
+    PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+            (GatherOp node:$def, node:$pred, node:$ptr, node:$idx), [{
+    auto MGN = cast<MaskedGatherSDNode>(N);
+    return (MGN->getExtensionType() == ISD::EXTLOAD ||
+            MGN->getExtensionType() == ISD::ZEXTLOAD) &&
+           MGN->getMemoryVT().getScalarType() == MVT::i32;
+  }]>;
+
+  // Sign extending masked gather fragments.
+  def sext_#NAME#_i8 :
+    PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+            (GatherOp node:$def, node:$pred, node:$ptr, node:$idx), [{
+    auto MGN = cast<MaskedGatherSDNode>(N);
+    return MGN->getExtensionType() == ISD::SEXTLOAD &&
+           MGN->getMemoryVT().getScalarType() == MVT::i8;
+  }]>;
+  def sext_#NAME#_i16 :
+    PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+            (GatherOp node:$def, node:$pred, node:$ptr, node:$idx), [{
+    auto MGN = cast<MaskedGatherSDNode>(N);
+    return MGN->getExtensionType() == ISD::SEXTLOAD &&
+           MGN->getMemoryVT().getScalarType() == MVT::i16;
+  }]>;
+  def sext_#NAME#_i32 :
+    PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
+            (GatherOp node:$def, node:$pred, node:$ptr, node:$idx), [{
+    auto MGN = cast<MaskedGatherSDNode>(N);
+    return MGN->getExtensionType() == ISD::SEXTLOAD &&
+           MGN->getMemoryVT().getScalarType() == MVT::i32;
+  }]>;
+}
+
+defm masked_gather_signed_scaled     : masked_gather<masked_gather_signed_scaled>;
+defm masked_gather_signed_unscaled   : masked_gather<masked_gather_signed_unscaled>;
+defm masked_gather_unsigned_scaled   : masked_gather<masked_gather_unsigned_scaled>;
+defm masked_gather_unsigned_unscaled : masked_gather<masked_gather_unsigned_unscaled>;
+
+def SDT_SVEMaskedScatter : SDTypeProfile<0, 4, [
+  SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>, SDTCisVec<3>,
+  SDTCVecEltisVT<1, i1>, SDTCisSameNumEltsAs<0, 1>, SDTCisSameNumEltsAs<0, 3>
+]>;
+def sve_masked_scatter : SDNode<"ISD::MSCATTER", SDT_SVEMaskedScatter,
+                                  [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
+// masked scatter fragment (signed scaled offsets).
+def masked_scatter_signed_scaled :
+  PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
+          (sve_masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{
+  auto MSN = cast<MaskedScatterSDNode>(N);
+  bool Signed =
+    MSN->isIndexSigned() || MSN->getIndex().getValueType() == MVT::nxv2i64;
+  return Signed && MSN->isIndexScaled();
+}]>;
+// masked scatter fragment (signed unscaled offsets).
+def masked_scatter_signed_unscaled :
+  PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
+          (sve_masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{
+  auto MSN = cast<MaskedScatterSDNode>(N);
+  bool Signed =
+    MSN->isIndexSigned() || MSN->getIndex().getValueType() == MVT::nxv2i64;
+  return Signed && !MSN->isIndexScaled();
+}]>;
+// masked scatter fragment (unsigned scaled offsets).
+def masked_scatter_unsigned_scaled :
+  PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
+          (sve_masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{
+  auto MSN = cast<MaskedScatterSDNode>(N);
+  bool Signed =
+    MSN->isIndexSigned() || MSN->getIndex().getValueType() == MVT::nxv2i64;
+  return !Signed && MSN->isIndexScaled();
+}]>;
+// masked scatter fragment (unsigned unscaled offsets).
+def masked_scatter_unsigned_unscaled :
+  PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
+          (sve_masked_scatter node:$val, node:$pred, node:$ptr, node:$idx), [{
+  auto MSN = cast<MaskedScatterSDNode>(N);
+  bool Signed =
+    MSN->isIndexSigned() || MSN->getIndex().getValueType() == MVT::nxv2i64;
+  return !Signed && !MSN->isIndexScaled();
+}]>;
+
+multiclass masked_scatter<PatFrag ScatterOp> {
+  def nontrunc_#NAME :
+    PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
+            (ScatterOp node:$val, node:$pred, node:$ptr, node:$idx), [{
+    return !cast<MaskedScatterSDNode>(N)->isTruncatingStore();
+  }]>;
+
+  // Truncating masked scatter fragments.
+  def trunc_#NAME#_i8 :
+    PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
+            (ScatterOp node:$val, node:$pred, node:$ptr, node:$idx), [{
+    auto MSN = cast<MaskedScatterSDNode>(N);
+    return MSN->isTruncatingStore() &&
+           MSN->getMemoryVT().getScalarType() == MVT::i8;
+  }]>;
+  def trunc_#NAME#_i16 :
+    PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
+            (ScatterOp node:$val, node:$pred, node:$ptr, node:$idx), [{
+    auto MSN = cast<MaskedScatterSDNode>(N);
+    return MSN->isTruncatingStore() &&
+           MSN->getMemoryVT().getScalarType() == MVT::i16;
+  }]>;
+  def trunc_#NAME#_i32 :
+    PatFrag<(ops node:$val, node:$pred, node:$ptr, node:$idx),
+            (ScatterOp node:$val, node:$pred, node:$ptr, node:$idx), [{
+    auto MSN = cast<MaskedScatterSDNode>(N);
+    return MSN->isTruncatingStore() &&
+           MSN->getMemoryVT().getScalarType() == MVT::i32;
+  }]>;
+}
+
+defm masked_scatter_signed_scaled     : masked_scatter<masked_scatter_signed_scaled>;
+defm masked_scatter_signed_unscaled   : masked_scatter<masked_scatter_signed_unscaled>;
+defm masked_scatter_unsigned_scaled   : masked_scatter<masked_scatter_unsigned_scaled>;
+defm masked_scatter_unsigned_unscaled : masked_scatter<masked_scatter_unsigned_unscaled>;
+
 // top16Zero - answer true if the upper 16 bits of $src are 0, false otherwise
 def top16Zero: PatLeaf<(i32 GPR32:$src), [{
   return SDValue(N,0)->getValueType(0) == MVT::i32 &&
          CurDAG->MaskedValueIsZero(SDValue(N,0), APInt::getHighBitsSet(32, 16));
   }]>;
 
 // top32Zero - answer true if the upper 32 bits of $src are 0, false otherwise
 def top32Zero: PatLeaf<(i64 GPR64:$src), [{

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let Predicates = [HasSVE] in {
   defm GLD1H_D    : sve_mem_64b_gld_sv_32_scaled<0b0110, "ld1h",    AArch64ld1_gather_sxtw_scaled_z,    AArch64ld1_gather_uxtw_scaled_z,    ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
   defm GLDFF1H_D  : sve_mem_64b_gld_sv_32_scaled<0b0111, "ldff1h",  AArch64ldff1_gather_sxtw_scaled_z,  AArch64ldff1_gather_uxtw_scaled_z,  ZPR64ExtSXTW16, ZPR64ExtUXTW16, nxv2i16>;
   defm GLD1SW_D   : sve_mem_64b_gld_sv_32_scaled<0b1000, "ld1sw",   AArch64ld1s_gather_sxtw_scaled_z,   AArch64ld1s_gather_uxtw_scaled_z,   ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
   defm GLDFF1SW_D : sve_mem_64b_gld_sv_32_scaled<0b1001, "ldff1sw", AArch64ldff1s_gather_sxtw_scaled_z, AArch64ldff1s_gather_uxtw_scaled_z, ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
   defm GLD1W_D    : sve_mem_64b_gld_sv_32_scaled<0b1010, "ld1w",    AArch64ld1_gather_sxtw_scaled_z,    AArch64ld1_gather_uxtw_scaled_z,    ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
   defm GLDFF1W_D  : sve_mem_64b_gld_sv_32_scaled<0b1011, "ldff1w",  AArch64ldff1_gather_sxtw_scaled_z,  AArch64ldff1_gather_uxtw_scaled_z,  ZPR64ExtSXTW32, ZPR64ExtUXTW32, nxv2i32>;
   defm GLD1D      : sve_mem_64b_gld_sv_32_scaled<0b1110, "ld1d",    AArch64ld1_gather_sxtw_scaled_z,    AArch64ld1_gather_uxtw_scaled_z,    ZPR64ExtSXTW64, ZPR64ExtUXTW64, nxv2i64>;
   defm GLDFF1D    : sve_mem_64b_gld_sv_32_scaled<0b1111, "ldff1d",  AArch64ldff1_gather_sxtw_scaled_z,  AArch64ldff1_gather_uxtw_scaled_z,  ZPR64ExtSXTW64, ZPR64ExtUXTW64, nxv2i64>;
+
+  multiclass sve_masked_gather_x2_scaled<ValueType Ty, SDPatternOperator Load, string Inst> {
+    // base + vector of scaled offsets
+    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (nxv2i64 ZPR:$offs))),
+              (!cast<Instruction>(Inst # _SCALED) PPR:$gp, GPR64:$base, ZPR:$offs)>;
+    // base + vector of signed 32bit scaled offsets
+    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (sext_inreg (nxv2i64 ZPR:$offs), nxv2i32))),
+              (!cast<Instruction>(Inst # _SXTW_SCALED) PPR:$gp, GPR64:$base, ZPR:$offs)>;
+    // base + vector of unsigned 32bit scaled offsets
+    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (and (nxv2i64 ZPR:$offs), (nxv2i64 (AArch64dup (i64 0xFFFFFFFF)))))),
+              (!cast<Instruction>(Inst # _UXTW_SCALED) PPR:$gp, GPR64:$base, ZPR:$offs)>;
+  }
+
+  multiclass sve_masked_gather_x2_unscaled<ValueType Ty, SDPatternOperator Load, string Inst, Operand ImmTy> {
+    // vector of pointers + immediate offset (includes zero)
+    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), (i64 ImmTy:$imm), (nxv2i64 ZPR:$ptrs))),
+              (!cast<Instruction>(Inst # _IMM) PPR:$gp, ZPR:$ptrs, ImmTy:$imm)>;
+    // base + vector of offsets
+    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (nxv2i64 ZPR:$offs))),
+              (!cast<Instruction>(Inst) PPR:$gp, GPR64:$base, ZPR:$offs)>;
+    // base + vector of signed 32bit offsets
+    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (sext_inreg (nxv2i64 ZPR:$offs), nxv2i32))),
+              (!cast<Instruction>(Inst # _SXTW) PPR:$gp, GPR64:$base, ZPR:$offs)>;
+    // base + vector of unsigned 32bit offsets
+    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (and (nxv2i64 ZPR:$offs), (nxv2i64 (AArch64dup (i64 0xFFFFFFFF)))))),
+              (!cast<Instruction>(Inst # _UXTW) PPR:$gp, GPR64:$base, ZPR:$offs)>;
+  }
+
+  multiclass sve_masked_gather_x4<ValueType Ty, SDPatternOperator Load, Instruction Inst> {
+    def : Pat<(Ty (Load (SVEDup0Undef), (nxv4i1 PPR:$gp), GPR64:$base, (nxv4i32 ZPR:$offs))),
+              (Inst PPR:$gp, GPR64:$base, ZPR:$offs)>;
+  }
+
+  defm : sve_masked_gather_x2_scaled<nxv2i64,  azext_masked_gather_signed_scaled_i16, "GLD1H_D">;
+  defm : sve_masked_gather_x2_scaled<nxv2i64,  sext_masked_gather_signed_scaled_i16,  "GLD1SH_D">;
+  defm : sve_masked_gather_x2_scaled<nxv2i64,  azext_masked_gather_signed_scaled_i32, "GLD1W_D">;
+  defm : sve_masked_gather_x2_scaled<nxv2i64,  sext_masked_gather_signed_scaled_i32,  "GLD1SW_D">;
+  defm : sve_masked_gather_x2_scaled<nxv2i64,  nonext_masked_gather_signed_scaled,    "GLD1D">;
+  defm : sve_masked_gather_x2_scaled<nxv2f16,  nonext_masked_gather_signed_scaled,    "GLD1H_D">;
+  defm : sve_masked_gather_x2_scaled<nxv2f32,  nonext_masked_gather_signed_scaled,    "GLD1W_D">;
+  defm : sve_masked_gather_x2_scaled<nxv2f64,  nonext_masked_gather_signed_scaled,    "GLD1D">;
+  defm : sve_masked_gather_x2_scaled<nxv2bf16, nonext_masked_gather_signed_scaled,    "GLD1H_D">;
+
+  defm : sve_masked_gather_x2_unscaled<nxv2i64,  azext_masked_gather_signed_unscaled_i8,  "GLD1B_D" , imm0_31>;
+  defm : sve_masked_gather_x2_unscaled<nxv2i64,  sext_masked_gather_signed_unscaled_i8,   "GLD1SB_D", imm0_31>;
+  defm : sve_masked_gather_x2_unscaled<nxv2i64,  azext_masked_gather_signed_unscaled_i16, "GLD1H_D",  uimm5s2>;
+  defm : sve_masked_gather_x2_unscaled<nxv2i64,  sext_masked_gather_signed_unscaled_i16,  "GLD1SH_D", uimm5s2>;
+  defm : sve_masked_gather_x2_unscaled<nxv2i64,  azext_masked_gather_signed_unscaled_i32, "GLD1W_D",  uimm5s4>;
+  defm : sve_masked_gather_x2_unscaled<nxv2i64,  sext_masked_gather_signed_unscaled_i32,  "GLD1SW_D", uimm5s4>;
+  defm : sve_masked_gather_x2_unscaled<nxv2i64,  nonext_masked_gather_signed_unscaled,    "GLD1D",    uimm5s8>;
+  defm : sve_masked_gather_x2_unscaled<nxv2f16,  nonext_masked_gather_signed_unscaled,    "GLD1H_D",  uimm5s2>;
+  defm : sve_masked_gather_x2_unscaled<nxv2f32,  nonext_masked_gather_signed_unscaled,    "GLD1W_D",  uimm5s4>;
+  defm : sve_masked_gather_x2_unscaled<nxv2f64,  nonext_masked_gather_signed_unscaled,    "GLD1D",    uimm5s8>;
+  defm : sve_masked_gather_x2_unscaled<nxv2bf16, nonext_masked_gather_signed_unscaled,    "GLD1H_D",  uimm5s2>;
+
+  defm : sve_masked_gather_x4<nxv4i32,  azext_masked_gather_signed_scaled_i16, GLD1H_S_SXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4i32,  sext_masked_gather_signed_scaled_i16,  GLD1SH_S_SXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4i32,  nonext_masked_gather_signed_scaled,    GLD1W_SXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4f16,  nonext_masked_gather_signed_scaled,    GLD1H_S_SXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4f32,  nonext_masked_gather_signed_scaled,    GLD1W_SXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4bf16, nonext_masked_gather_signed_scaled,    GLD1H_S_SXTW_SCALED>;
+
+  defm : sve_masked_gather_x4<nxv4i32,  azext_masked_gather_signed_unscaled_i8,  GLD1B_S_SXTW>;
+  defm : sve_masked_gather_x4<nxv4i32,  sext_masked_gather_signed_unscaled_i8,   GLD1SB_S_SXTW>;
+  defm : sve_masked_gather_x4<nxv4i32,  azext_masked_gather_signed_unscaled_i16, GLD1H_S_SXTW>;
+  defm : sve_masked_gather_x4<nxv4i32,  sext_masked_gather_signed_unscaled_i16,  GLD1SH_S_SXTW>;
+  defm : sve_masked_gather_x4<nxv4i32,  nonext_masked_gather_signed_unscaled,    GLD1W_SXTW>;
+  defm : sve_masked_gather_x4<nxv4f16,  nonext_masked_gather_signed_unscaled,    GLD1H_S_SXTW>;
+  defm : sve_masked_gather_x4<nxv4f32,  nonext_masked_gather_signed_unscaled,    GLD1W_SXTW>;
+  defm : sve_masked_gather_x4<nxv4bf16, nonext_masked_gather_signed_unscaled,    GLD1H_S_SXTW>;
+
+  defm : sve_masked_gather_x4<nxv4i32,  azext_masked_gather_unsigned_scaled_i16, GLD1H_S_UXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4i32,  sext_masked_gather_unsigned_scaled_i16,  GLD1SH_S_UXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4i32,  nonext_masked_gather_unsigned_scaled,    GLD1W_UXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4f16,  nonext_masked_gather_unsigned_scaled,    GLD1H_S_UXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4f32,  nonext_masked_gather_unsigned_scaled,    GLD1W_UXTW_SCALED>;
+  defm : sve_masked_gather_x4<nxv4bf16, nonext_masked_gather_unsigned_scaled,    GLD1H_S_UXTW_SCALED>;
+
+  defm : sve_masked_gather_x4<nxv4i32,  azext_masked_gather_unsigned_unscaled_i8,  GLD1B_S_UXTW>;
+  defm : sve_masked_gather_x4<nxv4i32,  sext_masked_gather_unsigned_unscaled_i8,   GLD1SB_S_UXTW>;
+  defm : sve_masked_gather_x4<nxv4i32,  azext_masked_gather_unsigned_unscaled_i16, GLD1H_S_UXTW>;
+  defm : sve_masked_gather_x4<nxv4i32,  sext_masked_gather_unsigned_unscaled_i16,  GLD1SH_S_UXTW>;
+  defm : sve_masked_gather_x4<nxv4i32,  nonext_masked_gather_unsigned_unscaled,    GLD1W_UXTW>;
+  defm : sve_masked_gather_x4<nxv4f16,  nonext_masked_gather_unsigned_unscaled,    GLD1H_S_UXTW>;
+  defm : sve_masked_gather_x4<nxv4f32,  nonext_masked_gather_unsigned_unscaled,    GLD1W_UXTW>;
+  defm : sve_masked_gather_x4<nxv4bf16, nonext_masked_gather_unsigned_unscaled,    GLD1H_S_UXTW>;
 } // End HasSVE
 
 let Predicates = [HasSVEorStreamingSVE] in {
   // Non-temporal contiguous loads (register + immediate)
   defm LDNT1B_ZRI : sve_mem_cldnt_si<0b00, "ldnt1b", Z_b, ZPR8>;
   defm LDNT1H_ZRI : sve_mem_cldnt_si<0b01, "ldnt1h", Z_h, ZPR16>;
   defm LDNT1W_ZRI : sve_mem_cldnt_si<0b10, "ldnt1w", Z_s, ZPR32>;
   defm LDNT1D_ZRI : sve_mem_cldnt_si<0b11, "ldnt1d", Z_d, ZPR64>;
let Predicates = [HasSVE] in {
   defm SST1W_D : sve_mem_sst_sv_64_unscaled<0b10, "st1w", AArch64st1_scatter, nxv2i32>;
   defm SST1D   : sve_mem_sst_sv_64_unscaled<0b11, "st1d", AArch64st1_scatter, nxv2i64>;
 
   // Scatters using scaled 64-bit offsets, e.g.
   //    st1h z0.d, p0, [x0, z0.d, lsl #1]
   defm SST1H_D : sve_mem_sst_sv_64_scaled<0b01, "st1h", AArch64st1_scatter_scaled, ZPR64ExtLSL16, nxv2i16>;
   defm SST1W_D : sve_mem_sst_sv_64_scaled<0b10, "st1w", AArch64st1_scatter_scaled, ZPR64ExtLSL32, nxv2i32>;
   defm SST1D   : sve_mem_sst_sv_64_scaled<0b11, "st1d", AArch64st1_scatter_scaled, ZPR64ExtLSL64, nxv2i64>;
+
+  multiclass sve_masked_scatter_x2_scaled<ValueType Ty, SDPatternOperator Store, string Inst> {
+    // base + vector of scaled offsets
+    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (nxv2i64 ZPR:$offs)),
+              (!cast<Instruction>(Inst # _SCALED) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
+    // base + vector of signed 32bit scaled offsets
+    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (sext_inreg (nxv2i64 ZPR:$offs), nxv2i32)),
+              (!cast<Instruction>(Inst # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
+    // base + vector of unsigned 32bit scaled offsets
+    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (and (nxv2i64 ZPR:$offs), (nxv2i64 (AArch64dup (i64 0xFFFFFFFF))))),
+              (!cast<Instruction>(Inst # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
+  }
+
+  multiclass sve_masked_scatter_x2_unscaled<ValueType Ty, SDPatternOperator Store, string Inst, Operand ImmTy> {
+    // vector of pointers + immediate offset (includes zero)
+    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), (i64 ImmTy:$imm), (nxv2i64 ZPR:$ptrs)),
+              (!cast<Instruction>(Inst # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, ImmTy:$imm)>;
+    // base + vector of offsets
+    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (nxv2i64 ZPR:$offs)),
+              (!cast<Instruction>(Inst) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
+    // base + vector of signed 32bit offsets
+    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (sext_inreg (nxv2i64 ZPR:$offs), nxv2i32)),
+              (!cast<Instruction>(Inst # _SXTW) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
+    // base + vector of unsigned 32bit offsets
+    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (and (nxv2i64 ZPR:$offs), (nxv2i64 (AArch64dup (i64 0xFFFFFFFF))))),
+              (!cast<Instruction>(Inst # _UXTW) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
+  }
+
+  multiclass sve_masked_scatter_x4<ValueType Ty, SDPatternOperator Store, Instruction Inst> {
+    def : Pat<(Store (Ty ZPR:$data), (nxv4i1 PPR:$gp), GPR64:$base, (nxv4i32 ZPR:$offs)),
+              (Inst ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
+  }
+
+  defm : sve_masked_scatter_x2_scaled<nxv2i64,  trunc_masked_scatter_signed_scaled_i16, "SST1H_D">;
+  defm : sve_masked_scatter_x2_scaled<nxv2i64,  trunc_masked_scatter_signed_scaled_i32, "SST1W_D">;
+  defm : sve_masked_scatter_x2_scaled<nxv2i64,  nontrunc_masked_scatter_signed_scaled,  "SST1D">;
+  defm : sve_masked_scatter_x2_scaled<nxv2f16,  nontrunc_masked_scatter_signed_scaled,  "SST1H_D">;
+  defm : sve_masked_scatter_x2_scaled<nxv2f32,  nontrunc_masked_scatter_signed_scaled,  "SST1W_D">;
+  defm : sve_masked_scatter_x2_scaled<nxv2f64,  nontrunc_masked_scatter_signed_scaled,  "SST1D">;
+  defm : sve_masked_scatter_x2_scaled<nxv2bf16, nontrunc_masked_scatter_signed_scaled,  "SST1H_D">;
+
+  defm : sve_masked_scatter_x2_unscaled<nxv2i64,  trunc_masked_scatter_signed_unscaled_i8,  "SST1B_D" , imm0_31>;
+  defm : sve_masked_scatter_x2_unscaled<nxv2i64,  trunc_masked_scatter_signed_unscaled_i16, "SST1H_D",  uimm5s2>;
+  defm : sve_masked_scatter_x2_unscaled<nxv2i64,  trunc_masked_scatter_signed_unscaled_i32, "SST1W_D",  uimm5s4>;
+  defm : sve_masked_scatter_x2_unscaled<nxv2i64,  nontrunc_masked_scatter_signed_unscaled,  "SST1D",    uimm5s8>;
+  defm : sve_masked_scatter_x2_unscaled<nxv2f16,  nontrunc_masked_scatter_signed_unscaled,  "SST1H_D",  uimm5s2>;
+  defm : sve_masked_scatter_x2_unscaled<nxv2f32,  nontrunc_masked_scatter_signed_unscaled,  "SST1W_D",  uimm5s4>;
+  defm : sve_masked_scatter_x2_unscaled<nxv2f64,  nontrunc_masked_scatter_signed_unscaled,  "SST1D",    uimm5s8>;
+  defm : sve_masked_scatter_x2_unscaled<nxv2bf16, nontrunc_masked_scatter_signed_unscaled,  "SST1H_D",  uimm5s2>;
+
+  defm : sve_masked_scatter_x4<nxv4i32,  trunc_masked_scatter_signed_scaled_i16, SST1H_S_SXTW_SCALED>;
+  defm : sve_masked_scatter_x4<nxv4i32,  nontrunc_masked_scatter_signed_scaled,  SST1W_SXTW_SCALED>;
+  defm : sve_masked_scatter_x4<nxv4f16,  nontrunc_masked_scatter_signed_scaled,  SST1H_S_SXTW_SCALED>;
+  defm : sve_masked_scatter_x4<nxv4f32,  nontrunc_masked_scatter_signed_scaled,  SST1W_SXTW_SCALED>;
+  defm : sve_masked_scatter_x4<nxv4bf16, nontrunc_masked_scatter_signed_scaled,  SST1H_S_SXTW_SCALED>;
+
+  defm : sve_masked_scatter_x4<nxv4i32,  trunc_masked_scatter_signed_unscaled_i8,  SST1B_S_SXTW>;
+  defm : sve_masked_scatter_x4<nxv4i32,  trunc_masked_scatter_signed_unscaled_i16, SST1H_S_SXTW>;
+  defm : sve_masked_scatter_x4<nxv4i32,  nontrunc_masked_scatter_signed_unscaled,  SST1W_SXTW>;
+  defm : sve_masked_scatter_x4<nxv4f16,  nontrunc_masked_scatter_signed_unscaled,  SST1H_S_SXTW>;
+  defm : sve_masked_scatter_x4<nxv4f32,  nontrunc_masked_scatter_signed_unscaled,  SST1W_SXTW>;
+  defm : sve_masked_scatter_x4<nxv4bf16, nontrunc_masked_scatter_signed_unscaled,  SST1H_S_SXTW>;
+
+  defm : sve_masked_scatter_x4<nxv4i32,  trunc_masked_scatter_unsigned_scaled_i16, SST1H_S_UXTW_SCALED>;
+  defm : sve_masked_scatter_x4<nxv4i32,  nontrunc_masked_scatter_unsigned_scaled,  SST1W_UXTW_SCALED>;
+  defm : sve_masked_scatter_x4<nxv4f16,  nontrunc_masked_scatter_unsigned_scaled,  SST1H_S_UXTW_SCALED>;
+  defm : sve_masked_scatter_x4<nxv4f32,  nontrunc_masked_scatter_unsigned_scaled,  SST1W_UXTW_SCALED>;
+  defm : sve_masked_scatter_x4<nxv4bf16, nontrunc_masked_scatter_unsigned_scaled,  SST1H_S_UXTW_SCALED>;
+
+  defm : sve_masked_scatter_x4<nxv4i32,  trunc_masked_scatter_unsigned_unscaled_i8,  SST1B_S_UXTW>;
+  defm : sve_masked_scatter_x4<nxv4i32,  trunc_masked_scatter_unsigned_unscaled_i16, SST1H_S_UXTW>;
+  defm : sve_masked_scatter_x4<nxv4i32,  nontrunc_masked_scatter_unsigned_unscaled,  SST1W_UXTW>;
+  defm : sve_masked_scatter_x4<nxv4f16,  nontrunc_masked_scatter_unsigned_unscaled,  SST1H_S_UXTW>;
+  defm : sve_masked_scatter_x4<nxv4f32,  nontrunc_masked_scatter_unsigned_unscaled,  SST1W_UXTW>;
+  defm : sve_masked_scatter_x4<nxv4bf16, nontrunc_masked_scatter_unsigned_unscaled,  SST1H_S_UXTW>;
 } // End HasSVE
 
 let Predicates = [HasSVEorStreamingSVE] in {
   // ST(2|3|4) structured stores (register + immediate)
   defm ST2B_IMM : sve_mem_est_si<0b00, 0b01, ZZ_b,   "st2b", simm4s2>;
   defm ST3B_IMM : sve_mem_est_si<0b00, 0b10, ZZZ_b,  "st3b", simm4s3>;
   defm ST4B_IMM : sve_mem_est_si<0b00, 0b11, ZZZZ_b, "st4b", simm4s4>;
   defm ST2H_IMM : sve_mem_est_si<0b01, 0b01, ZZ_h,   "st2h", simm4s2>;