[SVE][CodeGen] Improve codegen of scalable masked scatters

[kmclaughlin]

Summary

If the scatter store is able to perform the sign/zero extend of
its index, this is folded into the instruction with refineIndexType().
Additionally, refineUniformBase() will return the base pointer and index
from an add + splat_vector.

[sdesmalen]

The indentation seems off for some of the changes, could you run the patch through clang-format?

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

9408

This should probably use SelectionDAG::getSplatValue() instead, because there are multiple ways of representing splats (SPLAT_VECTOR is only used for scalable vectors at the moment).

[kmclaughlin]

• Replaced check for ISD::SPLAT_VECTOR opcode in refineUniformBase with SelectionDAG::getSplatVector()
• Applied changes suggested by clang-format

[sdesmalen]

Nice improvements to the test :) LGTM!

[yubing]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

9421

If Index is a undef, this line will assertion fail.

[kmclaughlin]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

9421

Hi @yubing, thank you for posting a fix for this in D92548!

llvm/include/llvm/CodeGen/TargetLowering.h

#include "llvm/IR/ConstrainedOps.def"
 
   /// Return true if the specified indexed load is legal on this target.
   bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const {
     return VT.isSimple() &&
            (getIndexedMaskedStoreAction(IdxMode, VT.getSimpleVT()) == Legal ||
             getIndexedMaskedStoreAction(IdxMode, VT.getSimpleVT()) == Custom);
   }
 
+  // Returns true if VT is a legal index type for masked gathers/scatters
+  // on this target
+  virtual bool shouldRemoveExtendFromGSIndex(EVT VT) const { return false; }
+
   /// Return how the condition code should be treated: either it is legal, needs
   /// to be expanded to some other code sequence, or the target has a custom
   /// expander for it.
   LegalizeAction
   getCondCodeAction(ISD::CondCode CC, MVT VT) const {
     assert((unsigned)CC < array_lengthof(CondCodeActions) &&
            ((unsigned)VT.SimpleTy >> 3) < array_lengthof(CondCodeActions[0]) &&
            "Table isn't big enough!");

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

assert(TopHalf && BottomHalf &&
          "One half of the selector was all UNDEFs and the other was all the "
          "same value. This should have been addressed before this function.");
   return DAG.getNode(
       ISD::CONCAT_VECTORS, DL, VT,
       BottomHalf->isNullValue() ? RHS->getOperand(0) : LHS->getOperand(0),
       TopHalf->isNullValue() ? RHS->getOperand(1) : LHS->getOperand(1));
 }
 
+bool refineUniformBase(SDValue &BasePtr, SDValue &Index, SelectionDAG &DAG) {
+  if (!isNullConstant(BasePtr) || Index.getOpcode() != ISD::ADD)
+    return false;
+
+  // For now we check only the LHS of the add.
+  SDValue LHS = Index.getOperand(0);
+  SDValue SplatVal = DAG.getSplatValue(LHS);
+  if (!SplatVal)
+    return false;
+
+  BasePtr = SplatVal;
+  Index = Index.getOperand(1);
+  return true;
+}
+
+// Fold sext/zext of index into index type.
+bool refineIndexType(MaskedScatterSDNode *MSC, SDValue &Index, bool Scaled,
+                     SelectionDAG &DAG) {
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  SDValue Op = Index.getOperand(0);
+
+  if (Index.getOpcode() == ISD::ZERO_EXTEND) {
+    MSC->setIndexType(Scaled ? ISD::UNSIGNED_SCALED : ISD::UNSIGNED_UNSCALED);
+    if (TLI.shouldRemoveExtendFromGSIndex(Op.getValueType())) {
+      Index = Op;
+      return true;
+    }
+  }
+
+  if (Index.getOpcode() == ISD::SIGN_EXTEND) {
+    MSC->setIndexType(Scaled ? ISD::SIGNED_SCALED : ISD::SIGNED_UNSCALED);
+    if (TLI.shouldRemoveExtendFromGSIndex(Op.getValueType())) {
+      Index = Op;
+      return true;
+    }
+  }
+
+  return false;
+}
+
 SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
   SDValue Mask = MSC->getMask();
   SDValue Chain = MSC->getChain();
+  SDValue Index = MSC->getIndex();
+  SDValue Scale = MSC->getScale();
+  SDValue StoreVal = MSC->getValue();
+  SDValue BasePtr = MSC->getBasePtr();
   SDLoc DL(N);
 
   // Zap scatters with a zero mask.
   if (ISD::isBuildVectorAllZeros(Mask.getNode()))
     return Chain;
 
+  if (refineUniformBase(BasePtr, Index, DAG)) {
+    SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
+    return DAG.getMaskedScatter(
+        DAG.getVTList(MVT::Other), StoreVal.getValueType(), DL, Ops,
+        MSC->getMemOperand(), MSC->getIndexType(), MSC->isTruncatingStore());
+  }
+
+  if (refineIndexType(MSC, Index, MSC->isIndexScaled(), DAG)) {
+    SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
+    return DAG.getMaskedScatter(
+        DAG.getVTList(MVT::Other), StoreVal.getValueType(), DL, Ops,
+        MSC->getMemOperand(), MSC->getIndexType(), MSC->isTruncatingStore());
+  }
+
   return SDValue();
 }
 
 SDValue DAGCombiner::visitMSTORE(SDNode *N) {
   MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
   SDValue Mask = MST->getMask();
   SDValue Chain = MST->getChain();
   SDLoc DL(N);

llvm/lib/Target/AArch64/AArch64ISelLowering.h

unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
     if (ConstraintCode == "Q")
       return InlineAsm::Constraint_Q;
     // FIXME: clang has code for 'Ump', 'Utf', 'Usa', and 'Ush' but these are
     //        followed by llvm_unreachable so we'll leave them unimplemented in
     //        the backend for now.
     return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
   }
 
+  bool shouldRemoveExtendFromGSIndex(EVT VT) const override;
   bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
   bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
   bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
   bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset,
                               ISD::MemIndexedMode &AM, bool &IsInc,
                               SelectionDAG &DAG) const;
   bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
                                  ISD::MemIndexedMode &AM,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   }
   case Intrinsic::aarch64_neon_sabd: {
     return DAG.getNode(AArch64ISD::SABD, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   }
   }
 }
 
+bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
+  if (VT.getVectorElementType() == MVT::i32 &&
+      VT.getVectorElementCount().getKnownMinValue() >= 4)
+    return true;
+
+  return false;
+}
+
 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
   return ExtVal.getValueType().isScalableVector();
 }
 
 unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
   std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
        AArch64ISD::SST1_PRED},
if (VT.isFloatingPoint()) {
     auto ScalarIntVT =
         MVT::getIntegerVT(AArch64::SVEBitsPerBlock / EC.getKnownMinValue());
     StoreVal = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL,
                            MVT::getVectorVT(ScalarIntVT, EC), StoreVal);
 
     InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
   }
 
-  if (getScatterIndexIsExtended(Index)) {
-    if (Index.getOpcode() == ISD::AND)
-      IsSigned = false;
+  if (getScatterIndexIsExtended(Index))
     Index = Index.getOperand(0);
-  }
 
   SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, InputVT};
   return DAG.getNode(getScatterVecOpcode(IsScaled, IsSigned, NeedsExtend), DL,
                      VTs, Ops);
 }
 
 // Custom lower trunc store for v4i8 vectors, since it is promoted to v4i16.
 static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,

llvm/test/CodeGen/AArch64/sve-masked-scatter-32b-scaled.ll

 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; scaled packed 32-bit offset
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 define void @masked_scatter_nxv4i16_sext(<vscale x 4 x i16> %data, i16* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i16_sext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x0, z1.d, sxtw #1]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z2.d, sxtw #1]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, sxtw #1]
 ; CHECK-NEXT:    ret
   %ext = sext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr i16, i16* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4i16(<vscale x 4 x i16> %data, <vscale x 4 x i16*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4i32_sext(<vscale x 4 x i32> %data, i32* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i32_sext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1w { z3.d }, p2, [x0, z1.d, sxtw #2]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z2.d, sxtw #2]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, z1.s, sxtw #2]
 ; CHECK-NEXT:    ret
   %ext = sext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr i32, i32* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x i32*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4f16_sext(<vscale x 4 x half> %data, half* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4f16_sext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x0, z1.d, sxtw #1]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z2.d, sxtw #1]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, sxtw #1]
 ; CHECK-NEXT:    ret
   %ext = sext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr half, half* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x half*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4bf16_sext(<vscale x 4 x bfloat> %data, bfloat* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv4bf16_sext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x0, z1.d, sxtw #1]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z2.d, sxtw #1]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, sxtw #1]
 ; CHECK-NEXT:    ret
   %ext = sext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr bfloat, bfloat* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4bf16(<vscale x 4 x bfloat> %data, <vscale x 4 x bfloat*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4f32_sext(<vscale x 4 x float> %data, float* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv4f32_sext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1w { z3.d }, p2, [x0, z1.d, sxtw #2]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z2.d, sxtw #2]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, z1.s, sxtw #2]
 ; CHECK-NEXT:    ret
   %ext = sext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr float, float* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x float*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4i16_zext(<vscale x 4 x i16> %data, i16* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i16_zext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x0, z1.d, uxtw #1]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z2.d, uxtw #1]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, uxtw #1]
 ; CHECK-NEXT:    ret
   %ext = zext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr i16, i16* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4i16(<vscale x 4 x i16> %data, <vscale x 4 x i16*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4i32_zext(<vscale x 4 x i32> %data, i32* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i32_zext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1w { z3.d }, p2, [x0, z1.d, uxtw #2]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z2.d, uxtw #2]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, z1.s, uxtw #2]
 ; CHECK-NEXT:    ret
   %ext = zext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr i32, i32* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x i32*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4f16_zext(<vscale x 4 x half> %data, half* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4f16_zext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x0, z1.d, uxtw #1]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z2.d, uxtw #1]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, uxtw #1]
 ; CHECK-NEXT:    ret
   %ext = zext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr half, half* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x half*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4bf16_zext(<vscale x 4 x bfloat> %data, bfloat* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv4bf16_zext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x0, z1.d, uxtw #1]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z2.d, uxtw #1]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, uxtw #1]
 ; CHECK-NEXT:    ret
   %ext = zext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr bfloat, bfloat* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4bf16(<vscale x 4 x bfloat> %data, <vscale x 4 x bfloat*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4f32_zext(<vscale x 4 x float> %data, float* %base, <vscale x 4 x i32> %indexes, <vscale x 4 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv4f32_zext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    uunpkhi z2.d, z1.s
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1w { z3.d }, p2, [x0, z1.d, uxtw #2]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z2.d, uxtw #2]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, z1.s, uxtw #2]
 ; CHECK-NEXT:    ret
   %ext = zext <vscale x 4 x i32> %indexes to <vscale x 4 x i64>
   %ptrs = getelementptr float, float* %base, <vscale x 4 x i64> %ext
   call void @llvm.masked.scatter.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x float*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 declare void @llvm.masked.scatter.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half*>, i32, <vscale x 2 x i1>)

llvm/test/CodeGen/AArch64/sve-masked-scatter-32b-unscaled.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; unscaled unpacked 32-bit offsets
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 define void @masked_scatter_nxv2i8_sext_offsets(<vscale x 2 x i8> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i8_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1b { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1b { z0.d }, p0, [x0, z1.d, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i8*>
   call void @llvm.masked.scatter.nxv2i8(<vscale x 2 x i8> %data, <vscale x 2 x i8*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i16_sext_offsets(<vscale x 2 x i16> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i16_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
   call void @llvm.masked.scatter.nxv2i16(<vscale x 2 x i16> %data, <vscale x 2 x i16*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i32_sext_offsets(<vscale x 2 x i32> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i32_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z1.d, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
   call void @llvm.masked.scatter.nxv2i32(<vscale x 2 x i32> %data, <vscale x 2 x i32*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i64_sext_offsets(<vscale x 2 x i64> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i64_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, z1.d, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i64*>
   call void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x i64*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f16_sext_offsets(<vscale x 2 x half> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2f16_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x half*>
   call void @llvm.masked.scatter.nxv2f16(<vscale x 2 x half> %data, <vscale x 2 x half*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2bf16_sext_offsets(<vscale x 2 x bfloat> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv2bf16_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x bfloat*>
   call void @llvm.masked.scatter.nxv2bf16(<vscale x 2 x bfloat> %data, <vscale x 2 x bfloat*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f32_sext_offsets(<vscale x 2 x float> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2f32_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z1.d, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x float*>
   call void @llvm.masked.scatter.nxv2f32(<vscale x 2 x float> %data, <vscale x 2 x float*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f64_sext_offsets(<vscale x 2 x double> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2f64_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p1.d
-; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, z1.d, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x double*>
   call void @llvm.masked.scatter.nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x double*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i8_zext_offsets(<vscale x 2 x i8> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i8_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1b { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1b { z0.d }, p0, [x0, z1.d, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i8*>
   call void @llvm.masked.scatter.nxv2i8(<vscale x 2 x i8> %data, <vscale x 2 x i8*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i16_zext_offsets(<vscale x 2 x i16> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i16_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
   call void @llvm.masked.scatter.nxv2i16(<vscale x 2 x i16> %data, <vscale x 2 x i16*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i32_zext_offsets(<vscale x 2 x i32> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i32_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z1.d, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
   call void @llvm.masked.scatter.nxv2i32(<vscale x 2 x i32> %data, <vscale x 2 x i32*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i64_zext_offsets(<vscale x 2 x i64> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i64_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, z1.d, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i64*>
   call void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x i64*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f16_zext_offsets(<vscale x 2 x half> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2f16_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x half*>
   call void @llvm.masked.scatter.nxv2f16(<vscale x 2 x half> %data, <vscale x 2 x half*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2bf16_zext_offsets(<vscale x 2 x bfloat> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv2bf16_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x bfloat*>
   call void @llvm.masked.scatter.nxv2bf16(<vscale x 2 x bfloat> %data, <vscale x 2 x bfloat*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f32_zext_offsets(<vscale x 2 x float> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2f32_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z1.d, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x float*>
   call void @llvm.masked.scatter.nxv2f32(<vscale x 2 x float> %data, <vscale x 2 x float*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f64_zext_offsets(<vscale x 2 x double> %data, i8* %base, <vscale x 2 x i32> %i32offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2f64_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, z1.d, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 2 x i32> %i32offsets to <vscale x 2 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x double*>
   call void @llvm.masked.scatter.nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x double*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; unscaled packed 32-bit offsets
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 define void @masked_scatter_nxv4i8_sext_offsets(<vscale x 4 x i8> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i8_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    sunpklo z3.d, z1.s
-; CHECK-NEXT:    sunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1b { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1b { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1b { z0.s }, p0, [x0, z1.s, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i8*>
   call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x i8*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4i16_sext_offsets(<vscale x 4 x i16> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i16_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    sunpklo z3.d, z1.s
-; CHECK-NEXT:    sunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i16*>
   call void @llvm.masked.scatter.nxv4i16(<vscale x 4 x i16> %data, <vscale x 4 x i16*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4i32_sext_offsets(<vscale x 4 x i32> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i32_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    sunpklo z3.d, z1.s
-; CHECK-NEXT:    sunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1w { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, z1.s, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i32*>
   call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x i32*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4f16_sext_offsets(<vscale x 4 x half> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4f16_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    sunpklo z3.d, z1.s
-; CHECK-NEXT:    sunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x half*>
   call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x half*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4bf16_sext_offsets(<vscale x 4 x bfloat> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv4bf16_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    sunpklo z3.d, z1.s
-; CHECK-NEXT:    sunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x bfloat*>
   call void @llvm.masked.scatter.nxv4bf16(<vscale x 4 x bfloat> %data, <vscale x 4 x bfloat*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4f32_sext_offsets(<vscale x 4 x float> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv4f32_sext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    sunpklo z3.d, z1.s
-; CHECK-NEXT:    sunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1w { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, z1.s, sxtw]
 ; CHECK-NEXT:    ret
   %offsets = sext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x float*>
   call void @llvm.masked.scatter.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x float*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4i8_zext_offsets(<vscale x 4 x i8> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i8_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1b { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1b { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1b { z0.s }, p0, [x0, z1.s, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i8*>
   call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x i8*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4i16_zext_offsets(<vscale x 4 x i16> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i16_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i16*>
   call void @llvm.masked.scatter.nxv4i16(<vscale x 4 x i16> %data, <vscale x 4 x i16*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4i32_zext_offsets(<vscale x 4 x i32> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4i32_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1w { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, z1.s, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i32*>
   call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x i32*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4f16_zext_offsets(<vscale x 4 x half> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv4f16_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x half*>
   call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x half*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4bf16_zext_offsets(<vscale x 4 x bfloat> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv4bf16_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1h { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.s }, p0, [x0, z1.s, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x bfloat*>
   call void @llvm.masked.scatter.nxv4bf16(<vscale x 4 x bfloat> %data, <vscale x 4 x bfloat*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv4f32_zext_offsets(<vscale x 4 x float> %data, i8* %base, <vscale x 4 x i32> %i32offsets, <vscale x 4 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv4f32_zext_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
-; CHECK-NEXT:    pfalse p1.b
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    add z2.d, z2.d, z3.d
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    zip1 p2.s, p0.s, p1.s
-; CHECK-NEXT:    zip2 p0.s, p0.s, p1.s
-; CHECK-NEXT:    st1w { z3.d }, p2, [x8, z2.d]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, z1.s, uxtw]
 ; CHECK-NEXT:    ret
   %offsets = zext <vscale x 4 x i32> %i32offsets to <vscale x 4 x i64>
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets
   %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x float*>
   call void @llvm.masked.scatter.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x float*> %ptrs, i32 0, <vscale x 4 x i1> %masks)
   ret void
 }
 

llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; unscaled 64-bit offsets
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 define void @masked_scatter_nxv2i8_unscaled_64bit_offsets(<vscale x 2 x i8> %data, i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i8_unscaled_64bit_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1b { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1b { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i8*>
   call void @llvm.masked.scatter.nxv2i8(<vscale x 2 x i8> %data, <vscale x 2 x i8*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i16_unscaled_64bit_offsets(<vscale x 2 x i16> %data, i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i16_unscaled_64bit_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
   call void @llvm.masked.scatter.nxv2i16(<vscale x 2 x i16> %data, <vscale x 2 x i16*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i32_unscaled_64bit_offsets(<vscale x 2 x i32> %data, i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i32_unscaled_64bit_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
   call void @llvm.masked.scatter.nxv2i32(<vscale x 2 x i32> %data, <vscale x 2 x i32*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2i64_unscaled_64bit_offsets(<vscale x 2 x i64> %data, i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2i64_unscaled_64bit_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i64*>
   call void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x i64*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f16_unscaled_64bit_offsets(<vscale x 2 x half> %data, i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %masks) nounwind {
 ; CHECK-LABEL: masked_scatter_nxv2f16_unscaled_64bit_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x half*>
   call void @llvm.masked.scatter.nxv2f16(<vscale x 2 x half> %data, <vscale x 2 x half*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2bf16_unscaled_64bit_offsets(<vscale x 2 x bfloat> %data, i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv2bf16_unscaled_64bit_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1h { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x bfloat*>
   call void @llvm.masked.scatter.nxv2bf16(<vscale x 2 x bfloat> %data, <vscale x 2 x bfloat*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f32_unscaled_64bit_offsets(<vscale x 2 x float> %data, i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv2f32_unscaled_64bit_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1w { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x float*>
   call void @llvm.masked.scatter.nxv2f32(<vscale x 2 x float> %data, <vscale x 2 x float*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 define void @masked_scatter_nxv2f64_unscaled_64bit_offsets(<vscale x 2 x double> %data, i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %masks) nounwind #0 {
 ; CHECK-LABEL: masked_scatter_nxv2f64_unscaled_64bit_offsets:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z2.d, x0
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add z1.d, z2.d, z1.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [x8, z1.d]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
   %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x double*>
   call void @llvm.masked.scatter.nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x double*> %ptrs, i32 0, <vscale x 2 x i1> %masks)
   ret void
 }
 
 declare void @llvm.masked.scatter.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half*>, i32, <vscale x 2 x i1>)