[SVE] Prefer zero-extending loads when lowering ISD::EXTLOAD.

[paulwalker-arm]

Summary

The decision is perhaps arbitrary but I figure zeroing has no
dependency on the value being loaded.

[david-arm]

LGTM!

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
   ISD::MemIndexType IndexType = MGT->getIndexType();
   bool IsScaled =
       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::UNSIGNED_SCALED;
   bool IsSigned =
       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::SIGNED_UNSCALED;
   bool IdxNeedsExtend =
       getGatherScatterIndexIsExtended(Index) ||
       Index.getSimpleValueType().getVectorElementType() == MVT::i32;
-  bool ResNeedsSignExtend = ExtTy == ISD::EXTLOAD || ExtTy == ISD::SEXTLOAD;
 
   EVT VT = PassThru.getSimpleValueType();
   EVT IndexVT = Index.getSimpleValueType();
   EVT MemVT = MGT->getMemoryVT();
   SDValue InputVT = DAG.getValueType(MemVT);
 
   if (VT.getVectorElementType() == MVT::bf16 &&
       !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
 
   if (getGatherScatterIndexIsExtended(Index))
     Index = Index.getOperand(0);
 
   unsigned Opcode = getGatherVecOpcode(IsScaled, IsSigned, IdxNeedsExtend);
   selectGatherScatterAddrMode(BasePtr, Index, MemVT, Opcode,
                               /*isGather=*/true, DAG);
 
-  if (ResNeedsSignExtend)
+  if (ExtTy == ISD::SEXTLOAD)
     Opcode = getSignExtendedGatherOpcode(Opcode);
 
   if (IsFixedLength) {
     if (Index.getSimpleValueType().isFixedLengthVector())
       Index = convertToScalableVector(DAG, IndexVT, Index);
     if (BasePtr.getSimpleValueType().isFixedLengthVector())
       BasePtr = convertToScalableVector(DAG, IndexVT, BasePtr);
     Mask = convertFixedMaskToScalableVector(Mask, DAG);

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 // non-extending masked load fragment.
 def nonext_masked_load :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
           (masked_ld node:$ptr, undef, node:$pred, node:$def), [{
   return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD &&
          cast<MaskedLoadSDNode>(N)->isUnindexed() &&
          !cast<MaskedLoadSDNode>(N)->isNonTemporal();
 }]>;
-// sign extending masked load fragments.
-def asext_masked_load :
+// Any/Zero extending masked load fragments.
+def azext_masked_load :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
           (masked_ld node:$ptr, undef, node:$pred, node:$def),[{
   return (cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD ||
-          cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD) &&
+          cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD) &&
          cast<MaskedLoadSDNode>(N)->isUnindexed();
 }]>;
-def asext_masked_load_i8 :
+def azext_masked_load_i8 :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
-          (asext_masked_load node:$ptr, node:$pred, node:$def), [{
+          (azext_masked_load node:$ptr, node:$pred, node:$def), [{
   return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
 }]>;
-def asext_masked_load_i16 :
+def azext_masked_load_i16 :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
-          (asext_masked_load node:$ptr, node:$pred, node:$def), [{
+          (azext_masked_load node:$ptr, node:$pred, node:$def), [{
   return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16;
 }]>;
-def asext_masked_load_i32 :
+def azext_masked_load_i32 :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
-          (asext_masked_load node:$ptr, node:$pred, node:$def), [{
+          (azext_masked_load node:$ptr, node:$pred, node:$def), [{
   return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32;
 }]>;
-// zero extending masked load fragments.
-def zext_masked_load :
+// Sign extending masked load fragments.
+def sext_masked_load :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
           (masked_ld node:$ptr, undef, node:$pred, node:$def), [{
-  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD &&
+  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD &&
          cast<MaskedLoadSDNode>(N)->isUnindexed();
 }]>;
-def zext_masked_load_i8 :
+def sext_masked_load_i8 :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
-          (zext_masked_load node:$ptr, node:$pred, node:$def), [{
+          (sext_masked_load node:$ptr, node:$pred, node:$def), [{
   return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
 }]>;
-def zext_masked_load_i16 :
+def sext_masked_load_i16 :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
-          (zext_masked_load node:$ptr, node:$pred, node:$def), [{
+          (sext_masked_load node:$ptr, node:$pred, node:$def), [{
   return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16;
 }]>;
-def zext_masked_load_i32 :
+def sext_masked_load_i32 :
   PatFrag<(ops node:$ptr, node:$pred, node:$def),
-          (zext_masked_load node:$ptr, node:$pred, node:$def), [{
+          (sext_masked_load node:$ptr, node:$pred, node:$def), [{
   return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32;
 }]>;
 
 def non_temporal_load :
    PatFrag<(ops node:$ptr, node:$pred, node:$def),
            (masked_ld node:$ptr, undef, node:$pred, node:$def), [{
    return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD &&
           cast<MaskedLoadSDNode>(N)->isUnindexed() &&

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

let AddedComplexity = 2 in {
       def _reg_imm_z : Pat<(Ty (Load (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), (PredTy PPR:$gp), (SVEDup0Undef))),
                            (RegImmInst PPR:$gp, GPR64:$base, simm4s1:$offset)>;
     }
     def _default_z : Pat<(Ty (Load  GPR64:$base, (PredTy PPR:$gp), (SVEDup0Undef))),
                          (RegImmInst PPR:$gp, GPR64:$base, (i64 0))>;
   }
 
   // 2-element contiguous loads
-  defm : pred_load<nxv2i64,  nxv2i1, zext_masked_load_i8,   LD1B_D,  LD1B_D_IMM,  am_sve_regreg_lsl0>;
-  defm : pred_load<nxv2i64,  nxv2i1, asext_masked_load_i8,  LD1SB_D, LD1SB_D_IMM, am_sve_regreg_lsl0>;
-  defm : pred_load<nxv2i64,  nxv2i1, zext_masked_load_i16,  LD1H_D,  LD1H_D_IMM,  am_sve_regreg_lsl1>;
-  defm : pred_load<nxv2i64,  nxv2i1, asext_masked_load_i16, LD1SH_D, LD1SH_D_IMM, am_sve_regreg_lsl1>;
-  defm : pred_load<nxv2i64,  nxv2i1, zext_masked_load_i32,  LD1W_D,  LD1W_D_IMM,  am_sve_regreg_lsl2>;
-  defm : pred_load<nxv2i64,  nxv2i1, asext_masked_load_i32, LD1SW_D, LD1SW_D_IMM, am_sve_regreg_lsl2>;
+  defm : pred_load<nxv2i64,  nxv2i1, azext_masked_load_i8,  LD1B_D,  LD1B_D_IMM,  am_sve_regreg_lsl0>;
+  defm : pred_load<nxv2i64,  nxv2i1, sext_masked_load_i8,   LD1SB_D, LD1SB_D_IMM, am_sve_regreg_lsl0>;
+  defm : pred_load<nxv2i64,  nxv2i1, azext_masked_load_i16, LD1H_D,  LD1H_D_IMM,  am_sve_regreg_lsl1>;
+  defm : pred_load<nxv2i64,  nxv2i1, sext_masked_load_i16,  LD1SH_D, LD1SH_D_IMM, am_sve_regreg_lsl1>;
+  defm : pred_load<nxv2i64,  nxv2i1, azext_masked_load_i32, LD1W_D,  LD1W_D_IMM,  am_sve_regreg_lsl2>;
+  defm : pred_load<nxv2i64,  nxv2i1, sext_masked_load_i32,  LD1SW_D, LD1SW_D_IMM, am_sve_regreg_lsl2>;
   defm : pred_load<nxv2i64,  nxv2i1, nonext_masked_load,    LD1D,    LD1D_IMM,    am_sve_regreg_lsl3>;
   defm : pred_load<nxv2f16,  nxv2i1, nonext_masked_load,    LD1H_D,  LD1H_D_IMM,  am_sve_regreg_lsl1>;
   defm : pred_load<nxv2bf16, nxv2i1, nonext_masked_load,    LD1H_D,  LD1H_D_IMM,  am_sve_regreg_lsl1>;
   defm : pred_load<nxv2f32,  nxv2i1, nonext_masked_load,    LD1W_D,  LD1W_D_IMM,  am_sve_regreg_lsl2>;
   defm : pred_load<nxv2f64,  nxv2i1, nonext_masked_load,    LD1D,    LD1D_IMM,    am_sve_regreg_lsl3>;
 
   // 4-element contiguous loads
-  defm : pred_load<nxv4i32,  nxv4i1, zext_masked_load_i8,   LD1B_S,  LD1B_S_IMM,  am_sve_regreg_lsl0>;
-  defm : pred_load<nxv4i32,  nxv4i1, asext_masked_load_i8,  LD1SB_S, LD1SB_S_IMM, am_sve_regreg_lsl0>;
-  defm : pred_load<nxv4i32,  nxv4i1, zext_masked_load_i16,  LD1H_S,  LD1H_S_IMM,  am_sve_regreg_lsl1>;
-  defm : pred_load<nxv4i32,  nxv4i1, asext_masked_load_i16, LD1SH_S, LD1SH_S_IMM, am_sve_regreg_lsl1>;
+  defm : pred_load<nxv4i32,  nxv4i1, azext_masked_load_i8,  LD1B_S,  LD1B_S_IMM,  am_sve_regreg_lsl0>;
+  defm : pred_load<nxv4i32,  nxv4i1, sext_masked_load_i8,   LD1SB_S, LD1SB_S_IMM, am_sve_regreg_lsl0>;
+  defm : pred_load<nxv4i32,  nxv4i1, azext_masked_load_i16, LD1H_S,  LD1H_S_IMM,  am_sve_regreg_lsl1>;
+  defm : pred_load<nxv4i32,  nxv4i1, sext_masked_load_i16,  LD1SH_S, LD1SH_S_IMM, am_sve_regreg_lsl1>;
   defm : pred_load<nxv4i32,  nxv4i1, nonext_masked_load,    LD1W,    LD1W_IMM,    am_sve_regreg_lsl2>;
   defm : pred_load<nxv4f16,  nxv4i1, nonext_masked_load,    LD1H_S,  LD1H_S_IMM,  am_sve_regreg_lsl1>;
   defm : pred_load<nxv4bf16, nxv4i1, nonext_masked_load,    LD1H_S,  LD1H_S_IMM,  am_sve_regreg_lsl1>;
   defm : pred_load<nxv4f32,  nxv4i1, nonext_masked_load,    LD1W,    LD1W_IMM,    am_sve_regreg_lsl2>;
 
   // 8-element contiguous loads
-  defm : pred_load<nxv8i16,  nxv8i1, zext_masked_load_i8,  LD1B_H,  LD1B_H_IMM,  am_sve_regreg_lsl0>;
-  defm : pred_load<nxv8i16,  nxv8i1, asext_masked_load_i8, LD1SB_H, LD1SB_H_IMM, am_sve_regreg_lsl0>;
+  defm : pred_load<nxv8i16,  nxv8i1, azext_masked_load_i8, LD1B_H,  LD1B_H_IMM,  am_sve_regreg_lsl0>;
+  defm : pred_load<nxv8i16,  nxv8i1, sext_masked_load_i8,  LD1SB_H, LD1SB_H_IMM, am_sve_regreg_lsl0>;
   defm : pred_load<nxv8i16,  nxv8i1, nonext_masked_load,   LD1H,    LD1H_IMM,    am_sve_regreg_lsl1>;
   defm : pred_load<nxv8f16,  nxv8i1, nonext_masked_load,   LD1H,    LD1H_IMM,    am_sve_regreg_lsl1>;
   defm : pred_load<nxv8bf16, nxv8i1, nonext_masked_load,   LD1H,    LD1H_IMM,    am_sve_regreg_lsl1>;
 
   // 16-element contiguous loads
   defm : pred_load<nxv16i8, nxv16i1, nonext_masked_load, LD1B, LD1B_IMM, am_sve_regreg_lsl0>;
 
   multiclass pred_store<ValueType Ty, ValueType PredTy, SDPatternOperator Store,

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll

; CHECK-NEXT:    ret
   store <4 x float> %res, <4 x float>* %b
   ret void
 }
 
 define void @fcvt_v8f16_v8f32(<8 x half>* %a, <8 x float>* %b) #0 {
 ; CHECK-LABEL: fcvt_v8f16_v8f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl8
-; CHECK-NEXT:    ld1sh { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x1]
 ; CHECK-NEXT:    ret
   %op1 = load <8 x half>, <8 x half>* %a
   %res = fpext <8 x half> %op1 to <8 x float>
   store <8 x float> %res, <8 x float>* %b
   ret void
 }
 
 define void @fcvt_v16f16_v16f32(<16 x half>* %a, <16 x float>* %b) #0 {
 ; Ensure sensible type legalisation.
 ; VBITS_EQ_256-LABEL: fcvt_v16f16_v16f32:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    mov x8, #8
 ; VBITS_EQ_256-NEXT:    ptrue p0.s, vl8
-; VBITS_EQ_256-NEXT:    ld1sh { z0.s }, p0/z, [x0, x8, lsl #1]
-; VBITS_EQ_256-NEXT:    ld1sh { z1.s }, p0/z, [x0]
+; VBITS_EQ_256-NEXT:    ld1h { z0.s }, p0/z, [x0, x8, lsl #1]
+; VBITS_EQ_256-NEXT:    ld1h { z1.s }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    fcvt z0.s, p0/m, z0.h
 ; VBITS_EQ_256-NEXT:    fcvt z1.s, p0/m, z1.h
 ; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
 ; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x1]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fcvt_v16f16_v16f32:
 ; VBITS_GE_512:       // %bb.0:
 ; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
-; VBITS_GE_512-NEXT:    ld1sh { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1h { z0.s }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    fcvt z0.s, p0/m, z0.h
 ; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x1]
 ; VBITS_GE_512-NEXT:    ret
 
   %op1 = load <16 x half>, <16 x half>* %a
   %res = fpext <16 x half> %op1 to <16 x float>
   store <16 x float> %res, <16 x float>* %b
   ret void
 }
 
 define void @fcvt_v32f16_v32f32(<32 x half>* %a, <32 x float>* %b) #0 {
 ; VBITS_GE_1024-LABEL: fcvt_v32f16_v32f32:
 ; VBITS_GE_1024:       // %bb.0:
 ; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_GE_1024-NEXT:    ld1sh { z0.s }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1h { z0.s }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    fcvt z0.s, p0/m, z0.h
 ; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x1]
 ; VBITS_GE_1024-NEXT:    ret
   %op1 = load <32 x half>, <32 x half>* %a
   %res = fpext <32 x half> %op1 to <32 x float>
   store <32 x float> %res, <32 x float>* %b
   ret void
 }
 
 define void @fcvt_v64f16_v64f32(<64 x half>* %a, <64 x float>* %b) #0 {
 ; VBITS_GE_2048-LABEL: fcvt_v64f16_v64f32:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.s, vl64
-; VBITS_GE_2048-NEXT:    ld1sh { z0.s }, p0/z, [x0]
+; VBITS_GE_2048-NEXT:    ld1h { z0.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    fcvt z0.s, p0/m, z0.h
 ; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x1]
 ; VBITS_GE_2048-NEXT:    ret
   %op1 = load <64 x half>, <64 x half>* %a
   %res = fpext <64 x half> %op1 to <64 x float>
   store <64 x float> %res, <64 x float>* %b
   ret void
 }
; CHECK-NEXT:    ret
   store <2 x double> %res, <2 x double>* %b
   ret void
 }
 
 define void @fcvt_v4f16_v4f64(<4 x half>* %a, <4 x double>* %b) #0 {
 ; CHECK-LABEL: fcvt_v4f16_v4f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl4
-; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z0.h
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
 ; CHECK-NEXT:    ret
   %op1 = load <4 x half>, <4 x half>* %a
   %res = fpext <4 x half> %op1 to <4 x double>
   store <4 x double> %res, <4 x double>* %b
   ret void
 }
 
 define void @fcvt_v8f16_v8f64(<8 x half>* %a, <8 x double>* %b) #0 {
 ; VBITS_EQ_256-LABEL: fcvt_v8f16_v8f64:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    mov x8, #4
 ; VBITS_EQ_256-NEXT:    ptrue p0.d, vl4
-; VBITS_EQ_256-NEXT:    ld1sh { z0.d }, p0/z, [x0, x8, lsl #1]
-; VBITS_EQ_256-NEXT:    ld1sh { z1.d }, p0/z, [x0]
+; VBITS_EQ_256-NEXT:    ld1h { z0.d }, p0/z, [x0, x8, lsl #1]
+; VBITS_EQ_256-NEXT:    ld1h { z1.d }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    fcvt z0.d, p0/m, z0.h
 ; VBITS_EQ_256-NEXT:    fcvt z1.d, p0/m, z1.h
 ; VBITS_EQ_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
 ; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x1]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fcvt_v8f16_v8f64:
 ; VBITS_GE_512:       // %bb.0:
 ; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
-; VBITS_GE_512-NEXT:    ld1sh { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1h { z0.d }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    fcvt z0.d, p0/m, z0.h
 ; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x1]
 ; VBITS_GE_512-NEXT:    ret
 
   %op1 = load <8 x half>, <8 x half>* %a
   %res = fpext <8 x half> %op1 to <8 x double>
   store <8 x double> %res, <8 x double>* %b
   ret void
 }
 
 define void @fcvt_v16f16_v16f64(<16 x half>* %a, <16 x double>* %b) #0 {
 ; VBITS_GE_1024-LABEL: fcvt_v16f16_v16f64:
 ; VBITS_GE_1024:       // %bb.0:
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    ld1sh { z0.d }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1h { z0.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    fcvt z0.d, p0/m, z0.h
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x1]
 ; VBITS_GE_1024-NEXT:    ret
   %op1 = load <16 x half>, <16 x half>* %a
   %res = fpext <16 x half> %op1 to <16 x double>
   store <16 x double> %res, <16 x double>* %b
   ret void
 }
 
 define void @fcvt_v32f16_v32f64(<32 x half>* %a, <32 x double>* %b) #0 {
 ; VBITS_GE_2048-LABEL: fcvt_v32f16_v32f64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
-; VBITS_GE_2048-NEXT:    ld1sh { z0.d }, p0/z, [x0]
+; VBITS_GE_2048-NEXT:    ld1h { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    fcvt z0.d, p0/m, z0.h
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x1]
 ; VBITS_GE_2048-NEXT:    ret
   %op1 = load <32 x half>, <32 x half>* %a
   %res = fpext <32 x half> %op1 to <32 x double>
   store <32 x double> %res, <32 x double>* %b
   ret void
 }
; CHECK-NEXT:    ret
   store <2 x double> %res, <2 x double>* %b
   ret void
 }
 
 define void @fcvt_v4f32_v4f64(<4 x float>* %a, <4 x double>* %b) #0 {
 ; CHECK-LABEL: fcvt_v4f32_v4f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl4
-; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z0.s
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
 ; CHECK-NEXT:    ret
   %op1 = load <4 x float>, <4 x float>* %a
   %res = fpext <4 x float> %op1 to <4 x double>
   store <4 x double> %res, <4 x double>* %b
   ret void
 }
 
 define void @fcvt_v8f32_v8f64(<8 x float>* %a, <8 x double>* %b) #0 {
 ; Ensure sensible type legalisation.
 ; VBITS_EQ_256-LABEL: fcvt_v8f32_v8f64:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    mov x8, #4
 ; VBITS_EQ_256-NEXT:    ptrue p0.d, vl4
-; VBITS_EQ_256-NEXT:    ld1sw { z0.d }, p0/z, [x0, x8, lsl #2]
-; VBITS_EQ_256-NEXT:    ld1sw { z1.d }, p0/z, [x0]
+; VBITS_EQ_256-NEXT:    ld1w { z0.d }, p0/z, [x0, x8, lsl #2]
+; VBITS_EQ_256-NEXT:    ld1w { z1.d }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    fcvt z0.d, p0/m, z0.s
 ; VBITS_EQ_256-NEXT:    fcvt z1.d, p0/m, z1.s
 ; VBITS_EQ_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
 ; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x1]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fcvt_v8f32_v8f64:
 ; VBITS_GE_512:       // %bb.0:
 ; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
-; VBITS_GE_512-NEXT:    ld1sw { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1w { z0.d }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    fcvt z0.d, p0/m, z0.s
 ; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x1]
 ; VBITS_GE_512-NEXT:    ret
   %op1 = load <8 x float>, <8 x float>* %a
   %res = fpext <8 x float> %op1 to <8 x double>
   store <8 x double> %res, <8 x double>* %b
   ret void
 }
 
 define void @fcvt_v16f32_v16f64(<16 x float>* %a, <16 x double>* %b) #0 {
 ; VBITS_GE_1024-LABEL: fcvt_v16f32_v16f64:
 ; VBITS_GE_1024:       // %bb.0:
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    ld1sw { z0.d }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1w { z0.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    fcvt z0.d, p0/m, z0.s
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x1]
 ; VBITS_GE_1024-NEXT:    ret
   %op1 = load <16 x float>, <16 x float>* %a
   %res = fpext <16 x float> %op1 to <16 x double>
   store <16 x double> %res, <16 x double>* %b
   ret void
 }
 
 define void @fcvt_v32f32_v32f64(<32 x float>* %a, <32 x double>* %b) #0 {
 ; VBITS_GE_2048-LABEL: fcvt_v32f32_v32f64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
-; VBITS_GE_2048-NEXT:    ld1sw { z0.d }, p0/z, [x0]
+; VBITS_GE_2048-NEXT:    ld1w { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    fcvt z0.d, p0/m, z0.s
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x1]
 ; VBITS_GE_2048-NEXT:    ret
   %op1 = load <32 x float>, <32 x float>* %a
   %res = fpext <32 x float> %op1 to <32 x double>
   store <32 x double> %res, <32 x double>* %b
   ret void
 }

llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll

 ; LD1B
 ;
 
 define void @masked_gather_v2i8(<2 x i8>* %a, <2 x i8*>* %b) #0 {
 ; CHECK-LABEL: masked_gather_v2i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x1]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    xtn v0.2s, v0.2d
 ; CHECK-NEXT:    st1b { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %ptrs = load <2 x i8*>, <2 x i8*>* %b
   %vals = call <2 x i8> @llvm.masked.gather.v2i8(<2 x i8*> %ptrs, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i8> undef)
   store <2 x i8> %vals, <2 x i8>* %a
   ret void
 }
 
 define void @masked_gather_v4i8(<4 x i8>* %a, <4 x i8*>* %b) #0 {
 ; CHECK-LABEL: masked_gather_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl4
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x1]
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %ptrs = load <4 x i8*>, <4 x i8*>* %b
   %vals = call <4 x i8> @llvm.masked.gather.v4i8(<4 x i8*> %ptrs, i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
   store <4 x i8> %vals, <4 x i8>* %a
   ret void
 }
 
 define void @masked_gather_v8i8(<8 x i8>* %a, <8 x i8*>* %b) #0 {
 ; VBITS_EQ_256-LABEL: masked_gather_v8i8:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    mov x8, #4
 ; VBITS_EQ_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_EQ_256-NEXT:    ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
 ; VBITS_EQ_256-NEXT:    ld1d { z1.d }, p0/z, [x1]
-; VBITS_EQ_256-NEXT:    ld1sb { z0.d }, p0/z, [z0.d]
-; VBITS_EQ_256-NEXT:    ld1sb { z1.d }, p0/z, [z1.d]
+; VBITS_EQ_256-NEXT:    ld1b { z0.d }, p0/z, [z0.d]
+; VBITS_EQ_256-NEXT:    ld1b { z1.d }, p0/z, [z1.d]
 ; VBITS_EQ_256-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; VBITS_EQ_256-NEXT:    uzp1 z1.s, z1.s, z1.s
 ; VBITS_EQ_256-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; VBITS_EQ_256-NEXT:    uzp1 z1.h, z1.h, z1.h
 ; VBITS_EQ_256-NEXT:    uzp1 v0.8b, v1.8b, v0.8b
 ; VBITS_EQ_256-NEXT:    str d0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;

 ; LD1H
 ;
 
 define void @masked_gather_v2i16(<2 x i16>* %a, <2 x i16*>* %b) #0 {
 ; CHECK-LABEL: masked_gather_v2i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x1]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    xtn v0.2s, v0.2d
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %ptrs = load <2 x i16*>, <2 x i16*>* %b
   %vals = call <2 x i16> @llvm.masked.gather.v2i16(<2 x i16*> %ptrs, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i16> undef)
   store <2 x i16> %vals, <2 x i16>* %a
   ret void

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll

 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    ldr q1, [x1]
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    ldrb w8, [x0, #1]
 ; CHECK-NEXT:    mov v0.s[1], w8
 ; CHECK-NEXT:    cmeq v0.2s, v0.2s, #0
 ; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [z1.d]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [z1.d]
 ; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    xtn v0.2s, v0.2d
 ; CHECK-NEXT:    st1b { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %cval = load <2 x i8>, <2 x i8>* %a
   %ptrs = load <2 x i8*>, <2 x i8*>* %b
   %mask = icmp eq <2 x i8> %cval, zeroinitializer
   %vals = call <2 x i8> @llvm.masked.gather.v2i8(<2 x i8*> %ptrs, i32 8, <2 x i1> %mask, <2 x i8> undef)
   store <2 x i8> %vals, <2 x i8>* %a
   ret void
 }
 
 define void @masked_gather_v4i8(<4 x i8>* %a, <4 x i8*>* %b) #0 {
 ; CHECK-LABEL: masked_gather_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl4
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-NEXT:    cmeq v0.4h, v0.4h, #0
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [z1.d]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [z1.d]
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %cval = load <4 x i8>, <4 x i8>* %a
   %ptrs = load <4 x i8*>, <4 x i8*>* %b
   %mask = icmp eq <4 x i8> %cval, zeroinitializer

 ; VBITS_EQ_256-NEXT:    sshr v1.4h, v1.4h, #8
 ; VBITS_EQ_256-NEXT:    sshr v0.4h, v0.4h, #8
 ; VBITS_EQ_256-NEXT:    sunpklo z1.s, z1.h
 ; VBITS_EQ_256-NEXT:    sunpklo z0.s, z0.h
 ; VBITS_EQ_256-NEXT:    sunpklo z1.d, z1.s
 ; VBITS_EQ_256-NEXT:    sunpklo z0.d, z0.s
 ; VBITS_EQ_256-NEXT:    cmpne p1.d, p0/z, z1.d, #0
 ; VBITS_EQ_256-NEXT:    cmpne p0.d, p0/z, z0.d, #0
-; VBITS_EQ_256-NEXT:    ld1sb { z0.d }, p1/z, [z2.d]
-; VBITS_EQ_256-NEXT:    ld1sb { z1.d }, p0/z, [z3.d]
+; VBITS_EQ_256-NEXT:    ld1b { z0.d }, p1/z, [z2.d]
+; VBITS_EQ_256-NEXT:    ld1b { z1.d }, p0/z, [z3.d]
 ; VBITS_EQ_256-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; VBITS_EQ_256-NEXT:    uzp1 z1.s, z1.s, z1.s
 ; VBITS_EQ_256-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; VBITS_EQ_256-NEXT:    uzp1 z1.h, z1.h, z1.h
 ; VBITS_EQ_256-NEXT:    uzp1 v0.8b, v1.8b, v0.8b
 ; VBITS_EQ_256-NEXT:    str d0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;

 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    ldr q1, [x1]
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    ldrh w8, [x0, #2]
 ; CHECK-NEXT:    mov v0.s[1], w8
 ; CHECK-NEXT:    cmeq v0.2s, v0.2s, #0
 ; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
-; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [z1.d]
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [z1.d]
 ; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    xtn v0.2s, v0.2d
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %cval = load <2 x i16>, <2 x i16>* %a
   %ptrs = load <2 x i16*>, <2 x i16*>* %b
   %mask = icmp eq <2 x i16> %cval, zeroinitializer
   %vals = call <2 x i16> @llvm.masked.gather.v2i16(<2 x i16*> %ptrs, i32 8, <2 x i1> %mask, <2 x i16> undef)

llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll

 }
 
 ; Ensure we use a "vscale x 4" gather for an offset in the limits of 32 bits.
 define <vscale x 4 x i8> @gather_i8_index_offset_8(i8* %base, i64 %offset, <vscale x 4 x i1> %pg) #0 {
 ; CHECK-LABEL: gather_i8_index_offset_8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add x8, x0, x1
 ; CHECK-NEXT:    index z0.s, #0, #1
-; CHECK-NEXT:    ld1sb { z0.s }, p0/z, [x8, z0.s, sxtw]
+; CHECK-NEXT:    ld1b { z0.s }, p0/z, [x8, z0.s, sxtw]
 ; CHECK-NEXT:    ret
   %splat.insert0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
   %splat0 = shufflevector <vscale x 4 x i64> %splat.insert0, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %step = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
   %splat.insert1 = insertelement <vscale x 4 x i64> undef, i64 1, i32 0
   %splat1 = shufflevector <vscale x 4 x i64> %splat.insert1, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %t1 = mul <vscale x 4 x i64> %splat1, %step
   %t2 = add <vscale x 4 x i64> %splat0, %t1

 ; Ensure the resulting load is "vscale x 4" wide, despite the offset giving the
 ; impression the gather must be split due to it's <vscale x 4 x i64> offset.
 ; gather_f32(base, index(offset, 8 * sizeof(float))
 define <vscale x 4 x i8> @gather_8i8_index_offset_8([8 x i8]* %base, i64 %offset, <vscale x 4 x i1> %pg) #0 {
 ; CHECK-LABEL: gather_8i8_index_offset_8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add x8, x0, x1, lsl #3
 ; CHECK-NEXT:    index z0.s, #0, #8
-; CHECK-NEXT:    ld1sb { z0.s }, p0/z, [x8, z0.s, sxtw]
+; CHECK-NEXT:    ld1b { z0.s }, p0/z, [x8, z0.s, sxtw]
 ; CHECK-NEXT:    ret
   %t0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
   %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %step = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
   %t2 = add <vscale x 4 x i64> %t1, %step
   %t3 = getelementptr [8 x i8], [8 x i8]* %base, <vscale x 4 x i64> %t2
   %t4 = bitcast <vscale x 4 x [8 x i8]*> %t3 to <vscale x 4 x i8*>
   %load = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*> %t4, i32 4, <vscale x 4 x i1> %pg, <vscale x 4 x i8> undef)

llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -aarch64-enable-mgather-combine=0 -enable-misched=false < %s | FileCheck %s
 ; RUN: llc -aarch64-enable-mgather-combine=1 -enable-misched=false < %s | FileCheck %s
 
 target triple = "aarch64-linux-gnu"
 
 ; Test for multiple uses of the mgather where the s/zext should not be combined
 
 define <vscale x 2 x i64> @masked_sgather_sext(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask, <vscale x 2 x i8> %vals) #0 {
 ; CHECK-LABEL: masked_sgather_sext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [x0, z0.d]
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sxtb z2.d, p0/m, z0.d
 ; CHECK-NEXT:    add z0.d, z0.d, z1.d
 ; CHECK-NEXT:    sxtb z0.d, p0/m, z0.d
 ; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z2.d
 ; CHECK-NEXT:    ret
   %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %data = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
   %data.sext = sext <vscale x 2 x i8> %data to <vscale x 2 x i64>
   %add = add <vscale x 2 x i8> %data, %vals
   %add.sext = sext <vscale x 2 x i8> %add to <vscale x 2 x i64>
   %mul = mul <vscale x 2 x i64> %data.sext, %add.sext
   ret <vscale x 2 x i64> %mul
 }
 
 define <vscale x 2 x i64> @masked_sgather_zext(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask, <vscale x 2 x i8> %vals) #0 {
 ; CHECK-LABEL: masked_sgather_zext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [x0, z0.d]
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    add z1.d, z0.d, z1.d
 ; CHECK-NEXT:    and z0.d, z0.d, #0xff
 ; CHECK-NEXT:    and z1.d, z1.d, #0xff
 ; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
   %data = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
   %data.zext = zext <vscale x 2 x i8> %data to <vscale x 2 x i64>
   %add = add <vscale x 2 x i8> %data, %vals
   %add.zext = zext <vscale x 2 x i8> %add to <vscale x 2 x i64>
   %mul = mul <vscale x 2 x i64> %data.zext, %add.zext
   ret <vscale x 2 x i64> %mul
 }
 
 ; Tests that exercise various type legalisation scenarios for ISD::MGATHER.
 
 ; Code generate load of an illegal datatype via promotion.
 define <vscale x 2 x i8> @masked_gather_nxv2i8(<vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %mask) #0 {
 ; CHECK-LABEL: masked_gather_nxv2i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    ret
   %data = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
   ret <vscale x 2 x i8> %data
 }
 
 ; Code generate load of an illegal datatype via promotion.
 define <vscale x 2 x i16> @masked_gather_nxv2i16(<vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %mask) #0 {
 ; CHECK-LABEL: masked_gather_nxv2i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    ret
   %data = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
   ret <vscale x 2 x i16> %data
 }
 
 ; Code generate load of an illegal datatype via promotion.
 define <vscale x 2 x i32> @masked_gather_nxv2i32(<vscale x 2 x i32*> %ptrs, <vscale x 2 x i1> %mask) #0 {
 ; CHECK-LABEL: masked_gather_nxv2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    ret
   %data = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
   ret <vscale x 2 x i32> %data
 }
 
 define <vscale x 4 x half> @masked_gather_nxv4f16(<vscale x 4 x half*> %ptrs, <vscale x 4 x i1> %mask) #0 {
 ; CHECK-LABEL: masked_gather_nxv4f16:
 ; CHECK:       // %bb.0:

 ; CHECK-LABEL: masked_gather_nxv16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sunpkhi z1.h, z0.b
 ; CHECK-NEXT:    punpkhi p1.h, p0.b
 ; CHECK-NEXT:    sunpkhi z2.s, z1.h
 ; CHECK-NEXT:    punpkhi p2.h, p1.b
 ; CHECK-NEXT:    sunpklo z1.s, z1.h
 ; CHECK-NEXT:    punpklo p1.h, p1.b
-; CHECK-NEXT:    ld1sb { z2.s }, p2/z, [x0, z2.s, sxtw]
-; CHECK-NEXT:    ld1sb { z1.s }, p1/z, [x0, z1.s, sxtw]
+; CHECK-NEXT:    ld1b { z2.s }, p2/z, [x0, z2.s, sxtw]
+; CHECK-NEXT:    ld1b { z1.s }, p1/z, [x0, z1.s, sxtw]
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
 ; CHECK-NEXT:    punpklo p0.h, p0.b
 ; CHECK-NEXT:    punpkhi p1.h, p0.b
 ; CHECK-NEXT:    punpklo p0.h, p0.b
 ; CHECK-NEXT:    uzp1 z1.h, z1.h, z2.h
 ; CHECK-NEXT:    sunpkhi z2.s, z0.h
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    ld1sb { z2.s }, p1/z, [x0, z2.s, sxtw]
-; CHECK-NEXT:    ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ld1b { z2.s }, p1/z, [x0, z2.s, sxtw]
+; CHECK-NEXT:    ld1b { z0.s }, p0/z, [x0, z0.s, sxtw]
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z2.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
   %ptrs = getelementptr i8, i8* %base, <vscale x 16 x i8> %indices
   %data = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8(<vscale x 16 x i8*> %ptrs, i32 1, <vscale x 16 x i1> %mask, <vscale x 16 x i8> undef)
   ret <vscale x 16 x i8> %data
 }
 

 ; registers, so it doesn't get folded away. Same for any other vector-of-pointers
 ; style gathers which don't fit in an <vscale x 2 x type*> single register. Better folding
 ; is required before we can check those off.
 define <vscale x 4 x i32> @masked_sgather_nxv4i8(<vscale x 4 x i8*> %ptrs, <vscale x 4 x i1> %mask) #0 {
 ; CHECK-LABEL: masked_sgather_nxv4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    punpkhi p1.h, p0.b
 ; CHECK-NEXT:    punpklo p0.h, p0.b
-; CHECK-NEXT:    ld1sb { z1.d }, p1/z, [z1.d]
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ld1b { z1.d }, p1/z, [z1.d]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z1.s
 ; CHECK-NEXT:    sxtb z0.s, p0/m, z0.s
 ; CHECK-NEXT:    ret
   %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef)
   %svals = sext <vscale x 4 x i8> %vals to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %svals
 }

llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll

 ; CHECK-NEXT: ret
   %load = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32(<vscale x 4 x i32> *%a, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %passthru)
   ret <vscale x 4 x i32> %load
 }
 
 ; Masked load requires promotion
 define <vscale x 2 x i16> @masked_load_nxv2i16(<vscale x 2 x i16>* noalias %in, <vscale x 2 x i1> %mask) {
 ; CHECK-LABEL: masked_load_nxv2i16
-; CHECK:       ld1sh { z0.d }, p0/z, [x0]
+; CHECK:       ld1h { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:  ret
   %wide.load = call <vscale x 2 x i16> @llvm.masked.load.nxv2i16(<vscale x 2 x i16>* %in, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
   ret <vscale x 2 x i16> %wide.load
 }
 
 ;
 ; Masked Stores
 ;

llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll

; CHECK-NEXT:    ret
   ret void
 }
 
 ; 2-lane contiguous load/stores
 
 define void @test_masked_ldst_sv2i8(<vscale x 2 x i8> * %base, <vscale x 2 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [x0, #-8, mul vl]
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [x0, #-8, mul vl]
 ; CHECK-NEXT:    st1b { z0.d }, p0, [x0, #-7, mul vl]
 ; CHECK-NEXT:    ret
   %base_load = getelementptr <vscale x 2 x i8>, <vscale x 2 x i8>* %base, i64 -8
   %data = call <vscale x 2 x i8> @llvm.masked.load.nxv2i8(<vscale x 2 x i8>* %base_load,
                                                           i32 1,
                                                           <vscale x 2 x i1> %mask,
                                                           <vscale x 2 x i8> undef)
   %base_store = getelementptr <vscale x 2 x i8>, <vscale x 2 x i8> * %base, i64 -7
   call void @llvm.masked.store.nxv2i8(<vscale x 2 x i8> %data,
                                       <vscale x 2 x i8>* %base_store,
                                       i32 1,
                                       <vscale x 2 x i1> %mask)
   ret void
 }
 
 define void @test_masked_ldst_sv2i16(<vscale x 2 x i16> * %base, <vscale x 2 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [x0, #-8, mul vl]
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, #-8, mul vl]
 ; CHECK-NEXT:    st1h { z0.d }, p0, [x0, #-7, mul vl]
 ; CHECK-NEXT:    ret
   %base_load = getelementptr <vscale x 2 x i16>, <vscale x 2 x i16>* %base, i64 -8
   %data = call <vscale x 2 x i16> @llvm.masked.load.nxv2i16(<vscale x 2 x i16>* %base_load,
                                                             i32 1,
                                                             <vscale x 2 x i1> %mask,
                                                             <vscale x 2 x i16> undef)
   %base_store = getelementptr <vscale x 2 x i16>, <vscale x 2 x i16> * %base, i64 -7
   call void @llvm.masked.store.nxv2i16(<vscale x 2 x i16> %data,
                                        <vscale x 2 x i16>* %base_store,
                                        i32 1,
                                        <vscale x 2 x i1> %mask)
   ret void
 }
 
 
 define void @test_masked_ldst_sv2i32(<vscale x 2 x i32> * %base, <vscale x 2 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0, #-8, mul vl]
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, #-8, mul vl]
 ; CHECK-NEXT:    st1w { z0.d }, p0, [x0, #-7, mul vl]
 ; CHECK-NEXT:    ret
   %base_load = getelementptr <vscale x 2 x i32>, <vscale x 2 x i32>* %base, i64 -8
   %data = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32(<vscale x 2 x i32>* %base_load,
                                                             i32 1,
                                                             <vscale x 2 x i1> %mask,
                                                             <vscale x 2 x i32> undef)
   %base_store = getelementptr <vscale x 2 x i32>, <vscale x 2 x i32> * %base, i64 -7
; CHECK-NEXT:    ret
   ret void
 }
 
 ; 4-lane contiguous load/stores.
 
 define void @test_masked_ldst_sv4i8(<vscale x 4 x i8> * %base, <vscale x 4 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sb { z0.s }, p0/z, [x0, #-1, mul vl]
+; CHECK-NEXT:    ld1b { z0.s }, p0/z, [x0, #-1, mul vl]
 ; CHECK-NEXT:    st1b { z0.s }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT:    ret
   %base_load = getelementptr <vscale x 4 x i8>, <vscale x 4 x i8>* %base, i64 -1
   %data = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8(<vscale x 4 x i8>* %base_load,
                                                           i32 1,
                                                           <vscale x 4 x i1> %mask,
                                                           <vscale x 4 x i8> undef)
   %base_store = getelementptr <vscale x 4 x i8>, <vscale x 4 x i8> * %base, i64 2
   call void @llvm.masked.store.nxv4i8(<vscale x 4 x i8> %data,
                                       <vscale x 4 x i8>* %base_store,
                                       i32 1,
                                       <vscale x 4 x i1> %mask)
   ret void
 }
 
 define void @test_masked_ldst_sv4i16(<vscale x 4 x i16> * %base, <vscale x 4 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sh { z0.s }, p0/z, [x0, #-1, mul vl]
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, #-1, mul vl]
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT:    ret
   %base_load = getelementptr <vscale x 4 x i16>, <vscale x 4 x i16>* %base, i64 -1
   %data = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16(<vscale x 4 x i16>* %base_load,
                                                             i32 1,
                                                             <vscale x 4 x i1> %mask,
                                                             <vscale x 4 x i16> undef)
   %base_store = getelementptr <vscale x 4 x i16>, <vscale x 4 x i16> * %base, i64 2
; CHECK-NEXT:    ret
   ret void
 }
 
 ; 8-lane contiguous load/stores.
 
 define void @test_masked_ldst_sv8i8(<vscale x 8 x i8> * %base, <vscale x 8 x i1> %mask) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sb { z0.h }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT:    ld1b { z0.h }, p0/z, [x0, #6, mul vl]
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x0, #7, mul vl]
 ; CHECK-NEXT:    ret
   %base_load = getelementptr <vscale x 8 x i8>, <vscale x 8 x i8>* %base, i64 6
   %data = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8(<vscale x 8 x i8>* %base_load,
                                                           i32 1,
                                                           <vscale x 8 x i1> %mask,
                                                           <vscale x 8 x i8> undef)
   %base_store = getelementptr <vscale x 8 x i8>, <vscale x 8 x i8> * %base, i64 7

llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-reg.ll

 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve --asm-verbose=false < %s | FileCheck %s
 
 ; 2-lane contiguous load/stores
 
 define void @test_masked_ldst_sv2i8(i8 * %base, <vscale x 2 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2i8:
-; CHECK-NEXT: ld1sb { z[[DATA:[0-9]+]].d }, p0/z, [x0, x1]
+; CHECK-NEXT: ld1b { z[[DATA:[0-9]+]].d }, p0/z, [x0, x1]
 ; CHECK-NEXT: st1b { z[[DATA]].d }, p0, [x0, x1]
 ; CHECK-NEXT: ret
   %base_i8 = getelementptr i8, i8* %base, i64 %offset
   %base_addr = bitcast i8* %base_i8 to <vscale x 2 x i8>*
   %data = call <vscale x 2 x i8> @llvm.masked.load.nxv2i8(<vscale x 2 x i8>* %base_addr,
                                                           i32 1,
                                                           <vscale x 2 x i1> %mask,
                                                           <vscale x 2 x i8> undef)
   call void @llvm.masked.store.nxv2i8(<vscale x 2 x i8> %data,
                                       <vscale x 2 x i8>* %base_addr,
                                       i32 1,
                                       <vscale x 2 x i1> %mask)
   ret void
 }
 
 define void @test_masked_ldst_sv2i16(i16 * %base, <vscale x 2 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2i16:
-; CHECK-NEXT: ld1sh { z[[DATA:[0-9]+]].d }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ld1h { z[[DATA:[0-9]+]].d }, p0/z, [x0, x1, lsl #1]
 ; CHECK-NEXT: st1h { z[[DATA]].d }, p0, [x0, x1, lsl #1]
 ; CHECK-NEXT: ret
   %base_i16 = getelementptr i16, i16* %base, i64 %offset
   %base_addr = bitcast i16* %base_i16 to <vscale x 2 x i16>*
   %data = call <vscale x 2 x i16> @llvm.masked.load.nxv2i16(<vscale x 2 x i16>* %base_addr,
                                                             i32 1,
                                                             <vscale x 2 x i1> %mask,
                                                             <vscale x 2 x i16> undef)
   call void @llvm.masked.store.nxv2i16(<vscale x 2 x i16> %data,
                                        <vscale x 2 x i16>* %base_addr,
                                        i32 1,
                                        <vscale x 2 x i1> %mask)
   ret void
 }
 
 define void @test_masked_ldst_sv2i32(i32 * %base, <vscale x 2 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv2i32:
-; CHECK-NEXT: ld1sw  { z0.d }, p0/z, [x0, x1, lsl #2]
+; CHECK-NEXT: ld1w  { z0.d }, p0/z, [x0, x1, lsl #2]
 ; CHECK-NEXT: st1w  { z0.d }, p0, [x0, x1, lsl #2]
 ; CHECK-NEXT: ret
   %base_i32 = getelementptr i32, i32* %base, i64 %offset
   %base_addr = bitcast i32* %base_i32 to <vscale x 2 x i32>*
   %data = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32(<vscale x 2 x i32>* %base_addr,
                                                             i32 1,
                                                             <vscale x 2 x i1> %mask,
                                                             <vscale x 2 x i32> undef)
call void @llvm.masked.store.nxv2i32(<vscale x 2 x i32> %trunc,
                                        <vscale x 2 x i1> %mask)
   ret void
 }
 
 ; 4-lane contiguous load/stores.
 
 define void @test_masked_ldst_sv4i8(i8 * %base, <vscale x 4 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv4i8:
-; CHECK-NEXT: ld1sb { z[[DATA:[0-9]+]].s }, p0/z, [x0, x1]
+; CHECK-NEXT: ld1b { z[[DATA:[0-9]+]].s }, p0/z, [x0, x1]
 ; CHECK-NEXT: st1b { z[[DATA]].s }, p0, [x0, x1]
 ; CHECK-NEXT: ret
   %base_i8 = getelementptr i8, i8* %base, i64 %offset
   %base_addr = bitcast i8* %base_i8 to <vscale x 4 x i8>*
   %data = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8(<vscale x 4 x i8>* %base_addr,
                                                           i32 1,
                                                           <vscale x 4 x i1> %mask,
                                                           <vscale x 4 x i8> undef)
   call void @llvm.masked.store.nxv4i8(<vscale x 4 x i8> %data,
                                       <vscale x 4 x i8>* %base_addr,
                                       i32 1,
                                       <vscale x 4 x i1> %mask)
   ret void
 }
 
 define void @test_masked_ldst_sv4i16(i16 * %base, <vscale x 4 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv4i16:
-; CHECK-NEXT: ld1sh { z[[DATA:[0-9]+]].s }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ld1h { z[[DATA:[0-9]+]].s }, p0/z, [x0, x1, lsl #1]
 ; CHECK-NEXT: st1h { z[[DATA]].s }, p0, [x0, x1, lsl #1]
 ; CHECK-NEXT: ret
   %base_i16 = getelementptr i16, i16* %base, i64 %offset
   %base_addr = bitcast i16* %base_i16 to <vscale x 4 x i16>*
   %data = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16(<vscale x 4 x i16>* %base_addr,
                                                             i32 1,
                                                             <vscale x 4 x i1> %mask,
                                                             <vscale x 4 x i16> undef)
call void @llvm.masked.store.nxv4i16(<vscale x 4 x i16> %trunc,
                                        <vscale x 4 x i1> %mask)
   ret void
 }
 
 ; 8-lane contiguous load/stores.
 
 define void @test_masked_ldst_sv8i8(i8 * %base, <vscale x 8 x i1> %mask, i64 %offset) nounwind {
 ; CHECK-LABEL: test_masked_ldst_sv8i8:
-; CHECK-NEXT: ld1sb { z[[DATA:[0-9]+]].h }, p0/z, [x0, x1]
+; CHECK-NEXT: ld1b { z[[DATA:[0-9]+]].h }, p0/z, [x0, x1]
 ; CHECK-NEXT: st1b { z[[DATA]].h }, p0, [x0, x1]
 ; CHECK-NEXT: ret
   %base_i8 = getelementptr i8, i8* %base, i64 %offset
   %base_addr = bitcast i8* %base_i8 to <vscale x 8 x i8>*
   %data = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8(<vscale x 8 x i8>* %base_addr,
                                                           i32 1,
                                                           <vscale x 8 x i1> %mask,
                                                           <vscale x 8 x i8> undef)

llvm/test/CodeGen/AArch64/sve-split-load.ll

; CHECK-NEXT:    ret
   ret <vscale x 16 x i64> %load
 }
 
 ; MASKED
 
 define <vscale x 2 x i32> @masked_load_promote_2i32(<vscale x 2 x i32> *%a, <vscale x 2 x i1> %pg) {
 ; CHECK-LABEL: masked_load_promote_2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %load = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32(<vscale x 2 x i32> *%a, i32 1, <vscale x 2 x i1> %pg, <vscale x 2 x i32> undef)
   ret <vscale x 2 x i32> %load
 }
 
 define <vscale x 32 x i8> @masked_load_split_32i8(<vscale x 32 x i8> *%a, <vscale x 32 x i1> %pg) {
 ; CHECK-LABEL: masked_load_split_32i8:
 ; CHECK:       // %bb.0: