[AArch64][SVE] Add DAG combine for load + dup_merge_passthru -> replicating load

[peterwaller-arm]

Summary

Since SVE replicating loads don't have post/pre increment addressing
modes, this is done as a DAG combine rather than an isel pattern so
that we can do this transform before post/pre increment are introduced.
As part of this, the existing replicating load patterns for splat have
been moved into this same DAG combine.

[peterwaller-arm]

LGTM with minor comments.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

20208

I replaced this return with an abort but don't see it fire for the check-llvm target. Is it possible to have a test for?

20218

Is this equivalent to SDValue(Load, 0).hasOneUse()?

[peterwaller-arm]

Thanks Bradley -- I'll continue this one in the near future though your input is welcomed.

[peterwaller-arm]

An alternate route to implementation landed via D139637, D139850.

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   LS64_EXTRACT,
 
   LD1_MERGE_ZERO,
   LD1S_MERGE_ZERO,
   LDNF1_MERGE_ZERO,
   LDNF1S_MERGE_ZERO,
   LDFF1_MERGE_ZERO,
   LDFF1S_MERGE_ZERO,
+  LD1R_MERGE_ZERO,
+  LD1RS_MERGE_ZERO,
   LD1RQ_MERGE_ZERO,
   LD1RO_MERGE_ZERO,
 
   // Structured loads.
   SVE_LD2_MERGE_ZERO,
   SVE_LD3_MERGE_ZERO,
   SVE_LD4_MERGE_ZERO,
 

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

#undef LCALLNAME5
   setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
                        ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
                        ISD::VECREDUCE_ADD, ISD::STEP_VECTOR});
 
   setTargetDAGCombine({ISD::MGATHER, ISD::MSCATTER});
 
   setTargetDAGCombine(ISD::FP_EXTEND);
 
+  setTargetDAGCombine(ISD::SPLAT_VECTOR);
+
   setTargetDAGCombine(ISD::GlobalAddress);
 
   // In case of strict alignment, avoid an excessive number of byte wide stores.
   MaxStoresPerMemsetOptSize = 8;
   MaxStoresPerMemset =
       Subtarget->requiresStrictAlign() ? MaxStoresPerMemsetOptSize : 32;
 
   MaxGluedStoresPerMemcpy = 4;
case AArch64ISD::FIRST_NUMBER:
     MAKE_CASE(AArch64ISD::PTEST_ANY)
     MAKE_CASE(AArch64ISD::PTRUE)
     MAKE_CASE(AArch64ISD::LD1_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::LD1S_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::LDNF1_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::LDNF1S_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::LDFF1_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::LDFF1S_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LD1R_MERGE_ZERO)
+    MAKE_CASE(AArch64ISD::LD1RS_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::LD1RQ_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::LD1RO_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::SVE_LD2_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::SVE_LD3_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::SVE_LD4_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::GLD1_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::GLD1_SCALED_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::GLD1_SXTW_MERGE_ZERO)
if (SDNode *LN = DCI.DAG.getNodeIfExists(
       return DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(LN, 0),
                              DCI.DAG.getConstant(0, DL, MVT::i64));
     }
   }
 
   return performPostLD1Combine(N, DCI, false);
 }
 
+static SDValue performDUPMergePassthruOrSplatCombine(SDNode *N,
+                                                     SelectionDAG &DAG) {
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+
+  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || !VT.isScalableVector() ||
+      VT.getVectorElementType() == MVT::i1)
+    return SDValue();
+
+  bool IsSplat = N->getOpcode() == ISD::SPLAT_VECTOR;
+  SDValue Pred =
+      IsSplat ? getPredicateForScalableVector(DAG, DL, VT) : N->getOperand(0);
+  SDValue Load = N->getOperand(IsSplat ? 0 : 1);
+
+  if (!IsSplat) {
+    SDValue Passthru = N->getOperand(2);
+
+    if (Passthru.getOpcode() != ISD::UNDEF &&
+        !isZerosVector(Passthru.getNode()) && !isAllActivePredicate(DAG, Pred))
+      return SDValue();
+  }
+
+  if (Load->getOpcode() != ISD::LOAD)
+    return SDValue();
+
+  LoadSDNode *LoadSDN = cast<LoadSDNode>(Load);
+  EVT MemVT = LoadSDN->getMemoryVT();
+  SDValue Offset = LoadSDN->getOffset();
+
+  unsigned Opcode = LoadSDN->getExtensionType() == ISD::SEXTLOAD
+                        ? AArch64ISD::LD1RS_MERGE_ZERO
+                        : AArch64ISD::LD1R_MERGE_ZERO;
+
+  if (Offset.isUndef())
+    Offset = DAG.getTargetConstant(0, DL, MVT::i64);
+
+  uint64_t Bytes = MemVT.getFixedSizeInBits() / 8;
+  ConstantSDNode *C = dyn_cast<ConstantSDNode>(Offset);
+  if (!C || C->getZExtValue() >= (1 << 7) * Bytes ||
+      C->getZExtValue() % Bytes != 0)
+    return SDValue();
+
+  // Check if there are other uses. If so, do not combine as it will introduce
+  // an extra load.
+  for (SDNode::use_iterator UI = Load->use_begin(), UE = Load->use_end();
+       UI != UE; ++UI) {
+    if (UI.getUse().getResNo() == 1) // Ignore uses of the chain result.
+      continue;
+    if (*UI != N)
+      return SDValue();
+  }
+
+  SDValue Ops[] = {LoadSDN->getChain(), Pred, LoadSDN->getBasePtr(), Offset,
+                   DAG.getValueType(getPackedSVEVectorVT(MemVT))};
+  SDValue NewLoad = DAG.getNode(Opcode, DL, {VT, MVT::Other}, Ops);
+  SDValue LoadChain = SDValue(NewLoad.getNode(), 1);
+
+  return DAG.getMergeValues({NewLoad, LoadChain}, DL);
+}
+
 /// Get rid of unnecessary NVCASTs (that don't change the type).
 static SDValue performNVCASTCombine(SDNode *N) {
   if (N->getValueType(0) == N->getOperand(0).getValueType())
     return N->getOperand(0);
 
   return SDValue();
 }
 
case ISD::MSTORE:
     return performMSTORECombine(N, DCI, DAG, Subtarget);
   case ISD::MGATHER:
   case ISD::MSCATTER:
     return performMaskedGatherScatterCombine(N, DCI, DAG);
   case ISD::VECTOR_SPLICE:
     return performSVESpliceCombine(N, DAG);
   case ISD::FP_EXTEND:
     return performFPExtendCombine(N, DAG, DCI, Subtarget);
+  case ISD::SPLAT_VECTOR:
+    return performDUPMergePassthruOrSplatCombine(N, DAG);
   case AArch64ISD::BRCOND:
     return performBRCONDCombine(N, DCI, DAG);
   case AArch64ISD::TBNZ:
   case AArch64ISD::TBZ:
     return performTBZCombine(N, DCI, DAG);
   case AArch64ISD::CSEL:
     return performCSELCombine(N, DCI, DAG);
   case AArch64ISD::DUP:
SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::VECREDUCE_ADD:
     return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
   case AArch64ISD::UADDV:
     return performUADDVCombine(N, DAG);
   case AArch64ISD::SMULL:
   case AArch64ISD::UMULL:
   case AArch64ISD::PMULL:
     return tryCombineLongOpWithDup(Intrinsic::not_intrinsic, N, DCI, DAG);
+  case AArch64ISD::DUP_MERGE_PASSTHRU:
+    return performDUPMergePassthruOrSplatCombine(N, DAG);
   case ISD::INTRINSIC_VOID:
   case ISD::INTRINSIC_W_CHAIN:
     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
     case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
       return combineSVEPrefetchVecBaseImmOff(N, DAG, 1 /*=ScalarSizeInBytes*/);
     case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
       return combineSVEPrefetchVecBaseImmOff(N, DAG, 2 /*=ScalarSizeInBytes*/);
     case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:

llvm/lib/Target/AArch64/AArch64InstrFormats.td

[{ return Imm >= 0 && Imm < (64*8) && ((Imm % 8) == 0); }]> {
   let ParserMatchClass = UImm6s8Operand;
 }
 def uimm6s16 : Operand<i64>, ImmLeaf<i64,
 [{ return Imm >= 0 && Imm < (64*16) && ((Imm % 16) == 0); }]> {
   let PrintMethod = "printImmScale<16>";
   let ParserMatchClass = UImm6s16Operand;
 }
 
+// tuimm6sN predicate - similiar to uimm6sN, but use TImmLeaf (TargetConstant)
+// instead of ImmLeaf (Constant)
+def tuimm6s1 : Operand<i64>, TImmLeaf<i64, [{ return Imm >= 0 && Imm < 64; }]> {
+  let ParserMatchClass = UImm6s1Operand;
+}
+def tuimm6s2 : Operand<i64>, TImmLeaf<i64,
+[{ return Imm >= 0 && Imm < (64*2) && ((Imm % 2) == 0); }]> {
+  let PrintMethod = "printImmScale<2>";
+  let ParserMatchClass = UImm6s2Operand;
+}
+def tuimm6s4 : Operand<i64>, TImmLeaf<i64,
+[{ return Imm >= 0 && Imm < (64*4) && ((Imm % 4) == 0); }]> {
+  let PrintMethod = "printImmScale<4>";
+  let ParserMatchClass = UImm6s4Operand;
+}
+def tuimm6s8 : Operand<i64>, TImmLeaf<i64,
+[{ return Imm >= 0 && Imm < (64*8) && ((Imm % 8) == 0); }]> {
+  let PrintMethod = "printImmScale<8>";
+  let ParserMatchClass = UImm6s8Operand;
+}
+
 def SImmS2XForm : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(N->getSExtValue() / 2, SDLoc(N), MVT::i64);
 }]>;
 def SImmS3XForm : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(N->getSExtValue() / 3, SDLoc(N), MVT::i64);
 }]>;
 def SImmS4XForm : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(N->getSExtValue() / 4, SDLoc(N), MVT::i64);

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 
 // Contiguous load and replicate - node definitions
 //
 
 def SDT_AArch64_LD1Replicate : SDTypeProfile<1, 2, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>,
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
+def SDT_AArch64_LD1Replicate_Imm : SDTypeProfile<1, 4, [
+  SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>, SDTCisInt<3>,
+  SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
+]>;
 
 def AArch64ld1rq_z : SDNode<"AArch64ISD::LD1RQ_MERGE_ZERO",  SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
 def AArch64ld1ro_z : SDNode<"AArch64ISD::LD1RO_MERGE_ZERO",  SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1r_z  : SDNode<"AArch64ISD::LD1R_MERGE_ZERO",   SDT_AArch64_LD1Replicate_Imm, [SDNPHasChain, SDNPMayLoad]>;
+def AArch64ld1rs_z : SDNode<"AArch64ISD::LD1RS_MERGE_ZERO",  SDT_AArch64_LD1Replicate_Imm, [SDNPHasChain, SDNPMayLoad]>;
 
 // Gather loads - node definitions
 //
 def SDT_AArch64_GATHER_SV : SDTypeProfile<1, 4, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisVT<4, OtherVT>,
   SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
 ]>;
 
let mayLoad = 1, hasSideEffects = 0 in {
     def LDR_ZZZZXI : Pseudo<(outs ZZZZ_b:$Zd), (ins GPR64sp:$sp, simm4s1:$offset),[]>, Sched<[]>;
   }
   let mayStore = 1, hasSideEffects = 0 in {
     def STR_ZZXI   : Pseudo<(outs), (ins   ZZ_b:$Zs, GPR64sp:$sp, simm4s1:$offset),[]>, Sched<[]>;
     def STR_ZZZXI  : Pseudo<(outs), (ins  ZZZ_b:$Zs, GPR64sp:$sp, simm4s1:$offset),[]>, Sched<[]>;
     def STR_ZZZZXI : Pseudo<(outs), (ins ZZZZ_b:$Zs, GPR64sp:$sp, simm4s1:$offset),[]>, Sched<[]>;
   }
 
+  multiclass LD1RPat<ValueType vt, SDPatternOperator operator, Instruction load_instr,
+                     ComplexPattern CP, Operand immtype, Operand timmtype, ValueType memvt> {
     let AddedComplexity = 1 in {
-  class LD1RPat<ValueType vt, SDPatternOperator operator,
-                Instruction load, Instruction ptrue, ValueType index_vt, ComplexPattern CP, Operand immtype> :
-        Pat<(vt (splat_vector (index_vt (operator (CP GPR64:$base, immtype:$offset))))),
-            (load (ptrue 31), GPR64:$base, $offset)>;
+      def : Pat<(vt (operator PPR:$pg, GPR64:$base, timmtype:$offset, memvt)),
+                (load_instr $pg, $base, $offset)>;
+      def : Pat<(vt (operator PPR:$pg, (CP GPR64:$base, immtype:$offset), (i64 0), memvt)),
+                (load_instr $pg, $base, $offset)>;
     }
-
-  // LDR1 of 8-bit data
-  def : LD1RPat<nxv16i8, extloadi8,  LD1RB_IMM,    PTRUE_B, i32, am_indexed8_6b, uimm6s1>;
-  def : LD1RPat<nxv8i16, zextloadi8, LD1RB_H_IMM,  PTRUE_H, i32, am_indexed8_6b, uimm6s1>;
-  def : LD1RPat<nxv4i32, zextloadi8, LD1RB_S_IMM,  PTRUE_S, i32, am_indexed8_6b, uimm6s1>;
-  def : LD1RPat<nxv2i64, zextloadi8, LD1RB_D_IMM,  PTRUE_D, i64, am_indexed8_6b, uimm6s1>;
-  def : LD1RPat<nxv8i16, sextloadi8, LD1RSB_H_IMM, PTRUE_H, i32, am_indexed8_6b, uimm6s1>;
-  def : LD1RPat<nxv4i32, sextloadi8, LD1RSB_S_IMM, PTRUE_S, i32, am_indexed8_6b, uimm6s1>;
-  def : LD1RPat<nxv2i64, sextloadi8, LD1RSB_D_IMM, PTRUE_D, i64, am_indexed8_6b, uimm6s1>;
-
-  // LDR1 of 16-bit data
-  def : LD1RPat<nxv8i16, extloadi16,  LD1RH_IMM,    PTRUE_H, i32, am_indexed16_6b, uimm6s2>;
-  def : LD1RPat<nxv4i32, zextloadi16, LD1RH_S_IMM,  PTRUE_S, i32, am_indexed16_6b, uimm6s2>;
-  def : LD1RPat<nxv2i64, zextloadi16, LD1RH_D_IMM,  PTRUE_D, i64, am_indexed16_6b, uimm6s2>;
-  def : LD1RPat<nxv4i32, sextloadi16, LD1RSH_S_IMM, PTRUE_S, i32, am_indexed16_6b, uimm6s2>;
-  def : LD1RPat<nxv2i64, sextloadi16, LD1RSH_D_IMM, PTRUE_D, i64, am_indexed16_6b, uimm6s2>;
-
-  // LDR1 of 32-bit data
-  def : LD1RPat<nxv4i32, load,        LD1RW_IMM,   PTRUE_S, i32, am_indexed32_6b, uimm6s4>;
-  def : LD1RPat<nxv2i64, zextloadi32, LD1RW_D_IMM, PTRUE_D, i64, am_indexed32_6b, uimm6s4>;
-  def : LD1RPat<nxv2i64, sextloadi32, LD1RSW_IMM,  PTRUE_D, i64, am_indexed32_6b, uimm6s4>;
+  }
+
+  // LD1R of 8-bit data
+  defm : LD1RPat<nxv16i8, AArch64ld1r_z,  LD1RB_IMM,    am_indexed8_6b,  uimm6s1, tuimm6s1, nxv16i8>;
+  defm : LD1RPat<nxv8i16, AArch64ld1r_z,  LD1RB_H_IMM,  am_indexed8_6b,  uimm6s1, tuimm6s1, nxv16i8>;
+  defm : LD1RPat<nxv4i32, AArch64ld1r_z,  LD1RB_S_IMM,  am_indexed8_6b,  uimm6s1, tuimm6s1, nxv16i8>;
+  defm : LD1RPat<nxv2i64, AArch64ld1r_z,  LD1RB_D_IMM,  am_indexed8_6b,  uimm6s1, tuimm6s1, nxv16i8>;
+  defm : LD1RPat<nxv8i16, AArch64ld1rs_z, LD1RSB_H_IMM, am_indexed8_6b,  uimm6s1, tuimm6s1, nxv16i8>;
+  defm : LD1RPat<nxv4i32, AArch64ld1rs_z, LD1RSB_S_IMM, am_indexed8_6b,  uimm6s1, tuimm6s1, nxv16i8>;
+  defm : LD1RPat<nxv2i64, AArch64ld1rs_z, LD1RSB_D_IMM, am_indexed8_6b,  uimm6s1, tuimm6s1, nxv16i8>;
+
+  // LD1R of 16-bit data
+  defm : LD1RPat<nxv8i16, AArch64ld1r_z,  LD1RH_IMM,    am_indexed16_6b, uimm6s2, tuimm6s2, nxv8i16>;
+  defm : LD1RPat<nxv4i32, AArch64ld1r_z,  LD1RH_S_IMM,  am_indexed16_6b, uimm6s2, tuimm6s2, nxv8i16>;
+  defm : LD1RPat<nxv2i64, AArch64ld1r_z,  LD1RH_D_IMM,  am_indexed16_6b, uimm6s2, tuimm6s2, nxv8i16>;
+  defm : LD1RPat<nxv4i32, AArch64ld1rs_z, LD1RSH_S_IMM, am_indexed16_6b, uimm6s2, tuimm6s2, nxv8i16>;
+  defm : LD1RPat<nxv2i64, AArch64ld1rs_z, LD1RSH_D_IMM, am_indexed16_6b, uimm6s2, tuimm6s2, nxv8i16>;
+
+  // LD1R of 32-bit data
+  defm : LD1RPat<nxv4i32, AArch64ld1r_z,  LD1RW_IMM,    am_indexed32_6b, uimm6s4, tuimm6s4, nxv4i32>;
+  defm : LD1RPat<nxv2i64, AArch64ld1r_z,  LD1RW_D_IMM,  am_indexed32_6b, uimm6s4, tuimm6s4, nxv4i32>;
+  defm : LD1RPat<nxv2i64, AArch64ld1rs_z, LD1RSW_IMM,   am_indexed32_6b, uimm6s4, tuimm6s4, nxv4i32>;
 
-  // LDR1 of 64-bit data
-  def : LD1RPat<nxv2i64, load, LD1RD_IMM, PTRUE_D, i64, am_indexed64_6b, uimm6s8>;
+  // LD1R of 64-bit data
+  defm : LD1RPat<nxv2i64, AArch64ld1r_z,  LD1RD_IMM,    am_indexed64_6b, uimm6s8, tuimm6s8, nxv2i64>;
 
   // LD1R of FP data
-  def : LD1RPat<nxv8f16, load, LD1RH_IMM,   PTRUE_H, f16, am_indexed16_6b, uimm6s2>;
-  def : LD1RPat<nxv4f16, load, LD1RH_S_IMM, PTRUE_S, f16, am_indexed16_6b, uimm6s2>;
-  def : LD1RPat<nxv2f16, load, LD1RH_D_IMM, PTRUE_D, f16, am_indexed16_6b, uimm6s2>;
-  def : LD1RPat<nxv4f32, load, LD1RW_IMM,   PTRUE_S, f32, am_indexed32_6b, uimm6s4>;
-  def : LD1RPat<nxv2f32, load, LD1RW_D_IMM, PTRUE_D, f32, am_indexed32_6b, uimm6s4>;
-  def : LD1RPat<nxv2f64, load, LD1RD_IMM,   PTRUE_D, f64, am_indexed64_6b, uimm6s8>;
+  defm : LD1RPat<nxv8bf16, AArch64ld1r_z, LD1RH_IMM,    am_indexed16_6b, uimm6s2, tuimm6s2, nxv8bf16>;
+  defm : LD1RPat<nxv4bf16, AArch64ld1r_z, LD1RH_S_IMM,  am_indexed16_6b, uimm6s2, tuimm6s2, nxv8bf16>;
+  defm : LD1RPat<nxv2bf16, AArch64ld1r_z, LD1RH_D_IMM,  am_indexed16_6b, uimm6s2, tuimm6s2, nxv8bf16>;
+  defm : LD1RPat<nxv8f16, AArch64ld1r_z,  LD1RH_IMM,    am_indexed16_6b, uimm6s2, tuimm6s2, nxv8f16>;
+  defm : LD1RPat<nxv4f16, AArch64ld1r_z,  LD1RH_S_IMM,  am_indexed16_6b, uimm6s2, tuimm6s2, nxv8f16>;
+  defm : LD1RPat<nxv2f16, AArch64ld1r_z,  LD1RH_D_IMM,  am_indexed16_6b, uimm6s2, tuimm6s2, nxv8f16>;
+  defm : LD1RPat<nxv4f32, AArch64ld1r_z,  LD1RW_IMM,    am_indexed32_6b, uimm6s4, tuimm6s4, nxv4f32>;
+  defm : LD1RPat<nxv2f32, AArch64ld1r_z,  LD1RW_D_IMM,  am_indexed32_6b, uimm6s4, tuimm6s4, nxv4f32>;
+  defm : LD1RPat<nxv2f64, AArch64ld1r_z,  LD1RD_IMM,    am_indexed64_6b, uimm6s8, tuimm6s8, nxv2f64>;
 
   // LD1R of 128-bit masked data
   multiclass ld1rq_pat<ValueType vt1, SDPatternOperator op, Instruction load_instr, ComplexPattern AddrCP>{
     def : Pat<(vt1 (AArch64ld1rq_z PPR:$gp, GPR64:$base)),
           (!cast<Instruction>(load_instr # _IMM) $gp, $base, (i64 0))>;
     let AddedComplexity = 2 in {
       def : Pat<(vt1 (op PPR:$gp, (add GPR64:$base, (i64 simm4s16:$imm)))),
             (!cast<Instruction>(load_instr # _IMM) $gp, $base, simm4s16:$imm)>;
     }
     def : Pat<(vt1 (op PPR:$gp, (AddrCP GPR64:$base, GPR64:$idx))),

llvm/test/CodeGen/AArch64/sve-ld1r.ll

 ; CHECK-NEXT:    ret
   %valp2 = getelementptr double, double* %valp, i32 -1
   %val = load double, double* %valp2
   %ins = insertelement <vscale x 2 x double> undef, double %val, i32 0
   %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   ret <vscale x 2 x double> %shf
 }
 
+define <vscale x 16 x i8> @ld1rb_dup(i8* %valp, <vscale x 16 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rb_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load i8, i8* %valp
+  %ret = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> %pg, i8 %val)
+  ret <vscale x 16 x i8> %ret
+}
+
+define <vscale x 8 x i16> @ld1rh_dup(i16* %valp, <vscale x 8 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rh_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load i16, i16* %valp
+  %ret = tail call <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, i16 %val)
+  ret <vscale x 8 x i16> %ret
+}
+
+define <vscale x 4 x i32> @ld1rw_dup(i32* %valp, <vscale x 4 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rw_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load i32, i32* %valp
+  %ret = tail call <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> %pg, i32 %val)
+  ret <vscale x 4 x i32> %ret
+}
+
+define <vscale x 2 x i64> @ld1rd_dup(i64* %valp, <vscale x 2 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rd_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load i64, i64* %valp
+  %ret = tail call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg, i64 %val)
+  ret <vscale x 2 x i64> %ret
+}
+
+define <vscale x 8 x bfloat> @ld1rh_bfloat_dup(bfloat* %valp, <vscale x 8 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rh_bfloat_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load bfloat, bfloat* %valp
+  %ret = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> %pg, bfloat %val)
+  ret <vscale x 8 x bfloat> %ret
+}
+
+define <vscale x 8 x half> @ld1rh_half_dup(half* %valp, <vscale x 8 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rh_half_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load half, half* %valp
+  %ret = tail call <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> %pg, half %val)
+  ret <vscale x 8 x half> %ret
+}
+
+define <vscale x 4 x float> @ld1rw_float_dup(float* %valp, <vscale x 4 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rw_float_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load float, float* %valp
+  %ret = tail call <vscale x 4 x float> @llvm.aarch64.sve.dup.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> %pg, float %val)
+  ret <vscale x 4 x float> %ret
+}
+
+define <vscale x 2 x double> @ld1rd_double_dup(double* %valp, <vscale x 2 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rd_double_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load double, double* %valp
+  %ret = tail call <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg, double %val)
+  ret <vscale x 2 x double> %ret
+}
+
+; Combine when passthru is zero
+define <vscale x 16 x i8> @ld1rb_dup_zero_pt(i8* %valp, <vscale x 16 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rb_dup_zero_pt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load i8, i8* %valp
+  %ret = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> %pg, i8 %val)
+  ret <vscale x 16 x i8> %ret
+}
+
+; Combine when passthru is non-zero/undef but pred is all active
+define <vscale x 16 x i8> @ld1rb_dup_pred_true(i8* %valp, <vscale x 16 x i8> %pt) #0 {
+; CHECK-LABEL: ld1rb_dup_pred_true:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %val = load i8, i8* %valp
+  %pg = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %ret = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8> %pt, <vscale x 16 x i1> %pg, i8 %val)
+  ret <vscale x 16 x i8> %ret
+}
+
+; Combine when load would otherwise be a post-inc
+define void @ld1rb_dup_postinc(i8* %valp, <vscale x 16 x i8>* %out, <vscale x 16 x i1> %pg) #0 {
+; CHECK-LABEL: ld1rb_dup_postinc:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #-2
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:  .LBB63_1: // %for.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [x0]
+; CHECK-NEXT:    add w8, w8, #2
+; CHECK-NEXT:    add x0, x0, #2
+; CHECK-NEXT:    cmp w8, #100
+; CHECK-NEXT:    st1b { z0.b }, p1, [x1]
+; CHECK-NEXT:    b.lo .LBB63_1
+; CHECK-NEXT:  // %bb.2: // %for.end
+; CHECK-NEXT:    ret
+entry:
+  br label %for.body
+
+for.body:
+  %inc = phi i32 [ 0, %entry ], [ %inc.next, %for.body ]
+  %ptr = getelementptr i8, i8* %valp, i32 %inc
+  %val = load i8, i8* %ptr
+  %load = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> %pg, i8 %val)
+  store volatile <vscale x 16 x i8> %load, <vscale x 16 x i8>* %out
+  %inc.next = add i32 %inc, 2
+  %cond = icmp uge i32 %inc, 100
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; Don't combine when passthru is non-zero/undef and pred is not all active
+define <vscale x 16 x i8> @ld1rb_dup_nonzero_pt(i8* %valp, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %pt) #0 {
+; CHECK-LABEL: ld1rb_dup_nonzero_pt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    mov z0.b, p0/m, w8
+; CHECK-NEXT:    ret
+  %val = load i8, i8* %valp
+  %ret = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8> %pt, <vscale x 16 x i1> %pg, i8 %val)
+  ret <vscale x 16 x i8> %ret
+}
+
 define <vscale x 2 x double> @dupq_ld1rqd_f64(<2 x double>* %a) {
 ; CHECK-LABEL: dupq_ld1rqd_f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ld1rqd { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %1 = load <2 x double>, <2 x double>* %a
   %2 = tail call fast <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> %1, i64 0)

 ; CHECK-NEXT:    ld1rqb { z0.b }, p0/z, [x0]
 ; CHECK-NEXT:    ret
   %1 = load <16 x i8>, <16 x i8>* %a
   %2 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> %1, i64 0)
   %3 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8> %2, i64 0)
   ret <vscale x 16 x i8> %3
 }
 
+declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32)
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64)
+declare <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dup.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat)
+declare <vscale x 4 x float> @llvm.aarch64.sve.dup.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float)
+declare <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double)
+
 declare <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8>, i64)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16>, i64)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32>, i64)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64>, i64)
 declare <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half>, i64)
 declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat>, i64)
 declare <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float>, i64)
 declare <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double>, i64)

llvm/test/tools/llvm-mca/AArch64/A64FX/A64FX-sve-instructions.s

 # CHECK-NEXT:  1      16    2.00    *             U     ld1h  { z31.d }, p7/z, [z31.d, #62]
 # CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z31.h }, p7/z, [sp, #-1, mul vl]
 # CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z31.s }, p7/z, [sp, #-1, mul vl]
 # CHECK-NEXT:  1      23    4.00    *             U     ld1h  { z31.s }, p7/z, [sp, z31.s, sxtw #1]
 # CHECK-NEXT:  1      23    4.00    *             U     ld1h  { z31.s }, p7/z, [sp, z31.s, uxtw #1]
 # CHECK-NEXT:  1      19    4.00    *             U     ld1h  { z31.s }, p7/z, [z31.s, #62]
 # CHECK-NEXT:  1      11    0.50    *                   ld1h  { z5.h }, p3/z, [sp, x16, lsl #1]
 # CHECK-NEXT:  1      11    0.50    *                   ld1h  { z5.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rb { z0.b }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rb { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rb { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rb { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rb { z31.b }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rb { z31.d }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rb { z31.h }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rb { z31.s }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rd { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rd { z31.d }, p7/z, [sp, #504]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rh { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rh { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rh { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rh { z31.d }, p7/z, [sp, #126]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rh { z31.h }, p7/z, [sp, #126]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rh { z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rb { z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rb { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rb { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rb { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rb { z31.b }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rb { z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rb { z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rb { z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rd { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rd { z31.d }, p7/z, [sp, #504]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rh { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rh { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rh { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rh { z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rh { z31.h }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rh { z31.s }, p7/z, [sp, #126]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqb  { z0.b }, p0/z, [x0, x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqb  { z0.b }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqb  { z21.b }, p5/z, [x10, #112]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqb  { z23.b }, p3/z, [x13, #-128]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqb  { z31.b }, p7/z, [sp, #-16]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqd  { z0.d }, p0/z, [x0, x0, lsl #3]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqd  { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqd  { z23.d }, p3/z, [x13, #-128]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqd  { z23.d }, p3/z, [x13, #112]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqd  { z31.d }, p7/z, [sp, #-16]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqh  { z0.h }, p0/z, [x0, x0, lsl #1]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqh  { z0.h }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqh  { z23.h }, p3/z, [x13, #-128]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqh  { z23.h }, p3/z, [x13, #112]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqh  { z31.h }, p7/z, [sp, #-16]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqw  { z0.s }, p0/z, [x0, x0, lsl #2]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqw  { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqw  { z23.s }, p3/z, [x13, #-128]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqw  { z23.s }, p3/z, [x13, #112]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rqw  { z31.s }, p7/z, [sp, #-16]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsb  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsb  { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsb  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsb  { z31.d }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsb  { z31.h }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsb  { z31.s }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsh  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsh  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsh  { z31.d }, p7/z, [sp, #126]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsh  { z31.s }, p7/z, [sp, #126]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsw  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rsw  { z31.d }, p7/z, [sp, #252]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rw { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rw { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rw { z31.d }, p7/z, [sp, #252]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1rw { z31.s }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsb  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsb  { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsb  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsb  { z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsb  { z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsb  { z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsh  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsh  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsh  { z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsh  { z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsw  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rsw  { z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rw { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rw { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rw { z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      11    0.50    *                   ld1rw { z31.s }, p7/z, [sp, #252]
 # CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      16    2.00    *             U     ld1sb { z0.d }, p0/z, [z0.d]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sb { z0.h }, p0/z, [sp, x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sb { z0.h }, p0/z, [x0, x0]
 # CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z0.h }, p0/z, [x0]
 # CHECK-NEXT:  1      23    4.00    *             U     ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
 # CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  1      19    4.00    *             U     ld1sb { z0.s }, p0/z, [z0.s]

llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s

 # CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z31.d }, p7/z, [z31.d, #62]
 # CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z31.h }, p7/z, [sp, #-1, mul vl]
 # CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z31.s }, p7/z, [sp, #-1, mul vl]
 # CHECK-NEXT:  4      10    1.00    *             U     ld1h  { z31.s }, p7/z, [sp, z31.s, sxtw #1]
 # CHECK-NEXT:  4      10    1.00    *             U     ld1h  { z31.s }, p7/z, [sp, z31.s, uxtw #1]
 # CHECK-NEXT:  2      9     0.50    *             U     ld1h  { z31.s }, p7/z, [z31.s, #62]
 # CHECK-NEXT:  1      6     0.50    *                   ld1h  { z5.h }, p3/z, [sp, x16, lsl #1]
 # CHECK-NEXT:  1      6     0.50    *                   ld1h  { z5.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rb { z0.b }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rb { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rb { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rb { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rb { z31.b }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rb { z31.d }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rb { z31.h }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rb { z31.s }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rd { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rd { z31.d }, p7/z, [sp, #504]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rh { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rh { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rh { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rh { z31.d }, p7/z, [sp, #126]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rh { z31.h }, p7/z, [sp, #126]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rh { z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rb { z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rb { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rb { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rb { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rb { z31.b }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rb { z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rb { z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rb { z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rd { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rd { z31.d }, p7/z, [sp, #504]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rh { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rh { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rh { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rh { z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rh { z31.h }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rh { z31.s }, p7/z, [sp, #126]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqb  { z0.b }, p0/z, [x0, x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqb  { z0.b }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqb  { z21.b }, p5/z, [x10, #112]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqb  { z23.b }, p3/z, [x13, #-128]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqb  { z31.b }, p7/z, [sp, #-16]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqd  { z0.d }, p0/z, [x0, x0, lsl #3]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqd  { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqd  { z23.d }, p3/z, [x13, #-128]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqd  { z23.d }, p3/z, [x13, #112]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqd  { z31.d }, p7/z, [sp, #-16]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqh  { z0.h }, p0/z, [x0, x0, lsl #1]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqh  { z0.h }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqh  { z23.h }, p3/z, [x13, #-128]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqh  { z23.h }, p3/z, [x13, #112]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqh  { z31.h }, p7/z, [sp, #-16]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqw  { z0.s }, p0/z, [x0, x0, lsl #2]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqw  { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqw  { z23.s }, p3/z, [x13, #-128]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqw  { z23.s }, p3/z, [x13, #112]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rqw  { z31.s }, p7/z, [sp, #-16]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsb  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsb  { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsb  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsb  { z31.d }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsb  { z31.h }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsb  { z31.s }, p7/z, [sp, #63]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsh  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsh  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsh  { z31.d }, p7/z, [sp, #126]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsh  { z31.s }, p7/z, [sp, #126]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsw  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rsw  { z31.d }, p7/z, [sp, #252]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rw { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rw { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rw { z31.d }, p7/z, [sp, #252]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1rw { z31.s }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsb  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsb  { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsb  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsb  { z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsb  { z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsb  { z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsh  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsh  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsh  { z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsh  { z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsw  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rsw  { z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rw { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rw { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rw { z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      6     0.33    *                   ld1rw { z31.s }, p7/z, [sp, #252]
 # CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  4      9     1.00    *             U     ld1sb { z0.d }, p0/z, [z0.d]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb { z0.h }, p0/z, [sp, x0]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb { z0.h }, p0/z, [x0, x0]
 # CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z0.h }, p0/z, [x0]
 # CHECK-NEXT:  2      9     0.50    *             U     ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
 # CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  2      9     0.50    *             U     ld1sb { z0.s }, p0/z, [z0.s]