[SVE] Remove AArch64ISD::ADD_PRED and AArch64ISD::SUB_PRED.

[paulwalker-arm]

Summary

These nodes provide an indirection that is not necessary because
SVE has unpredicated add/sub instructions and there's no downside
to using them for partial register operations. In fact, the test
changes show that unifying how fixed-length and scalable vector
add/sub are lowered enables better use of existing isel patterns.

[paulwalker-arm]

Updated CHECK line within srem_v16i16.

[sdesmalen]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   // ELF.
   THREAD_POINTER,
   ADC,
   SBC, // adc, sbc instructions
 
   // Predicated instructions where inactive lanes produce undefined results.
   ABDS_PRED,
   ABDU_PRED,
-  ADD_PRED,
   FADD_PRED,
   FDIV_PRED,
   FMA_PRED,
   FMAX_PRED,
   FMAXNM_PRED,
   FMIN_PRED,
   FMINNM_PRED,
   FMUL_PRED,
   FSUB_PRED,
   MUL_PRED,
   MULHS_PRED,
   MULHU_PRED,
   SDIV_PRED,
   SHL_PRED,
   SMAX_PRED,
   SMIN_PRED,
   SRA_PRED,
   SRL_PRED,
-  SUB_PRED,
   UDIV_PRED,
   UMAX_PRED,
   UMIN_PRED,
 
   // Unpredicated vector instructions
   BIC,
 
   SRAD_MERGE_OP1,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

case AArch64ISD::FIRST_NUMBER:
     MAKE_CASE(AArch64ISD::CSEL)
     MAKE_CASE(AArch64ISD::CSINV)
     MAKE_CASE(AArch64ISD::CSNEG)
     MAKE_CASE(AArch64ISD::CSINC)
     MAKE_CASE(AArch64ISD::THREAD_POINTER)
     MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
     MAKE_CASE(AArch64ISD::ABDS_PRED)
     MAKE_CASE(AArch64ISD::ABDU_PRED)
-    MAKE_CASE(AArch64ISD::ADD_PRED)
     MAKE_CASE(AArch64ISD::MUL_PRED)
     MAKE_CASE(AArch64ISD::MULHS_PRED)
     MAKE_CASE(AArch64ISD::MULHU_PRED)
     MAKE_CASE(AArch64ISD::SDIV_PRED)
     MAKE_CASE(AArch64ISD::SHL_PRED)
     MAKE_CASE(AArch64ISD::SMAX_PRED)
     MAKE_CASE(AArch64ISD::SMIN_PRED)
     MAKE_CASE(AArch64ISD::SRA_PRED)
     MAKE_CASE(AArch64ISD::SRL_PRED)
-    MAKE_CASE(AArch64ISD::SUB_PRED)
     MAKE_CASE(AArch64ISD::UDIV_PRED)
     MAKE_CASE(AArch64ISD::UMAX_PRED)
     MAKE_CASE(AArch64ISD::UMIN_PRED)
     MAKE_CASE(AArch64ISD::SRAD_MERGE_OP1)
     MAKE_CASE(AArch64ISD::FNEG_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::FCEIL_MERGE_PASSTHRU)
case ISD::TRUNCATE:
     return LowerTRUNCATE(Op, DAG);
   case ISD::MLOAD:
     return LowerMLOAD(Op, DAG);
   case ISD::LOAD:
     if (useSVEForFixedLengthVectorVT(Op.getValueType()))
       return LowerFixedLengthVectorLoadToSVE(Op, DAG);
     return LowerLOAD(Op, DAG);
   case ISD::ADD:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::ADD_PRED);
   case ISD::AND:
-    return LowerToScalableOp(Op, DAG);
   case ISD::SUB:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SUB_PRED);
+    return LowerToScalableOp(Op, DAG);
   case ISD::FMAXIMUM:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAX_PRED);
   case ISD::FMAXNUM:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMAXNM_PRED);
   case ISD::FMINIMUM:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMIN_PRED);
   case ISD::FMINNUM:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMINNM_PRED);

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

 ]>;
 
 def SDT_AArch64FMA : SDTypeProfile<1, 4, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, SDTCisVec<4>,
   SDTCVecEltisVT<1,i1>, SDTCisSameAs<0,2>, SDTCisSameAs<2,3>, SDTCisSameAs<3,4>
 ]>;
 
 // Predicated operations with the result of inactive lanes being unspecified.
-def AArch64add_p  : SDNode<"AArch64ISD::ADD_PRED",  SDT_AArch64Arith>;
 def AArch64asr_p  : SDNode<"AArch64ISD::SRA_PRED",  SDT_AArch64Arith>;
 def AArch64fadd_p : SDNode<"AArch64ISD::FADD_PRED", SDT_AArch64Arith>;
 def AArch64fdiv_p : SDNode<"AArch64ISD::FDIV_PRED", SDT_AArch64Arith>;
 def AArch64fma_p  : SDNode<"AArch64ISD::FMA_PRED",  SDT_AArch64FMA>;
 def AArch64fmax_p : SDNode<"AArch64ISD::FMAX_PRED", SDT_AArch64Arith>;
 def AArch64fmaxnm_p : SDNode<"AArch64ISD::FMAXNM_PRED", SDT_AArch64Arith>;
 def AArch64fmin_p : SDNode<"AArch64ISD::FMIN_PRED", SDT_AArch64Arith>;
 def AArch64fminnm_p : SDNode<"AArch64ISD::FMINNM_PRED", SDT_AArch64Arith>;
 def AArch64fmul_p : SDNode<"AArch64ISD::FMUL_PRED", SDT_AArch64Arith>;
 def AArch64fsub_p : SDNode<"AArch64ISD::FSUB_PRED", SDT_AArch64Arith>;
 def AArch64lsl_p  : SDNode<"AArch64ISD::SHL_PRED",  SDT_AArch64Arith>;
 def AArch64lsr_p  : SDNode<"AArch64ISD::SRL_PRED",  SDT_AArch64Arith>;
 def AArch64mul_p  : SDNode<"AArch64ISD::MUL_PRED",  SDT_AArch64Arith>;
 def AArch64sabd_p : SDNode<"AArch64ISD::ABDS_PRED", SDT_AArch64Arith>;
 def AArch64sdiv_p : SDNode<"AArch64ISD::SDIV_PRED", SDT_AArch64Arith>;
 def AArch64smax_p : SDNode<"AArch64ISD::SMAX_PRED", SDT_AArch64Arith>;
 def AArch64smin_p : SDNode<"AArch64ISD::SMIN_PRED", SDT_AArch64Arith>;
 def AArch64smulh_p : SDNode<"AArch64ISD::MULHS_PRED", SDT_AArch64Arith>;
-def AArch64sub_p  : SDNode<"AArch64ISD::SUB_PRED",  SDT_AArch64Arith>;
 def AArch64uabd_p : SDNode<"AArch64ISD::ABDU_PRED", SDT_AArch64Arith>;
 def AArch64udiv_p : SDNode<"AArch64ISD::UDIV_PRED", SDT_AArch64Arith>;
 def AArch64umax_p : SDNode<"AArch64ISD::UMAX_PRED", SDT_AArch64Arith>;
 def AArch64umin_p : SDNode<"AArch64ISD::UMIN_PRED", SDT_AArch64Arith>;
 def AArch64umulh_p : SDNode<"AArch64ISD::MULHU_PRED", SDT_AArch64Arith>;
 
 def SDT_AArch64Arith_Imm : SDTypeProfile<1, 3, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVT<3,i32>,
let Predicates = [HasSVEorStreamingSVE] in {
   defm AND_ZZZ : sve_int_bin_cons_log<0b00, "and", and>;
   defm ORR_ZZZ : sve_int_bin_cons_log<0b01, "orr", or>;
   defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>;
   defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>;
 
   defm ADD_ZPmZ  : sve_int_bin_pred_arit_0<0b000, "add",  "ADD_ZPZZ", int_aarch64_sve_add, DestructiveBinaryComm>;
   defm SUB_ZPmZ  : sve_int_bin_pred_arit_0<0b001, "sub",  "SUB_ZPZZ", int_aarch64_sve_sub, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">;
   defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", "SUBR_ZPZZ", int_aarch64_sve_subr, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>;
-
-  defm ADD_ZPZZ  : sve_int_bin_pred_bhsd<AArch64add_p>;
-  defm SUB_ZPZZ  : sve_int_bin_pred_bhsd<AArch64sub_p>;
 } // End HasSVEorStreamingSVE
 
 let Predicates = [HasSVEorStreamingSVE, UseExperimentalZeroingPseudos] in {
   defm ADD_ZPZZ  : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_add>;
   defm SUB_ZPZZ  : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_sub>;
   defm SUBR_ZPZZ : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_subr>;
 } // End HasSVEorStreamingSVE, UseExperimentalZeroingPseudos
 

llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll

; CHECK: ret
   ret <16 x i8> %res
 }
 
 define void @add_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 {
 ; CHECK-LABEL: add_v32i8:
 ; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,32)]]
 ; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK: add [[RES:z[0-9]+]].b, [[OP1]].b, [[OP2]].b
 ; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i8>, <32 x i8>* %a
   %op2 = load <32 x i8>, <32 x i8>* %b
   %res = add <32 x i8> %op1, %op2
   store <32 x i8> %res, <32 x i8>* %a
   ret void
 }
 
 define void @add_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 {
 ; CHECK-LABEL: add_v64i8:
 ; CHECK-DAG: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,64)]]
 ; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
-; CHECK-DAG: add [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK-DAG: add [[RES:z[0-9]+]].b, [[OP1]].b, [[OP2]].b
 ; CHECK-DAG: st1b { [[RES]].b }, [[PG]], [x0]
 ; VBITS_LE_256-DAG: mov w[[OFF_1:[0-9]+]], #[[#VBYTES]]
 ; VBITS_LE_256-DAG: ld1b { [[OP1_1:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_1]]]
 ; VBITS_LE_256-DAG: ld1b { [[OP2_1:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_1]]]
-; VBITS_LE_256-DAG: add [[RES_1:z[0-9]+]].b, [[PG]]/m, [[OP1_1]].b, [[OP2_1]].b
+; VBITS_LE_256-DAG: add [[RES_1:z[0-9]+]].b, [[OP1_1]].b, [[OP2_1]].b
 ; VBITS_LE_256-DAG: st1b { [[RES_1]].b }, [[PG]], [x0, x[[OFF_1]]]
 ; CHECK: ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %res = add <64 x i8> %op1, %op2
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 
 define void @add_v128i8(<128 x i8>* %a, <128 x i8>* %b) #0 {
 ; CHECK-LABEL: add_v128i8:
 ; CHECK-DAG: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,128)]]
 ; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
-; CHECK-DAG: add [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK-DAG: add [[RES:z[0-9]+]].b, [[OP1]].b, [[OP2]].b
 ; CHECK-DAG: st1b { [[RES]].b }, [[PG]], [x0]
 ; VBITS_LE_512-DAG: mov w[[OFF_1:[0-9]+]], #[[#VBYTES]]
 ; VBITS_LE_512-DAG: ld1b { [[OP1_1:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_1]]]
 ; VBITS_LE_512-DAG: ld1b { [[OP2_1:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_1]]]
-; VBITS_LE_512-DAG: add [[RES_1:z[0-9]+]].b, [[PG]]/m, [[OP1_1]].b, [[OP2_1]].b
+; VBITS_LE_512-DAG: add [[RES_1:z[0-9]+]].b, [[OP1_1]].b, [[OP2_1]].b
 ; VBITS_LE_512-DAG: st1b { [[RES_1]].b }, [[PG]], [x0, x[[OFF_1]]]
 ; VBITS_LE_256-DAG: mov w[[OFF_2:[0-9]+]], #[[#mul(VBYTES,2)]]
 ; VBITS_LE_256-DAG: ld1b { [[OP1_2:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_2]]]
 ; VBITS_LE_256-DAG: ld1b { [[OP2_2:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_2]]]
-; VBITS_LE_256-DAG: add [[RES_2:z[0-9]+]].b, [[PG]]/m, [[OP1_2]].b, [[OP2_2]].b
+; VBITS_LE_256-DAG: add [[RES_2:z[0-9]+]].b, [[OP1_2]].b, [[OP2_2]].b
 ; VBITS_LE_256-DAG: st1b { [[RES_2]].b }, [[PG]], [x0, x[[OFF_2]]]
 ; VBITS_LE_256-DAG: mov w[[OFF_3:[0-9]+]], #[[#mul(VBYTES,3)]]
 ; VBITS_LE_256-DAG: ld1b { [[OP1_3:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_3]]]
 ; VBITS_LE_256-DAG: ld1b { [[OP2_3:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_3]]]
-; VBITS_LE_256-DAG: add [[RES_3:z[0-9]+]].b, [[PG]]/m, [[OP1_3]].b, [[OP2_3]].b
+; VBITS_LE_256-DAG: add [[RES_3:z[0-9]+]].b, [[OP1_3]].b, [[OP2_3]].b
 ; VBITS_LE_256-DAG: st1b { [[RES_3]].b }, [[PG]], [x0, x[[OFF_3]]]
 ; CHECK: ret
   %op1 = load <128 x i8>, <128 x i8>* %a
   %op2 = load <128 x i8>, <128 x i8>* %b
   %res = add <128 x i8> %op1, %op2
   store <128 x i8> %res, <128 x i8>* %a
   ret void
 }
 
 define void @add_v256i8(<256 x i8>* %a, <256 x i8>* %b) #0 {
 ; CHECK-LABEL: add_v256i8:
 ; CHECK-DAG: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,256)]]
 ; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
-; CHECK-DAG: add [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK-DAG: add [[RES:z[0-9]+]].b, [[OP1]].b, [[OP2]].b
 ; CHECK-DAG: st1b { [[RES]].b }, [[PG]], [x0]
 ; VBITS_LE_1024-DAG: mov w[[OFF_1:[0-9]+]], #[[#VBYTES]]
 ; VBITS_LE_1024-DAG: ld1b { [[OP1_1:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_1]]]
 ; VBITS_LE_1024-DAG: ld1b { [[OP2_1:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_1]]]
-; VBITS_LE_1024-DAG: add [[RES_1:z[0-9]+]].b, [[PG]]/m, [[OP1_1]].b, [[OP2_1]].b
+; VBITS_LE_1024-DAG: add [[RES_1:z[0-9]+]].b, [[OP1_1]].b, [[OP2_1]].b
 ; VBITS_LE_1024-DAG: st1b { [[RES_1]].b }, [[PG]], [x0, x[[OFF_1]]]
 ; VBITS_LE_512-DAG: mov w[[OFF_2:[0-9]+]], #[[#mul(VBYTES,2)]]
 ; VBITS_LE_512-DAG: ld1b { [[OP1_2:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_2]]]
 ; VBITS_LE_512-DAG: ld1b { [[OP2_2:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_2]]]
-; VBITS_LE_512-DAG: add [[RES_2:z[0-9]+]].b, [[PG]]/m, [[OP1_2]].b, [[OP2_2]].b
+; VBITS_LE_512-DAG: add [[RES_2:z[0-9]+]].b, [[OP1_2]].b, [[OP2_2]].b
 ; VBITS_LE_512-DAG: st1b { [[RES_2]].b }, [[PG]], [x0, x[[OFF_2]]]
 ; VBITS_LE_512-DAG: mov w[[OFF_3:[0-9]+]], #[[#mul(VBYTES,3)]]
 ; VBITS_LE_512-DAG: ld1b { [[OP1_3:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_3]]]
 ; VBITS_LE_512-DAG: ld1b { [[OP2_3:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_3]]]
-; VBITS_LE_512-DAG: add [[RES_3:z[0-9]+]].b, [[PG]]/m, [[OP1_3]].b, [[OP2_3]].b
+; VBITS_LE_512-DAG: add [[RES_3:z[0-9]+]].b, [[OP1_3]].b, [[OP2_3]].b
 ; VBITS_LE_512-DAG: st1b { [[RES_3]].b }, [[PG]], [x0, x[[OFF_3]]]
 ; VBITS_LE_256-DAG: mov w[[OFF_4:[0-9]+]], #[[#mul(VBYTES,4)]]
 ; VBITS_LE_256-DAG: ld1b { [[OP1_4:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_4]]]
 ; VBITS_LE_256-DAG: ld1b { [[OP2_4:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_4]]]
-; VBITS_LE_256-DAG: add [[RES_4:z[0-9]+]].b, [[PG]]/m, [[OP1_4]].b, [[OP2_4]].b
+; VBITS_LE_256-DAG: add [[RES_4:z[0-9]+]].b, [[OP1_4]].b, [[OP2_4]].b
 ; VBITS_LE_256-DAG: st1b { [[RES_4]].b }, [[PG]], [x0, x[[OFF_4]]]
 ; VBITS_LE_256-DAG: mov w[[OFF_5:[0-9]+]], #[[#mul(VBYTES,5)]]
 ; VBITS_LE_256-DAG: ld1b { [[OP1_5:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_5]]]
 ; VBITS_LE_256-DAG: ld1b { [[OP2_5:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_5]]]
-; VBITS_LE_256-DAG: add [[RES_5:z[0-9]+]].b, [[PG]]/m, [[OP1_5]].b, [[OP2_5]].b
+; VBITS_LE_256-DAG: add [[RES_5:z[0-9]+]].b, [[OP1_5]].b, [[OP2_5]].b
 ; VBITS_LE_256-DAG: st1b { [[RES_5]].b }, [[PG]], [x0, x[[OFF_5]]]
 ; VBITS_LE_256-DAG: mov w[[OFF_6:[0-9]+]], #[[#mul(VBYTES,6)]]
 ; VBITS_LE_256-DAG: ld1b { [[OP1_6:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_6]]]
 ; VBITS_LE_256-DAG: ld1b { [[OP2_6:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_6]]]
-; VBITS_LE_256-DAG: add [[RES_6:z[0-9]+]].b, [[PG]]/m, [[OP1_6]].b, [[OP2_6]].b
+; VBITS_LE_256-DAG: add [[RES_6:z[0-9]+]].b, [[OP1_6]].b, [[OP2_6]].b
 ; VBITS_LE_256-DAG: st1b { [[RES_6]].b }, [[PG]], [x0, x[[OFF_6]]]
 ; VBITS_LE_256-DAG: mov w[[OFF_7:[0-9]+]], #[[#mul(VBYTES,7)]]
 ; VBITS_LE_256-DAG: ld1b { [[OP1_7:z[0-9]+]].b }, [[PG]]/z, [x0, x[[OFF_7]]]
 ; VBITS_LE_256-DAG: ld1b { [[OP2_7:z[0-9]+]].b }, [[PG]]/z, [x1, x[[OFF_7]]]
-; VBITS_LE_256-DAG: add [[RES_7:z[0-9]+]].b, [[PG]]/m, [[OP1_7]].b, [[OP2_7]].b
+; VBITS_LE_256-DAG: add [[RES_7:z[0-9]+]].b, [[OP1_7]].b, [[OP2_7]].b
 ; VBITS_LE_256-DAG: st1b { [[RES_7]].b }, [[PG]], [x0, x[[OFF_7]]]
 ; CHECK: ret
   %op1 = load <256 x i8>, <256 x i8>* %a
   %op2 = load <256 x i8>, <256 x i8>* %b
   %res = add <256 x i8> %op1, %op2
   store <256 x i8> %res, <256 x i8>* %a
   ret void
 }
; CHECK: ret
   ret <8 x i16> %res
 }
 
 define void @add_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 {
 ; CHECK-LABEL: add_v16i16:
 ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
 ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: add [[RES:z[0-9]+]].h, [[OP1]].h, [[OP2]].h
 ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <16 x i16>, <16 x i16>* %a
   %op2 = load <16 x i16>, <16 x i16>* %b
   %res = add <16 x i16> %op1, %op2
   store <16 x i16> %res, <16 x i16>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 {
 ; CHECK-LABEL: add_v32i16:
 ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
 ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: add [[RES:z[0-9]+]].h, [[OP1]].h, [[OP2]].h
 ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %res = add <32 x i16> %op1, %op2
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v64i16(<64 x i16>* %a, <64 x i16>* %b) #0 {
 ; CHECK-LABEL: add_v64i16:
 ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
 ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: add [[RES:z[0-9]+]].h, [[OP1]].h, [[OP2]].h
 ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i16>, <64 x i16>* %a
   %op2 = load <64 x i16>, <64 x i16>* %b
   %res = add <64 x i16> %op1, %op2
   store <64 x i16> %res, <64 x i16>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v128i16(<128 x i16>* %a, <128 x i16>* %b) #0 {
 ; CHECK-LABEL: add_v128i16:
 ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
 ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: add [[RES:z[0-9]+]].h, [[OP1]].h, [[OP2]].h
 ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <128 x i16>, <128 x i16>* %a
   %op2 = load <128 x i16>, <128 x i16>* %b
   %res = add <128 x i16> %op1, %op2
   store <128 x i16> %res, <128 x i16>* %a
   ret void
 }
; CHECK: ret
   ret <4 x i32> %res
 }
 
 define void @add_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
 ; CHECK-LABEL: add_v8i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
 ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: add [[RES:z[0-9]+]].s, [[OP1]].s, [[OP2]].s
 ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <8 x i32>, <8 x i32>* %a
   %op2 = load <8 x i32>, <8 x i32>* %b
   %res = add <8 x i32> %op1, %op2
   store <8 x i32> %res, <8 x i32>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 {
 ; CHECK-LABEL: add_v16i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
 ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: add [[RES:z[0-9]+]].s, [[OP1]].s, [[OP2]].s
 ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = add <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 {
 ; CHECK-LABEL: add_v32i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
 ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: add [[RES:z[0-9]+]].s, [[OP1]].s, [[OP2]].s
 ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i32>, <32 x i32>* %a
   %op2 = load <32 x i32>, <32 x i32>* %b
   %res = add <32 x i32> %op1, %op2
   store <32 x i32> %res, <32 x i32>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v64i32(<64 x i32>* %a, <64 x i32>* %b) #0 {
 ; CHECK-LABEL: add_v64i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
 ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: add [[RES:z[0-9]+]].s, [[OP1]].s, [[OP2]].s
 ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i32>, <64 x i32>* %a
   %op2 = load <64 x i32>, <64 x i32>* %b
   %res = add <64 x i32> %op1, %op2
   store <64 x i32> %res, <64 x i32>* %a
   ret void
 }
; CHECK: ret
   ret <2 x i64> %res
 }
 
 define void @add_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 {
 ; CHECK-LABEL: add_v4i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
 ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: add [[RES:z[0-9]+]].d, [[OP1]].d, [[OP2]].d
 ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <4 x i64>, <4 x i64>* %a
   %op2 = load <4 x i64>, <4 x i64>* %b
   %res = add <4 x i64> %op1, %op2
   store <4 x i64> %res, <4 x i64>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 {
 ; CHECK-LABEL: add_v8i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
 ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: add [[RES:z[0-9]+]].d, [[OP1]].d, [[OP2]].d
 ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <8 x i64>, <8 x i64>* %a
   %op2 = load <8 x i64>, <8 x i64>* %b
   %res = add <8 x i64> %op1, %op2
   store <8 x i64> %res, <8 x i64>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v16i64(<16 x i64>* %a, <16 x i64>* %b) #0 {
 ; CHECK-LABEL: add_v16i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
 ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: add [[RES:z[0-9]+]].d, [[OP1]].d, [[OP2]].d
 ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <16 x i64>, <16 x i64>* %a
   %op2 = load <16 x i64>, <16 x i64>* %b
   %res = add <16 x i64> %op1, %op2
   store <16 x i64> %res, <16 x i64>* %a
   ret void
 }
 
 ; NOTE: Check lines only cover the first VBYTES because the add_v#i8 tests
 ; already cover the general legalisation cases.
 define void @add_v32i64(<32 x i64>* %a, <32 x i64>* %b) #0 {
 ; CHECK-LABEL: add_v32i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
 ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
-; CHECK: add [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: add [[RES:z[0-9]+]].d, [[OP1]].d, [[OP2]].d
 ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i64>, <32 x i64>* %a
   %op2 = load <32 x i64>, <32 x i64>* %b
   %res = add <32 x i64> %op1, %op2
   store <32 x i64> %res, <32 x i64>* %a
   ret void
 }
; CHECK: ret
   ret <16 x i8> %res
 }
 
 define void @sub_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 {
 ; CHECK-LABEL: sub_v32i8:
 ; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,32)]]
 ; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK: sub [[RES:z[0-9]+]].b, [[OP1]].b, [[OP2]].b
 ; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i8>, <32 x i8>* %a
   %op2 = load <32 x i8>, <32 x i8>* %b
   %res = sub <32 x i8> %op1, %op2
   store <32 x i8> %res, <32 x i8>* %a
   ret void
 }
 
 define void @sub_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 {
 ; CHECK-LABEL: sub_v64i8:
 ; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,64)]]
 ; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK: sub [[RES:z[0-9]+]].b, [[OP1]].b, [[OP2]].b
 ; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %res = sub <64 x i8> %op1, %op2
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 
 define void @sub_v128i8(<128 x i8>* %a, <128 x i8>* %b) #0 {
 ; CHECK-LABEL: sub_v128i8:
 ; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,128)]]
 ; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK: sub [[RES:z[0-9]+]].b, [[OP1]].b, [[OP2]].b
 ; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <128 x i8>, <128 x i8>* %a
   %op2 = load <128 x i8>, <128 x i8>* %b
   %res = sub <128 x i8> %op1, %op2
   store <128 x i8> %res, <128 x i8>* %a
   ret void
 }
 
 define void @sub_v256i8(<256 x i8>* %a, <256 x i8>* %b) #0 {
 ; CHECK-LABEL: sub_v256i8:
 ; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,256)]]
 ; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
+; CHECK: sub [[RES:z[0-9]+]].b, [[OP1]].b, [[OP2]].b
 ; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <256 x i8>, <256 x i8>* %a
   %op2 = load <256 x i8>, <256 x i8>* %b
   %res = sub <256 x i8> %op1, %op2
   store <256 x i8> %res, <256 x i8>* %a
   ret void
 }
; CHECK: ret
   ret <8 x i16> %res
 }
 
 define void @sub_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 {
 ; CHECK-LABEL: sub_v16i16:
 ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
 ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: sub [[RES:z[0-9]+]].h, [[OP1]].h, [[OP2]].h
 ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <16 x i16>, <16 x i16>* %a
   %op2 = load <16 x i16>, <16 x i16>* %b
   %res = sub <16 x i16> %op1, %op2
   store <16 x i16> %res, <16 x i16>* %a
   ret void
 }
 
 define void @sub_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 {
 ; CHECK-LABEL: sub_v32i16:
 ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
 ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: sub [[RES:z[0-9]+]].h, [[OP1]].h, [[OP2]].h
 ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %res = sub <32 x i16> %op1, %op2
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }
 
 define void @sub_v64i16(<64 x i16>* %a, <64 x i16>* %b) #0 {
 ; CHECK-LABEL: sub_v64i16:
 ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
 ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: sub [[RES:z[0-9]+]].h, [[OP1]].h, [[OP2]].h
 ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i16>, <64 x i16>* %a
   %op2 = load <64 x i16>, <64 x i16>* %b
   %res = sub <64 x i16> %op1, %op2
   store <64 x i16> %res, <64 x i16>* %a
   ret void
 }
 
 define void @sub_v128i16(<128 x i16>* %a, <128 x i16>* %b) #0 {
 ; CHECK-LABEL: sub_v128i16:
 ; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
 ; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: sub [[RES:z[0-9]+]].h, [[OP1]].h, [[OP2]].h
 ; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <128 x i16>, <128 x i16>* %a
   %op2 = load <128 x i16>, <128 x i16>* %b
   %res = sub <128 x i16> %op1, %op2
   store <128 x i16> %res, <128 x i16>* %a
   ret void
 }
; CHECK: ret
   ret <4 x i32> %res
 }
 
 define void @sub_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
 ; CHECK-LABEL: sub_v8i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
 ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: sub [[RES:z[0-9]+]].s, [[OP1]].s, [[OP2]].s
 ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <8 x i32>, <8 x i32>* %a
   %op2 = load <8 x i32>, <8 x i32>* %b
   %res = sub <8 x i32> %op1, %op2
   store <8 x i32> %res, <8 x i32>* %a
   ret void
 }
 
 define void @sub_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 {
 ; CHECK-LABEL: sub_v16i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
 ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: sub [[RES:z[0-9]+]].s, [[OP1]].s, [[OP2]].s
 ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = sub <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 define void @sub_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 {
 ; CHECK-LABEL: sub_v32i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
 ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: sub [[RES:z[0-9]+]].s, [[OP1]].s, [[OP2]].s
 ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i32>, <32 x i32>* %a
   %op2 = load <32 x i32>, <32 x i32>* %b
   %res = sub <32 x i32> %op1, %op2
   store <32 x i32> %res, <32 x i32>* %a
   ret void
 }
 
 define void @sub_v64i32(<64 x i32>* %a, <64 x i32>* %b) #0 {
 ; CHECK-LABEL: sub_v64i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
 ; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: sub [[RES:z[0-9]+]].s, [[OP1]].s, [[OP2]].s
 ; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i32>, <64 x i32>* %a
   %op2 = load <64 x i32>, <64 x i32>* %b
   %res = sub <64 x i32> %op1, %op2
   store <64 x i32> %res, <64 x i32>* %a
   ret void
 }
; CHECK: ret
   ret <2 x i64> %res
 }
 
 define void @sub_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 {
 ; CHECK-LABEL: sub_v4i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
 ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: sub [[RES:z[0-9]+]].d, [[OP1]].d, [[OP2]].d
 ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <4 x i64>, <4 x i64>* %a
   %op2 = load <4 x i64>, <4 x i64>* %b
   %res = sub <4 x i64> %op1, %op2
   store <4 x i64> %res, <4 x i64>* %a
   ret void
 }
 
 define void @sub_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 {
 ; CHECK-LABEL: sub_v8i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
 ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: sub [[RES:z[0-9]+]].d, [[OP1]].d, [[OP2]].d
 ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <8 x i64>, <8 x i64>* %a
   %op2 = load <8 x i64>, <8 x i64>* %b
   %res = sub <8 x i64> %op1, %op2
   store <8 x i64> %res, <8 x i64>* %a
   ret void
 }
 
 define void @sub_v16i64(<16 x i64>* %a, <16 x i64>* %b) #0 {
 ; CHECK-LABEL: sub_v16i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
 ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: sub [[RES:z[0-9]+]].d, [[OP1]].d, [[OP2]].d
 ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <16 x i64>, <16 x i64>* %a
   %op2 = load <16 x i64>, <16 x i64>* %b
   %res = sub <16 x i64> %op1, %op2
   store <16 x i64> %res, <16 x i64>* %a
   ret void
 }
 
 define void @sub_v32i64(<32 x i64>* %a, <32 x i64>* %b) #0 {
 ; CHECK-LABEL: sub_v32i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
 ; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
-; CHECK: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: sub [[RES:z[0-9]+]].d, [[OP1]].d, [[OP2]].d
 ; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i64>, <32 x i64>* %a
   %op2 = load <32 x i64>, <32 x i64>* %b
   %res = sub <32 x i64> %op1, %op2
   store <32 x i64> %res, <32 x i64>* %a
   ret void
 }

llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll

; CHECK-NEXT: ret
   %b = sext <16 x i8> %a to <16 x i16>
   store <16 x i16>%b, <16 x i16>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the extend being combined with the load.
 define void @sext_v32i8_v32i16(<32 x i8>* %in, <32 x i16>* %out) #0 {
 ; CHECK-LABEL: sext_v32i8_v32i16:
-; VBITS_GE_512: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_512-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_512-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
 ; VBITS_GE_512-NEXT: ret
   %a = load <32 x i8>, <32 x i8>* %in
   %b = add <32 x i8> %a, %a
   %c = sext <32 x i8> %b to <32 x i16>
   store <32 x i16> %c, <32 x i16>* %out
   ret void
 }
 
 define void @sext_v64i8_v64i16(<64 x i8>* %in, <64 x i16>* %out) #0 {
 ; CHECK-LABEL: sext_v64i8_v64i16:
-; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_1024-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_1024-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <64 x i8>, <64 x i8>* %in
   %b = add <64 x i8> %a, %a
   %c = sext <64 x i8> %b to <64 x i16>
   store <64 x i16> %c, <64 x i16>* %out
   ret void
 }
 
 define void @sext_v128i8_v128i16(<128 x i8>* %in, <128 x i16>* %out) #0 {
 ; CHECK-LABEL: sext_v128i8_v128i16:
-; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_2048-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <128 x i8>, <128 x i8>* %in
   %b = add <128 x i8> %a, %a
   %c = sext <128 x i8> %b to <128 x i16>
   store <128 x i16> %c, <128 x i16>* %out
   ret void

 ; VBITS_EQ_256-NEXT: ret
   %b = sext <16 x i8> %a to <16 x i32>
   store <16 x i32> %b, <16 x i32>* %out
   ret void
 }
 
 define void @sext_v32i8_v32i32(<32 x i8>* %in, <32 x i32>* %out) #0 {
 ; CHECK-LABEL: sext_v32i8_v32i32:
-; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_1024-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <32 x i8>, <32 x i8>* %in
   %b = add <32 x i8> %a, %a
   %c = sext <32 x i8> %b to <32 x i32>
   store <32 x i32> %c, <32 x i32>* %out
   ret void
 }
 
 define void @sext_v64i8_v64i32(<64 x i8>* %in, <64 x i32>* %out) #0 {
 ; CHECK-LABEL: sext_v64i8_v64i32:
-; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <64 x i8>, <64 x i8>* %in
   %b = add <64 x i8> %a, %a
   %c = sext <64 x i8> %b to <64 x i32>
   store <64 x i32> %c, <64 x i32>* %out

 ; VBITS_GE_1024-NEXT: ret
   %b = sext <16 x i8> %a to <16 x i64>
   store <16 x i64> %b, <16 x i64>* %out
   ret void
 }
 
 define void @sext_v32i8_v32i64(<32 x i8>* %in, <32 x i64>* %out) #0 {
 ; CHECK-LABEL: sext_v32i8_v32i64:
-; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <32 x i8>, <32 x i8>* %in
   %b = add <32 x i8> %a, %a
   %c = sext <32 x i8> %b to <32 x i64>

 ; CHECK-NEXT: ret
   %b = sext <8 x i16> %a to <8 x i32>
   store <8 x i32>%b, <8 x i32>* %out
   ret void
 }
 
 define void @sext_v16i16_v16i32(<16 x i16>* %in, <16 x i32>* %out) #0 {
 ; CHECK-LABEL: sext_v16i16_v16i32:
-; VBITS_GE_512: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_512-NEXT: ret
   %a = load <16 x i16>, <16 x i16>* %in
   %b = add <16 x i16> %a, %a
   %c = sext <16 x i16> %b to <16 x i32>
   store <16 x i32> %c, <16 x i32>* %out
   ret void
 }
 
 define void @sext_v32i16_v32i32(<32 x i16>* %in, <32 x i32>* %out) #0 {
 ; CHECK-LABEL: sext_v32i16_v32i32:
-; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <32 x i16>, <32 x i16>* %in
   %b = add <32 x i16> %a, %a
   %c = sext <32 x i16> %b to <32 x i32>
   store <32 x i32> %c, <32 x i32>* %out
   ret void
 }
 
 define void @sext_v64i16_v64i32(<64 x i16>* %in, <64 x i32>* %out) #0 {
 ; CHECK-LABEL: sext_v64i16_v64i32:
-; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <64 x i16>, <64 x i16>* %in
   %b = add <64 x i16> %a, %a
   %c = sext <64 x i16> %b to <64 x i32>
   store <64 x i32> %c, <64 x i32>* %out
   ret void

 ; VBITS_GE_512-NEXT: ret
   %b = sext <8 x i16> %a to <8 x i64>
   store <8 x i64>%b, <8 x i64>* %out
   ret void
 }
 
 define void @sext_v16i16_v16i64(<16 x i16>* %in, <16 x i64>* %out) #0 {
 ; CHECK-LABEL: sext_v16i16_v16i64:
-; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_1024-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <16 x i16>, <16 x i16>* %in
   %b = add <16 x i16> %a, %a
   %c = sext <16 x i16> %b to <16 x i64>
   store <16 x i64> %c, <16 x i64>* %out
   ret void
 }
 
 define void @sext_v32i16_v32i64(<32 x i16>* %in, <32 x i64>* %out) #0 {
 ; CHECK-LABEL: sext_v32i16_v32i64:
-; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <32 x i16>, <32 x i16>* %in
   %b = add <32 x i16> %a, %a
   %c = sext <32 x i16> %b to <32 x i64>
   store <32 x i64> %c, <32 x i64>* %out

 ; CHECK-NEXT: ret
   %b = sext <4 x i32> %a to <4 x i64>
   store <4 x i64>%b, <4 x i64>* %out
   ret void
 }
 
 define void @sext_v8i32_v8i64(<8 x i32>* %in, <8 x i64>* %out) #0 {
 ; CHECK-LABEL: sext_v8i32_v8i64:
-; VBITS_GE_512: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
-; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: add [[A_WORDS:z[0-9]+]].s, {{z[0-9]+}}.s, {{z[0-9]+}}.s
 ; VBITS_GE_512-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_512-NEXT: ret
   %a = load <8 x i32>, <8 x i32>* %in
   %b = add <8 x i32> %a, %a
   %c = sext <8 x i32> %b to <8 x i64>
   store <8 x i64> %c, <8 x i64>* %out
   ret void
 }
 
 define void @sext_v16i32_v16i64(<16 x i32>* %in, <16 x i64>* %out) #0 {
 ; CHECK-LABEL: sext_v16i32_v16i64:
-; VBITS_GE_1024: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: add [[A_WORDS:z[0-9]+]].s, {{z[0-9]+}}.s, {{z[0-9]+}}.s
 ; VBITS_GE_1024-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <16 x i32>, <16 x i32>* %in
   %b = add <16 x i32> %a, %a
   %c = sext <16 x i32> %b to <16 x i64>
   store <16 x i64> %c, <16 x i64>* %out
   ret void
 }
 
 define void @sext_v32i32_v32i64(<32 x i32>* %in, <32 x i64>* %out) #0 {
 ; CHECK-LABEL: sext_v32i32_v32i64:
-; VBITS_GE_2048: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: add [[A_WORDS:z[0-9]+]].s, {{z[0-9]+}}.s, {{z[0-9]+}}.s
 ; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <32 x i32>, <32 x i32>* %in
   %b = add <32 x i32> %a, %a
   %c = sext <32 x i32> %b to <32 x i64>
   store <32 x i64> %c, <32 x i64>* %out
   ret void
; CHECK-NEXT: ret
   %b = zext <16 x i8> %a to <16 x i16>
   store <16 x i16>%b, <16 x i16>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the extend being combined with the load.
 define void @zext_v32i8_v32i16(<32 x i8>* %in, <32 x i16>* %out) #0 {
 ; CHECK-LABEL: zext_v32i8_v32i16:
-; VBITS_GE_512: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_512-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_512-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
 ; VBITS_GE_512-NEXT: ret
   %a = load <32 x i8>, <32 x i8>* %in
   %b = add <32 x i8> %a, %a
   %c = zext <32 x i8> %b to <32 x i16>
   store <32 x i16> %c, <32 x i16>* %out
   ret void
 }
 
 define void @zext_v64i8_v64i16(<64 x i8>* %in, <64 x i16>* %out) #0 {
 ; CHECK-LABEL: zext_v64i8_v64i16:
-; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_1024-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_1024-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <64 x i8>, <64 x i8>* %in
   %b = add <64 x i8> %a, %a
   %c = zext <64 x i8> %b to <64 x i16>
   store <64 x i16> %c, <64 x i16>* %out
   ret void
 }
 
 define void @zext_v128i8_v128i16(<128 x i8>* %in, <128 x i16>* %out) #0 {
 ; CHECK-LABEL: zext_v128i8_v128i16:
-; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_2048-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <128 x i8>, <128 x i8>* %in
   %b = add <128 x i8> %a, %a
   %c = zext <128 x i8> %b to <128 x i16>
   store <128 x i16> %c, <128 x i16>* %out
   ret void

 ; VBITS_EQ_256-NEXT: ret
   %b = zext <16 x i8> %a to <16 x i32>
   store <16 x i32> %b, <16 x i32>* %out
   ret void
 }
 
 define void @zext_v32i8_v32i32(<32 x i8>* %in, <32 x i32>* %out) #0 {
 ; CHECK-LABEL: zext_v32i8_v32i32:
-; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_1024-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <32 x i8>, <32 x i8>* %in
   %b = add <32 x i8> %a, %a
   %c = zext <32 x i8> %b to <32 x i32>
   store <32 x i32> %c, <32 x i32>* %out
   ret void
 }
 
 define void @zext_v64i8_v64i32(<64 x i8>* %in, <64 x i32>* %out) #0 {
 ; CHECK-LABEL: zext_v64i8_v64i32:
-; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <64 x i8>, <64 x i8>* %in
   %b = add <64 x i8> %a, %a
   %c = zext <64 x i8> %b to <64 x i32>
   store <64 x i32> %c, <64 x i32>* %out

 ; VBITS_GE_1024-NEXT: ret
   %b = zext <16 x i8> %a to <16 x i64>
   store <16 x i64> %b, <16 x i64>* %out
   ret void
 }
 
 define void @zext_v32i8_v32i64(<32 x i8>* %in, <32 x i64>* %out) #0 {
 ; CHECK-LABEL: zext_v32i8_v32i64:
-; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: add [[A_BYTES:z[0-9]+]].b, {{z[0-9]+}}.b, {{z[0-9]+}}.b
 ; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
 ; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <32 x i8>, <32 x i8>* %in
   %b = add <32 x i8> %a, %a
   %c = zext <32 x i8> %b to <32 x i64>

 ; CHECK-NEXT: ret
   %b = zext <8 x i16> %a to <8 x i32>
   store <8 x i32>%b, <8 x i32>* %out
   ret void
 }
 
 define void @zext_v16i16_v16i32(<16 x i16>* %in, <16 x i32>* %out) #0 {
 ; CHECK-LABEL: zext_v16i16_v16i32:
-; VBITS_GE_512: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_512-NEXT: ret
   %a = load <16 x i16>, <16 x i16>* %in
   %b = add <16 x i16> %a, %a
   %c = zext <16 x i16> %b to <16 x i32>
   store <16 x i32> %c, <16 x i32>* %out
   ret void
 }
 
 define void @zext_v32i16_v32i32(<32 x i16>* %in, <32 x i32>* %out) #0 {
 ; CHECK-LABEL: zext_v32i16_v32i32:
-; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <32 x i16>, <32 x i16>* %in
   %b = add <32 x i16> %a, %a
   %c = zext <32 x i16> %b to <32 x i32>
   store <32 x i32> %c, <32 x i32>* %out
   ret void
 }
 
 define void @zext_v64i16_v64i32(<64 x i16>* %in, <64 x i32>* %out) #0 {
 ; CHECK-LABEL: zext_v64i16_v64i32:
-; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <64 x i16>, <64 x i16>* %in
   %b = add <64 x i16> %a, %a
   %c = zext <64 x i16> %b to <64 x i32>
   store <64 x i32> %c, <64 x i32>* %out
   ret void

 ; VBITS_GE_512-NEXT: ret
   %b = zext <8 x i16> %a to <8 x i64>
   store <8 x i64>%b, <8 x i64>* %out
   ret void
 }
 
 define void @zext_v16i16_v16i64(<16 x i16>* %in, <16 x i64>* %out) #0 {
 ; CHECK-LABEL: zext_v16i16_v16i64:
-; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_1024-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <16 x i16>, <16 x i16>* %in
   %b = add <16 x i16> %a, %a
   %c = zext <16 x i16> %b to <16 x i64>
   store <16 x i64> %c, <16 x i64>* %out
   ret void
 }
 
 define void @zext_v32i16_v32i64(<32 x i16>* %in, <32 x i64>* %out) #0 {
 ; CHECK-LABEL: zext_v32i16_v32i64:
-; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: add [[A_HALFS:z[0-9]+]].h, {{z[0-9]+}}.h, {{z[0-9]+}}.h
 ; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
 ; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <32 x i16>, <32 x i16>* %in
   %b = add <32 x i16> %a, %a
   %c = zext <32 x i16> %b to <32 x i64>
   store <32 x i64> %c, <32 x i64>* %out

 ; CHECK-NEXT: ret
   %b = zext <4 x i32> %a to <4 x i64>
   store <4 x i64>%b, <4 x i64>* %out
   ret void
 }
 
 define void @zext_v8i32_v8i64(<8 x i32>* %in, <8 x i64>* %out) #0 {
 ; CHECK-LABEL: zext_v8i32_v8i64:
-; VBITS_GE_512: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
-; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: add [[A_WORDS:z[0-9]+]].s, {{z[0-9]+}}.s, {{z[0-9]+}}.s
 ; VBITS_GE_512-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_512-NEXT: ret
   %a = load <8 x i32>, <8 x i32>* %in
   %b = add <8 x i32> %a, %a
   %c = zext <8 x i32> %b to <8 x i64>
   store <8 x i64> %c, <8 x i64>* %out
   ret void
 }
 
 define void @zext_v16i32_v16i64(<16 x i32>* %in, <16 x i64>* %out) #0 {
 ; CHECK-LABEL: zext_v16i32_v16i64:
-; VBITS_GE_1024: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
-; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: add [[A_WORDS:z[0-9]+]].s, {{z[0-9]+}}.s, {{z[0-9]+}}.s
 ; VBITS_GE_1024-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_1024-NEXT: ret
   %a = load <16 x i32>, <16 x i32>* %in
   %b = add <16 x i32> %a, %a
   %c = zext <16 x i32> %b to <16 x i64>
   store <16 x i64> %c, <16 x i64>* %out
   ret void
 }
 
 define void @zext_v32i32_v32i64(<32 x i32>* %in, <32 x i64>* %out) #0 {
 ; CHECK-LABEL: zext_v32i32_v32i64:
-; VBITS_GE_2048: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
-; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: add [[A_WORDS:z[0-9]+]].s, {{z[0-9]+}}.s, {{z[0-9]+}}.s
 ; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
 ; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
 ; VBITS_GE_2048-NEXT: ret
   %a = load <32 x i32>, <32 x i32>* %in
   %b = add <32 x i32> %a, %a
   %c = zext <32 x i32> %b to <32 x i64>
   store <32 x i64> %c, <32 x i64>* %out
   ret void
 }
 
 attributes #0 = { nounwind "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-fixed-length-int-immediates.ll

 ;
 ; ADD
 ;
 
 define void @add_v64i8(<64 x i8>* %a) #0 {
 ; CHECK-LABEL: add_v64i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b, vl64
-; CHECK-NEXT:    mov z1.b, #7 // =0x7
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
-; CHECK-NEXT:    add z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT:    add z0.b, z0.b, #7 // =0x7
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %ins = insertelement <64 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <64 x i8> %ins, <64 x i8> undef, <64 x i32> zeroinitializer
   %res = add <64 x i8> %op1, %op2
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 
 define void @add_v32i16(<32 x i16>* %a) #0 {
 ; CHECK-LABEL: add_v32i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl32
-; CHECK-NEXT:    mov z1.h, #15 // =0xf
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
-; CHECK-NEXT:    add z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    add z0.h, z0.h, #15 // =0xf
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %ins = insertelement <32 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <32 x i16> %ins, <32 x i16> undef, <32 x i32> zeroinitializer
   %res = add <32 x i16> %op1, %op2
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }
 
 define void @add_v16i32(<16 x i32>* %a) #0 {
 ; CHECK-LABEL: add_v16i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl16
-; CHECK-NEXT:    mov z1.s, #31 // =0x1f
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT:    add z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    add z0.s, z0.s, #31 // =0x1f
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %ins = insertelement <16 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <16 x i32> %ins, <16 x i32> undef, <16 x i32> zeroinitializer
   %res = add <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 define void @add_v8i64(<8 x i64>* %a) #0 {
 ; CHECK-LABEL: add_v8i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl8
-; CHECK-NEXT:    mov z1.d, #63 // =0x3f
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT:    add z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    add z0.d, z0.d, #63 // =0x3f
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <8 x i64>, <8 x i64>* %a
   %ins = insertelement <8 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <8 x i64> %ins, <8 x i64> undef, <8 x i32> zeroinitializer
   %res = add <8 x i64> %op1, %op2
   store <8 x i64> %res, <8 x i64>* %a
   ret void

 ;
 ; SUB
 ;
 
 define void @sub_v64i8(<64 x i8>* %a) #0 {
 ; CHECK-LABEL: sub_v64i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b, vl64
-; CHECK-NEXT:    mov z1.b, #7 // =0x7
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
-; CHECK-NEXT:    sub z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT:    sub z0.b, z0.b, #7 // =0x7
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %ins = insertelement <64 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <64 x i8> %ins, <64 x i8> undef, <64 x i32> zeroinitializer
   %res = sub <64 x i8> %op1, %op2
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 
 define void @sub_v32i16(<32 x i16>* %a) #0 {
 ; CHECK-LABEL: sub_v32i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl32
-; CHECK-NEXT:    mov z1.h, #15 // =0xf
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
-; CHECK-NEXT:    sub z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    sub z0.h, z0.h, #15 // =0xf
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %ins = insertelement <32 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <32 x i16> %ins, <32 x i16> undef, <32 x i32> zeroinitializer
   %res = sub <32 x i16> %op1, %op2
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }
 
 define void @sub_v16i32(<16 x i32>* %a) #0 {
 ; CHECK-LABEL: sub_v16i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl16
-; CHECK-NEXT:    mov z1.s, #31 // =0x1f
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT:    sub z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    sub z0.s, z0.s, #31 // =0x1f
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %ins = insertelement <16 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <16 x i32> %ins, <16 x i32> undef, <16 x i32> zeroinitializer
   %res = sub <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 define void @sub_v8i64(<8 x i64>* %a) #0 {
 ; CHECK-LABEL: sub_v8i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl8
-; CHECK-NEXT:    mov z1.d, #63 // =0x3f
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT:    sub z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    sub z0.d, z0.d, #63 // =0x3f
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <8 x i64>, <8 x i64>* %a
   %ins = insertelement <8 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <8 x i64> %ins, <8 x i64> undef, <8 x i32> zeroinitializer
   %res = sub <8 x i64> %op1, %op2
   store <8 x i64> %res, <8 x i64>* %a
   ret void

llvm/test/CodeGen/AArch64/sve-fixed-length-int-reduce.ll

 ; VBITS_GE_512-NEXT: fmov x0, [[REDUCE]]
 ; VBITS_GE_512-NEXT: ret
 
 ; Ensure sensible type legalisation.
 ; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].b, vl32
 ; VBITS_EQ_256-DAG: mov w[[NUMELTS:[0-9]+]], #32
 ; VBITS_EQ_256-DAG: ld1b { [[LO:z[0-9]+]].b }, [[PG]]/z, [x0]
 ; VBITS_EQ_256-DAG: ld1b { [[HI:z[0-9]+]].b }, [[PG]]/z, [x0, x[[NUMELTS]]]
-; VBITS_EQ_256-DAG: add [[ADD:z[0-9]+]].b, [[PG]]/m, [[HI]].b, [[LO]].b
+; VBITS_EQ_256-DAG: add [[ADD:z[0-9]+]].b, [[LO]].b, [[HI]].b
 ; VBITS_EQ_256-DAG: addv [[REDUCE:d[0-9]+]], [[PG]], [[ADD]].b
 ; VBITS_EQ_256-NEXT: fmov x0, [[REDUCE]]
 ; VBITS_EQ_256-NEXT: ret
   %op = load <64 x i8>, <64 x i8>* %a
   %res = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> %op)
   ret i8 %res
 }
 

 ; VBITS_GE_512-NEXT: fmov x0, [[REDUCE]]
 ; VBITS_GE_512-NEXT: ret
 
 ; Ensure sensible type legalisation.
 ; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].h, vl16
 ; VBITS_EQ_256-DAG: mov x[[NUMELTS:[0-9]+]], #16
 ; VBITS_EQ_256-DAG: ld1h { [[LO:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; VBITS_EQ_256-DAG: ld1h { [[HI:z[0-9]+]].h }, [[PG]]/z, [x0, x[[NUMELTS]], lsl #1]
-; VBITS_EQ_256-DAG: add [[ADD:z[0-9]+]].h, [[PG]]/m, [[HI]].h, [[LO]].h
+; VBITS_EQ_256-DAG: add [[ADD:z[0-9]+]].h, [[LO]].h, [[HI]].h
 ; VBITS_EQ_256-DAG: addv [[REDUCE:d[0-9]+]], [[PG]], [[ADD]].h
 ; VBITS_EQ_256-NEXT: fmov x0, [[REDUCE]]
 ; VBITS_EQ_256-NEXT: ret
   %op = load <32 x i16>, <32 x i16>* %a
   %res = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %op)
   ret i16 %res
 }
 

 ; VBITS_GE_512-NEXT: fmov x0, [[REDUCE]]
 ; VBITS_GE_512-NEXT: ret
 
 ; Ensure sensible type legalisation.
 ; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
 ; VBITS_EQ_256-DAG: mov x[[NUMELTS:[0-9]+]], #8
 ; VBITS_EQ_256-DAG: ld1w { [[LO:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_EQ_256-DAG: ld1w { [[HI:z[0-9]+]].s }, [[PG]]/z, [x0, x[[NUMELTS]], lsl #2]
-; VBITS_EQ_256-DAG: add [[ADD:z[0-9]+]].s, [[PG]]/m, [[HI]].s, [[LO]].s
+; VBITS_EQ_256-DAG: add [[ADD:z[0-9]+]].s, [[LO]].s, [[HI]].s
 ; VBITS_EQ_256-DAG: addv [[REDUCE:d[0-9]+]], [[PG]], [[ADD]].s
 ; VBITS_EQ_256-NEXT: fmov x0, [[REDUCE]]
 ; VBITS_EQ_256-NEXT: ret
   %op = load <16 x i32>, <16 x i32>* %a
   %res = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %op)
   ret i32 %res
 }
 

 ; VBITS_GE_512-NEXT: fmov x0, [[REDUCE]]
 ; VBITS_GE_512-NEXT: ret
 
 ; Ensure sensible type legalisation.
 ; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].d, vl4
 ; VBITS_EQ_256-DAG: mov x[[NUMELTS:[0-9]+]], #4
 ; VBITS_EQ_256-DAG: ld1d { [[LO:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_EQ_256-DAG: ld1d { [[HI:z[0-9]+]].d }, [[PG]]/z, [x0, x[[NUMELTS]], lsl #3]
-; VBITS_EQ_256-DAG: add [[ADD:z[0-9]+]].d, [[PG]]/m, [[HI]].d, [[LO]].d
+; VBITS_EQ_256-DAG: add [[ADD:z[0-9]+]].d, [[LO]].d, [[HI]].d
 ; VBITS_EQ_256-DAG: addv [[REDUCE:d[0-9]+]], [[PG]], [[ADD]].d
 ; VBITS_EQ_256-NEXT: fmov x0, [[REDUCE]]
 ; VBITS_EQ_256-NEXT: ret
   %op = load <8 x i64>, <8 x i64>* %a
   %res = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %op)
   ret i64 %res
 }
 

llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll

 ; VBITS_EQ_256-NEXT: sunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_256-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_256-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_256-NEXT: sdiv [[DIV3:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_HI]].s, [[OP2_LO_HI]].s
 ; VBITS_EQ_256-NEXT: sdivr [[DIV4:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_256-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_256-NEXT: uzp1 [[UZP2:z[0-9]+]].h, [[DIV4]].h, [[DIV3]].h
 ; VBITS_EQ_256-NEXT: uzp1 [[UZP3:z[0-9]+]].b, [[UZP2]].b, [[UZP1]].b
-; VBITS_EQ_256-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP3]].b
-; VBITS_EQ_256-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_256-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_256-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP3]].b, [[OP2]].b
+; VBITS_EQ_256-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; HALF VECTOR:
 ; VBITS_EQ_512: ptrue [[PG1:p[0-9]+]].b, vl32
 ; VBITS_EQ_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
 ; VBITS_EQ_512-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_EQ_512-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_EQ_512-NEXT: sunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_EQ_512-NEXT: sunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_EQ_512-NEXT: sunpkhi [[OP2_LO_HI:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_512-NEXT: sunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_512-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_512-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_512-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_HI]].s, [[OP1_LO_HI]].s
 ; VBITS_EQ_512-NEXT: sdivr [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_EQ_512-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_EQ_512-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_512-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_512-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_EQ_512-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; QUARTER VECTOR OR SMALLER:
 ; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].b, vl32
 ; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
 ; VBITS_GE_1024-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_GE_1024-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_GE_1024-NEXT: sunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_GE_1024-NEXT: sunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_GE_1024-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_GE_1024-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_GE_1024-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_GE_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
 ; VBITS_GE_1024-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_GE_1024-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_GE_1024-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_GE_1024-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_GE_1024-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_GE_1024-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i8>, <32 x i8>* %a
   %op2 = load <32 x i8>, <32 x i8>* %b
   %res = srem <32 x i8> %op1, %op2
   store <32 x i8> %res, <32 x i8>* %a
   ret void
 }
 

 ; VBITS_EQ_512-NEXT: sunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_512-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_512-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_512-NEXT: sdiv [[DIV3:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_HI]].s, [[OP2_LO_HI]].s
 ; VBITS_EQ_512-NEXT: sdivr [[DIV4:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP2:z[0-9]+]].h, [[DIV4]].h, [[DIV3]].h
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP3:z[0-9]+]].b, [[UZP2]].b, [[UZP1]].b
-; VBITS_EQ_512-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP3]].b
-; VBITS_EQ_512-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_512-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_512-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP3]].b, [[OP2]].b
+; VBITS_EQ_512-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; HALF VECTOR:
 ; VBITS_EQ_1024: ptrue [[PG1:p[0-9]+]].b, vl64
 ; VBITS_EQ_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
 ; VBITS_EQ_1024-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_EQ_1024-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_EQ_1024-NEXT: sunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_EQ_1024-NEXT: sunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_EQ_1024-NEXT: sunpkhi [[OP2_LO_HI:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_1024-NEXT: sunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_1024-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_1024-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_1024-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_HI]].s, [[OP1_LO_HI]].s
 ; VBITS_EQ_1024-NEXT: sdivr [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_EQ_1024-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_EQ_1024-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_1024-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_1024-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_EQ_1024-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; QUARTER VECTOR OR SMALLER:
 ; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].b, vl64
 ; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
 ; VBITS_GE_2048-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_GE_2048-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_GE_2048-NEXT: sunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_GE_2048-NEXT: sunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_GE_2048-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_GE_2048-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_GE_2048-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_GE_2048-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
 ; VBITS_GE_2048-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_GE_2048-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_GE_2048-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_GE_2048-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_GE_2048-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_GE_2048-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %res = srem <64 x i8> %op1, %op2
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 

 ; VBITS_EQ_1024-NEXT: sunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_1024-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_1024-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_1024-NEXT: sdiv [[DIV3:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_HI]].s, [[OP2_LO_HI]].s
 ; VBITS_EQ_1024-NEXT: sdivr [[DIV4:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP2:z[0-9]+]].h, [[DIV4]].h, [[DIV3]].h
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP3:z[0-9]+]].b, [[UZP2]].b, [[UZP1]].b
-; VBITS_EQ_1024-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP3]].b
-; VBITS_EQ_1024-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_1024-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_1024-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP3]].b, [[OP2]].b
+; VBITS_EQ_1024-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; HALF VECTOR:
 ; VBITS_EQ_2048: ptrue [[PG1:p[0-9]+]].b, vl128
 ; VBITS_EQ_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
 ; VBITS_EQ_2048-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_EQ_2048-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_EQ_2048-NEXT: sunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_EQ_2048-NEXT: sunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_EQ_2048-NEXT: sunpkhi [[OP2_LO_HI:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_2048-NEXT: sunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_2048-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_2048-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_2048-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_HI]].s, [[OP1_LO_HI]].s
 ; VBITS_EQ_2048-NEXT: sdivr [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_2048-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_2048-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_EQ_2048-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_EQ_2048-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_2048-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_2048-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_EQ_2048-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <128 x i8>, <128 x i8>* %a
   %op2 = load <128 x i8>, <128 x i8>* %b
   %res = srem <128 x i8> %op1, %op2
   store <128 x i8> %res, <128 x i8>* %a
   ret void
 }
 

 ; VBITS_EQ_2048-NEXT: sunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_2048-NEXT: sunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_2048-NEXT: sunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_2048-NEXT: sdiv    [[RES_LO_HI:z[0-9]+]].s, [[PG1]]/m, [[OP1_LO_HI]].s, [[OP2_LO_HI]].s
 ; VBITS_EQ_2048-NEXT: sdivr   [[RES_LO_LO:z[0-9]+]].s, [[PG1]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_2048-NEXT: uzp1    [[RES_HI:z[0-9]+]].h, [[RES_HI_LO]].h, [[RES_HI_HI]].h
 ; VBITS_EQ_2048-NEXT: uzp1    [[RES_LO:z[0-9]+]].h, [[RES_LO_LO]].h, [[RES_LO_HI]].h
 ; VBITS_EQ_2048-NEXT: uzp1    [[ZIP:z[0-9]+]].b, [[RES_LO]].b, [[RES_HI]].b
-; VBITS_EQ_2048-NEXT: mul     [[MUL:z[0-9]+]].b, [[PG]]/m, [[OP2]].b, [[ZIP]].b
-; VBITS_EQ_2048-NEXT: sub     [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[MUL]].b
-; VBITS_EQ_2048-NEXT: st1b    { [[RES]].b }, [[PG]], [x0]
+; VBITS_EQ_2048-NEXT: mls     [[OP1]].b, [[PG]]/m, [[ZIP]].b, [[OP2]].b
+; VBITS_EQ_2048-NEXT: st1b    { [[OP1]].b }, [[PG]], [x0]
 ; VBITS_EQ_2048-NEXT: ret
   %op1 = load <256 x i8>, <256 x i8>* %a
   %op2 = load <256 x i8>, <256 x i8>* %b
   %res = srem <256 x i8> %op1, %op2
   store <256 x i8> %res, <256 x i8>* %a
   ret void
 }
 

 ; VBITS_EQ_256-NEXT: sunpkhi [[OP2_HI:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_256-NEXT: sunpkhi [[OP1_HI:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_256-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_HI]].s, [[OP1_HI]].s
 ; VBITS_EQ_256-NEXT: sunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_256-NEXT: sunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_256-NEXT: movprfx [[OP1_LO_:z[0-9]+]], [[OP1_LO]]
 ; VBITS_EQ_256-NEXT: sdiv [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_]].s, [[OP2_LO]].s
 ; VBITS_EQ_256-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
-; VBITS_EQ_256-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_EQ_256-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_EQ_256-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_EQ_256-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_EQ_256-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 
 ; HALF VECTOR OR SMALLER:
 ; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].h, vl16
 ; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
 ; VBITS_GE_512-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_GE_512-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_GE_512-NEXT: sunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_GE_512-NEXT: sunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_GE_512-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO]].s, [[OP1_LO]].s
 ; VBITS_GE_512-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
-; VBITS_GE_512-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_GE_512-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_GE_512-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_GE_512-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_GE_512-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 ; CHECK: ret
 
   %op1 = load <16 x i16>, <16 x i16>* %a
   %op2 = load <16 x i16>, <16 x i16>* %b
   %res = srem <16 x i16> %op1, %op2
   store <16 x i16> %res, <16 x i16>* %a
   ret void
 }

 ; VBITS_EQ_512-NEXT: sunpkhi [[OP2_HI:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_512-NEXT: sunpkhi [[OP1_HI:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_512-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_HI]].s, [[OP1_HI]].s
 ; VBITS_EQ_512-NEXT: sunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_512-NEXT: sunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_512-NEXT: movprfx [[OP1_LO_:z[0-9]+]], [[OP1_LO]]
 ; VBITS_EQ_512-NEXT: sdiv [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_]].s, [[OP2_LO]].s
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
-; VBITS_EQ_512-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_EQ_512-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_EQ_512-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_EQ_512-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_EQ_512-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 
 ; HALF VECTOR OR SMALLER:
 ; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl32
 ; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
 ; VBITS_GE_1024-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_GE_1024-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_GE_1024-NEXT: sunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_GE_1024-NEXT: sunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_GE_1024-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO]].s, [[OP1_LO]].s
 ; VBITS_GE_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
-; VBITS_GE_1024-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_GE_1024-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_GE_1024-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_GE_1024-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_GE_1024-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %res = srem <32 x i16> %op1, %op2
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }
 
 define void @srem_v64i16(<64 x i16>* %a, <64 x i16>* %b) #0 {
 ; CHECK-LABEL: srem_v64i16:
 ; VBITS_EQ_1024: ptrue [[PG1:p[0-9]+]].h, vl64
 ; VBITS_EQ_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
 ; VBITS_EQ_1024-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_EQ_1024-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_EQ_1024-NEXT: sunpkhi [[OP2_HI:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_1024-NEXT: sunpkhi [[OP1_HI:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_1024-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_HI]].s, [[OP1_HI]].s
 ; VBITS_EQ_1024-NEXT: sunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_1024-NEXT: sunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_1024-NEXT: movprfx [[OP1_LO_:z[0-9]+]], [[OP1_LO]]
 ; VBITS_EQ_1024-NEXT: sdiv [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_]].s, [[OP2_LO]].s
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
-; VBITS_EQ_1024-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_EQ_1024-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_EQ_1024-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_EQ_1024-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_EQ_1024-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 
 ; HALF VECTOR OR SMALLER:
 ; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl64
 ; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
 ; VBITS_GE_2048-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_GE_2048-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_GE_2048-NEXT: sunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_GE_2048-NEXT: sunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_GE_2048-NEXT: sdivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO]].s, [[OP1_LO]].s
 ; VBITS_GE_2048-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
-; VBITS_GE_2048-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_GE_2048-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_GE_2048-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_GE_2048-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_GE_2048-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i16>, <64 x i16>* %a
   %op2 = load <64 x i16>, <64 x i16>* %b
   %res = srem <64 x i16> %op1, %op2
   store <64 x i16> %res, <64 x i16>* %a
   ret void
 }
 
 define void @srem_v128i16(<128 x i16>* %a, <128 x i16>* %b) #0 {
 ; CHECK-LABEL: srem_v128i16:
 ; VBITS_EQ_2048: ptrue [[PG:p[0-9]+]].h, vl128
 ; VBITS_EQ_2048-NEXT: ptrue [[PG1:p[0-9]+]].s, vl64
 ; VBITS_EQ_2048-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; VBITS_EQ_2048-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
 ; VBITS_EQ_2048-NEXT: sunpkhi [[OP2_HI:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_2048-NEXT: sunpkhi [[OP1_HI:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_2048-NEXT: sdivr   [[RES_HI:z[0-9]+]].s, [[PG1]]/m, [[OP2_HI]].s, [[OP1_HI]].s
 ; VBITS_EQ_2048-NEXT: sunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_2048-NEXT: sunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_2048-NEXT: movprfx [[OP3_LO:z[0-9]+]], [[OP1_LO]]
 ; VBITS_EQ_2048-NEXT: sdiv    [[RES_LO:z[0-9]+]].s, [[PG1]]/m, [[OP3_LO]].s, [[OP2_LO]].s
 ; VBITS_EQ_2048-NEXT: uzp1 [[ZIP:z[0-9]+]].h, [[RES_LO]].h, [[RES_HI]].h
-; VBITS_EQ_2048-NEXT: mul [[MUL:z[0-9]+]].h, [[PG]]/m, [[OP2]].h, [[ZIP]].h
-; VBITS_EQ_2048-NEXT: sub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[MUL]].h
-; VBITS_EQ_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_EQ_2048-NEXT: mls [[OP1]].h, [[PG]]/m, [[ZIP]].h, [[OP2]].h
+; VBITS_EQ_2048-NEXT: st1h { [[OP1]].h }, [[PG]], [x0]
 ; VBITS_EQ_2048-NEXT: ret
   %op1 = load <128 x i16>, <128 x i16>* %a
   %op2 = load <128 x i16>, <128 x i16>* %b
   %res = srem <128 x i16> %op1, %op2
   store <128 x i16> %res, <128 x i16>* %a
   ret void
 }
 

 
 define void @srem_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
 ; CHECK-LABEL: srem_v8i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
 ; CHECK-NEXT: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-NEXT: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
 ; CHECK-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; CHECK-NEXT: sdiv [[DIV:z[0-9]+]].s, [[PG]]/m, [[PFX]].s, [[OP2]].s
-; CHECK-NEXT: mul [[MUL:z[0-9]+]].s, [[PG]]/m, [[OP2]].s, [[DIV]].s
-; CHECK-NEXT: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[MUL]].s
-; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK-NEXT: mls [[OP1]].s, [[PG]]/m, [[DIV]].s, [[OP2]].s
+; CHECK-NEXT: st1w { [[OP1]].s }, [[PG]], [x0]
 ; CHECK-NEXT: ret
   %op1 = load <8 x i32>, <8 x i32>* %a
   %op2 = load <8 x i32>, <8 x i32>* %b
   %res = srem <8 x i32> %op1, %op2
   store <8 x i32> %res, <8 x i32>* %a
   ret void
 }
 
 define void @srem_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 {
 ; CHECK-LABEL: srem_v16i32:
 ; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
 ; VBITS_GE_512-NEXT: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_512-NEXT: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
 ; VBITS_GE_512-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_512-NEXT: sdiv [[DIV:z[0-9]+]].s, [[PG]]/m, [[PFX]].s, [[OP2]].s
-; VBITS_GE_512-NEXT: mul [[MUL:z[0-9]+]].s, [[PG]]/m, [[OP2]].s, [[DIV]].s
-; VBITS_GE_512-NEXT: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[MUL]].s
-; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: mls [[OP1]].s, [[PG]]/m, [[DIV]].s, [[OP2]].s
+; VBITS_GE_512-NEXT: st1w { [[OP1]].s }, [[PG]], [x0]
 ; VBITS_GE_512-NEXT: ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = srem <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 define void @srem_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 {
 ; CHECK-LABEL: srem_v32i32:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
 ; VBITS_GE_1024-NEXT: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_1024-NEXT: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
 ; VBITS_GE_1024-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_1024-NEXT: sdiv [[DIV:z[0-9]+]].s, [[PG]]/m, [[PFX]].s, [[OP2]].s
-; VBITS_GE_1024-NEXT: mul [[MUL:z[0-9]+]].s, [[PG]]/m, [[OP2]].s, [[DIV]].s
-; VBITS_GE_1024-NEXT: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[MUL]].s
-; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: mls [[OP1]].s, [[PG]]/m, [[DIV]].s, [[OP2]].s
+; VBITS_GE_1024-NEXT: st1w { [[OP1]].s }, [[PG]], [x0]
 ; VBITS_GE_1024-NEXT: ret
   %op1 = load <32 x i32>, <32 x i32>* %a
   %op2 = load <32 x i32>, <32 x i32>* %b
   %res = srem <32 x i32> %op1, %op2
   store <32 x i32> %res, <32 x i32>* %a
   ret void
 }
 
 define void @srem_v64i32(<64 x i32>* %a, <64 x i32>* %b) #0 {
 ; CHECK-LABEL: srem_v64i32:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
 ; VBITS_GE_2048-NEXT: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_2048-NEXT: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
 ; VBITS_GE_2048-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_2048-NEXT: sdiv [[DIV:z[0-9]+]].s, [[PG]]/m, [[PFX]].s, [[OP2]].s
-; VBITS_GE_2048-NEXT: mul [[MUL:z[0-9]+]].s, [[PG]]/m, [[OP2]].s, [[DIV]].s
-; VBITS_GE_2048-NEXT: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[MUL]].s
-; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: mls [[OP1]].s, [[PG]]/m, [[DIV]].s, [[OP2]].s
+; VBITS_GE_2048-NEXT: st1w { [[OP1]].s }, [[PG]], [x0]
 ; VBITS_GE_2048-NEXT: ret
   %op1 = load <64 x i32>, <64 x i32>* %a
   %op2 = load <64 x i32>, <64 x i32>* %b
   %res = srem <64 x i32> %op1, %op2
   store <64 x i32> %res, <64 x i32>* %a
   ret void
 }
 

 
 define void @srem_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 {
 ; CHECK-LABEL: srem_v4i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
 ; CHECK-NEXT: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-NEXT: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
 ; CHECK-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; CHECK-NEXT: sdiv [[DIV:z[0-9]+]].d, [[PG]]/m, [[PFX]].d, [[OP2]].d
-; CHECK-NEXT: mul [[MUL:z[0-9]+]].d, [[PG]]/m, [[OP2]].d, [[DIV]].d
-; CHECK-NEXT: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[MUL]].d
-; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK-NEXT: mls [[OP1]].d, [[PG]]/m, [[DIV]].d, [[OP2]].d
+; CHECK-NEXT: st1d { [[OP1]].d }, [[PG]], [x0]
 ; CHECK-NEXT: ret
   %op1 = load <4 x i64>, <4 x i64>* %a
   %op2 = load <4 x i64>, <4 x i64>* %b
   %res = srem <4 x i64> %op1, %op2
   store <4 x i64> %res, <4 x i64>* %a
   ret void
 }
 
 define void @srem_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 {
 ; CHECK-LABEL: srem_v8i64:
 ; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
 ; VBITS_GE_512-NEXT: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_512-NEXT: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
 ; VBITS_GE_512-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_512-NEXT: sdiv [[DIV:z[0-9]+]].d, [[PG]]/m, [[PFX]].d, [[OP2]].d
-; VBITS_GE_512-NEXT: mul [[MUL:z[0-9]+]].d, [[PG]]/m, [[OP2]].d, [[DIV]].d
-; VBITS_GE_512-NEXT: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[MUL]].d
-; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: mls [[OP1]].d, [[PG]]/m, [[DIV]].d, [[OP2]].d
+; VBITS_GE_512-NEXT: st1d { [[OP1]].d }, [[PG]], [x0]
 ; VBITS_GE_512-NEXT: ret
   %op1 = load <8 x i64>, <8 x i64>* %a
   %op2 = load <8 x i64>, <8 x i64>* %b
   %res = srem <8 x i64> %op1, %op2
   store <8 x i64> %res, <8 x i64>* %a
   ret void
 }
 
 define void @srem_v16i64(<16 x i64>* %a, <16 x i64>* %b) #0 {
 ; CHECK-LABEL: srem_v16i64:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
 ; VBITS_GE_1024-NEXT: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_1024-NEXT: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
 ; VBITS_GE_1024-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_1024-NEXT: sdiv [[DIV:z[0-9]+]].d, [[PG]]/m, [[PFX]].d, [[OP2]].d
-; VBITS_GE_1024-NEXT: mul [[MUL:z[0-9]+]].d, [[PG]]/m, [[OP2]].d, [[DIV]].d
-; VBITS_GE_1024-NEXT: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[MUL]].d
-; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: mls [[OP1]].d, [[PG]]/m, [[DIV]].d, [[OP2]].d
+; VBITS_GE_1024-NEXT: st1d { [[OP1]].d }, [[PG]], [x0]
 ; VBITS_GE_1024-NEXT: ret
   %op1 = load <16 x i64>, <16 x i64>* %a
   %op2 = load <16 x i64>, <16 x i64>* %b
   %res = srem <16 x i64> %op1, %op2
   store <16 x i64> %res, <16 x i64>* %a
   ret void
 }
 
 define void @srem_v32i64(<32 x i64>* %a, <32 x i64>* %b) #0 {
 ; CHECK-LABEL: srem_v32i64:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
 ; VBITS_GE_2048-NEXT: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_2048-NEXT: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
 ; VBITS_GE_2048-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_2048-NEXT: sdiv [[DIV:z[0-9]+]].d, [[PG]]/m, [[PFX]].d, [[OP2]].d
-; VBITS_GE_2048-NEXT: mul [[MUL:z[0-9]+]].d, [[PG]]/m, [[OP2]].d, [[DIV]].d
-; VBITS_GE_2048-NEXT: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[MUL]].d
-; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: mls [[OP1]].d, [[PG]]/m, [[DIV]].d, [[OP2]].d
+; VBITS_GE_2048-NEXT: st1d { [[OP1]].d }, [[PG]], [x0]
 ; VBITS_GE_2048-NEXT: ret
   %op1 = load <32 x i64>, <32 x i64>* %a
   %op2 = load <32 x i64>, <32 x i64>* %b
   %res = srem <32 x i64> %op1, %op2
   store <32 x i64> %res, <32 x i64>* %a
   ret void
 }
 

 ; VBITS_EQ_256-NEXT: uunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_256-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_256-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_256-NEXT: udiv [[DIV3:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_HI]].s, [[OP2_LO_HI]].s
 ; VBITS_EQ_256-NEXT: udivr [[DIV4:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_256-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_256-NEXT: uzp1 [[UZP2:z[0-9]+]].h, [[DIV4]].h, [[DIV3]].h
 ; VBITS_EQ_256-NEXT: uzp1 [[UZP3:z[0-9]+]].b, [[UZP2]].b, [[UZP1]].b
-; VBITS_EQ_256-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP3]].b
-; VBITS_EQ_256-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_256-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_256-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP3]].b, [[OP2]].b
+; VBITS_EQ_256-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; HALF VECTOR:
 ; VBITS_EQ_512: ptrue [[PG1:p[0-9]+]].b, vl32
 ; VBITS_EQ_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
 ; VBITS_EQ_512-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_EQ_512-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_EQ_512-NEXT: uunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_EQ_512-NEXT: uunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_EQ_512-NEXT: uunpkhi [[OP2_LO_HI:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_512-NEXT: uunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_512-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_512-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_512-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_HI]].s, [[OP1_LO_HI]].s
 ; VBITS_EQ_512-NEXT: udivr [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_EQ_512-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_EQ_512-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_512-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_512-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_EQ_512-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; QUARTER VECTOR OR SMALLER:
 ; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].b, vl32
 ; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
 ; VBITS_GE_1024-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_GE_1024-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_GE_1024-NEXT: uunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_GE_1024-NEXT: uunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_GE_1024-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_GE_1024-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_GE_1024-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_GE_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
 ; VBITS_GE_1024-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_GE_1024-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_GE_1024-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_GE_1024-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_GE_1024-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_GE_1024-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i8>, <32 x i8>* %a
   %op2 = load <32 x i8>, <32 x i8>* %b
   %res = urem <32 x i8> %op1, %op2
   store <32 x i8> %res, <32 x i8>* %a
   ret void
 }
 

 ; VBITS_EQ_512-NEXT: uunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_512-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_512-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_512-NEXT: udiv [[DIV3:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_HI]].s, [[OP2_LO_HI]].s
 ; VBITS_EQ_512-NEXT: udivr [[DIV4:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP2:z[0-9]+]].h, [[DIV4]].h, [[DIV3]].h
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP3:z[0-9]+]].b, [[UZP2]].b, [[UZP1]].b
-; VBITS_EQ_512-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP3]].b
-; VBITS_EQ_512-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_512-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_512-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP3]].b, [[OP2]].b
+; VBITS_EQ_512-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; HALF VECTOR:
 ; VBITS_EQ_1024: ptrue [[PG1:p[0-9]+]].b, vl64
 ; VBITS_EQ_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
 ; VBITS_EQ_1024-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_EQ_1024-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_EQ_1024-NEXT: uunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_EQ_1024-NEXT: uunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_EQ_1024-NEXT: uunpkhi [[OP2_LO_HI:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_1024-NEXT: uunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_1024-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_1024-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_1024-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_HI]].s, [[OP1_LO_HI]].s
 ; VBITS_EQ_1024-NEXT: udivr [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_EQ_1024-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_EQ_1024-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_1024-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_1024-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_EQ_1024-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; QUARTER VECTOR OR SMALLER:
 ; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].b, vl64
 ; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
 ; VBITS_GE_2048-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_GE_2048-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_GE_2048-NEXT: uunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_GE_2048-NEXT: uunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_GE_2048-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_GE_2048-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_GE_2048-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_GE_2048-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
 ; VBITS_GE_2048-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_GE_2048-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_GE_2048-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_GE_2048-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_GE_2048-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_GE_2048-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %res = urem <64 x i8> %op1, %op2
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 

 ; VBITS_EQ_1024-NEXT: uunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_1024-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_1024-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_1024-NEXT: udiv [[DIV3:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_HI]].s, [[OP2_LO_HI]].s
 ; VBITS_EQ_1024-NEXT: udivr [[DIV4:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP2:z[0-9]+]].h, [[DIV4]].h, [[DIV3]].h
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP3:z[0-9]+]].b, [[UZP2]].b, [[UZP1]].b
-; VBITS_EQ_1024-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP3]].b
-; VBITS_EQ_1024-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_1024-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_1024-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP3]].b, [[OP2]].b
+; VBITS_EQ_1024-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 
 ; HALF VECTOR:
 ; VBITS_EQ_2048: ptrue [[PG1:p[0-9]+]].b, vl128
 ; VBITS_EQ_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
 ; VBITS_EQ_2048-NEXT: ld1b { [[OP1:z[0-9]+]].b }, [[PG1]]/z, [x0]
 ; VBITS_EQ_2048-NEXT: ld1b { [[OP2:z[0-9]+]].b }, [[PG1]]/z, [x1]
 ; VBITS_EQ_2048-NEXT: uunpklo [[OP2_LO:z[0-9]+]].h, [[OP2]].b
 ; VBITS_EQ_2048-NEXT: uunpklo [[OP1_LO:z[0-9]+]].h, [[OP1]].b
 ; VBITS_EQ_2048-NEXT: uunpkhi [[OP2_LO_HI:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_2048-NEXT: uunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_2048-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_2048-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_2048-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_HI]].s, [[OP1_LO_HI]].s
 ; VBITS_EQ_2048-NEXT: udivr [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_2048-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
 ; VBITS_EQ_2048-NEXT: uzp1 [[UZP2:z[0-9]+]].b, [[UZP1]].b, [[UZP1]].b
-; VBITS_EQ_2048-NEXT: mul [[OP2]].b, [[PG1]]/m, [[OP2]].b, [[UZP2]].b
-; VBITS_EQ_2048-NEXT: sub [[OP1]].b, [[PG1]]/m, [[OP1]].b, [[OP2]].b
-; VBITS_EQ_2048-NEXT: st1b { [[OP1:z[0-9]+]].b }, [[PG1]], [x0]
+; VBITS_EQ_2048-NEXT: mls [[OP1]].b, [[PG1]]/m, [[UZP2]].b, [[OP2]].b
+; VBITS_EQ_2048-NEXT: st1b { [[OP1]].b }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <128 x i8>, <128 x i8>* %a
   %op2 = load <128 x i8>, <128 x i8>* %b
   %res = urem <128 x i8> %op1, %op2
   store <128 x i8> %res, <128 x i8>* %a
   ret void
 }
 

 ; VBITS_EQ_2048-NEXT: uunpkhi [[OP1_LO_HI:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_2048-NEXT: uunpklo [[OP2_LO_LO:z[0-9]+]].s, [[OP2_LO]].h
 ; VBITS_EQ_2048-NEXT: uunpklo [[OP1_LO_LO:z[0-9]+]].s, [[OP1_LO]].h
 ; VBITS_EQ_2048-NEXT: udiv    [[RES_LO_HI:z[0-9]+]].s, [[PG1]]/m, [[OP1_LO_HI]].s, [[OP2_LO_HI]].s
 ; VBITS_EQ_2048-NEXT: udivr   [[RES_LO_LO:z[0-9]+]].s, [[PG1]]/m, [[OP2_LO_LO]].s, [[OP1_LO_LO]].s
 ; VBITS_EQ_2048-NEXT: uzp1    [[RES_HI:z[0-9]+]].h, [[RES_HI_LO]].h, [[RES_HI_HI]].h
 ; VBITS_EQ_2048-NEXT: uzp1    [[RES_LO:z[0-9]+]].h, [[RES_LO_LO]].h, [[RES_LO_HI]].h
 ; VBITS_EQ_2048-NEXT: uzp1    [[ZIP:z[0-9]+]].b, [[RES_LO]].b, [[RES_HI]].b
-; VBITS_EQ_2048-NEXT: mul     [[MUL:z[0-9]+]].b, [[PG]]/m, [[OP2]].b, [[ZIP]].b
-; VBITS_EQ_2048-NEXT: sub     [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[MUL]].b
-; VBITS_EQ_2048-NEXT: st1b    { [[RES]].b }, [[PG]], [x0]
+; VBITS_EQ_2048-NEXT: mls     [[OP1]].b, [[PG]]/m, [[ZIP]].b, [[OP2]].b
+; VBITS_EQ_2048-NEXT: st1b    { [[OP1]].b }, [[PG]], [x0]
 ; VBITS_EQ_2048-NEXT: ret
   %op1 = load <256 x i8>, <256 x i8>* %a
   %op2 = load <256 x i8>, <256 x i8>* %b
   %res = urem <256 x i8> %op1, %op2
   store <256 x i8> %res, <256 x i8>* %a
   ret void
 }
 

 ; VBITS_EQ_256-NEXT: uunpkhi [[OP2_HI:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_256-NEXT: uunpkhi [[OP1_HI:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_256-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_HI]].s, [[OP1_HI]].s
 ; VBITS_EQ_256-NEXT: uunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_256-NEXT: uunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_256-NEXT: movprfx [[OP1_LO_:z[0-9]+]], [[OP1_LO]]
 ; VBITS_EQ_256-NEXT: udiv [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_]].s, [[OP2_LO]].s
 ; VBITS_EQ_256-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
-; VBITS_EQ_256-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_EQ_256-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_EQ_256-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_EQ_256-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_EQ_256-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 
 ; HALF VECTOR OR SMALLER:
 ; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].h, vl16
 ; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
 ; VBITS_GE_512-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_GE_512-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_GE_512-NEXT: uunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_GE_512-NEXT: uunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_GE_512-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO]].s, [[OP1_LO]].s
 ; VBITS_GE_512-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
-; VBITS_GE_512-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_GE_512-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_GE_512-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_GE_512-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_GE_512-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <16 x i16>, <16 x i16>* %a
   %op2 = load <16 x i16>, <16 x i16>* %b
   %res = urem <16 x i16> %op1, %op2
   store <16 x i16> %res, <16 x i16>* %a
   ret void
 }
 
 define void @urem_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 {
 ; CHECK-LABEL: urem_v32i16:
 
 ; FULL VECTOR:
 ; VBITS_EQ_512: ptrue [[PG1:p[0-9]+]].h, vl32
 ; VBITS_EQ_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
 ; VBITS_EQ_512-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_EQ_512-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_EQ_512-NEXT: uunpkhi [[OP2_HI:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_512-NEXT: uunpkhi [[OP1_HI:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_512-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_HI]].s, [[OP1_HI]].s
 ; VBITS_EQ_512-NEXT: uunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_512-NEXT: uunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_512-NEXT: movprfx [[OP1_LO_:z[0-9]+]], [[OP1_LO]]
 ; VBITS_EQ_512-NEXT: udiv [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_]].s, [[OP2_LO]].s
 ; VBITS_EQ_512-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
-; VBITS_EQ_512-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_EQ_512-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_EQ_512-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_EQ_512-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_EQ_512-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 
 ; HALF VECTOR OR SMALLER:
 ; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl32
 ; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
 ; VBITS_GE_1024-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_GE_1024-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_GE_1024-NEXT: uunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_GE_1024-NEXT: uunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_GE_1024-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO]].s, [[OP1_LO]].s
 ; VBITS_GE_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
-; VBITS_GE_1024-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_GE_1024-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_GE_1024-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_GE_1024-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_GE_1024-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %res = urem <32 x i16> %op1, %op2
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }
 
 define void @urem_v64i16(<64 x i16>* %a, <64 x i16>* %b) #0 {
 ; CHECK-LABEL: urem_v64i16:
 ; VBITS_EQ_1024: ptrue [[PG1:p[0-9]+]].h, vl64
 ; VBITS_EQ_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
 ; VBITS_EQ_1024-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_EQ_1024-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_EQ_1024-NEXT: uunpkhi [[OP2_HI:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_1024-NEXT: uunpkhi [[OP1_HI:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_1024-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_HI]].s, [[OP1_HI]].s
 ; VBITS_EQ_1024-NEXT: uunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_1024-NEXT: uunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_1024-NEXT: movprfx [[OP1_LO_:z[0-9]+]], [[OP1_LO]]
 ; VBITS_EQ_1024-NEXT: udiv [[DIV2:z[0-9]+]].s, [[PG2]]/m, [[OP1_LO_]].s, [[OP2_LO]].s
 ; VBITS_EQ_1024-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV2]].h, [[DIV1]].h
-; VBITS_EQ_1024-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_EQ_1024-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_EQ_1024-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_EQ_1024-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_EQ_1024-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 
 ; HALF VECTOR OR SMALLER:
 ; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl64
 ; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
 ; VBITS_GE_2048-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG1]]/z, [x0]
 ; VBITS_GE_2048-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG1]]/z, [x1]
 ; VBITS_GE_2048-NEXT: uunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_GE_2048-NEXT: uunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_GE_2048-NEXT: udivr [[DIV1:z[0-9]+]].s, [[PG2]]/m, [[OP2_LO]].s, [[OP1_LO]].s
 ; VBITS_GE_2048-NEXT: uzp1 [[UZP1:z[0-9]+]].h, [[DIV1]].h, [[DIV1]].h
-; VBITS_GE_2048-NEXT: mul [[OP2]].h, [[PG1]]/m, [[OP2]].h, [[UZP1]].h
-; VBITS_GE_2048-NEXT: sub [[OP1]].h, [[PG1]]/m, [[OP1]].h, [[OP2]].h
-; VBITS_GE_2048-NEXT: st1h { [[OP1:z[0-9]+]].h }, [[PG1]], [x0]
+; VBITS_GE_2048-NEXT: mls [[OP1]].h, [[PG1]]/m, [[UZP1]].h, [[OP2]].h
+; VBITS_GE_2048-NEXT: st1h { [[OP1]].h }, [[PG1]], [x0]
 ; CHECK: ret
   %op1 = load <64 x i16>, <64 x i16>* %a
   %op2 = load <64 x i16>, <64 x i16>* %b
   %res = urem <64 x i16> %op1, %op2
   store <64 x i16> %res, <64 x i16>* %a
   ret void
 }
 
 define void @urem_v128i16(<128 x i16>* %a, <128 x i16>* %b) #0 {
 ; CHECK-LABEL: urem_v128i16:
 ; VBITS_EQ_2048: ptrue [[PG:p[0-9]+]].h, vl128
 ; VBITS_EQ_2048-NEXT: ptrue [[PG1:p[0-9]+]].s, vl64
 ; VBITS_EQ_2048-NEXT: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; VBITS_EQ_2048-NEXT: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
 ; VBITS_EQ_2048-NEXT: uunpkhi [[OP2_HI:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_2048-NEXT: uunpkhi [[OP1_HI:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_2048-NEXT: udivr   [[RES_HI:z[0-9]+]].s, [[PG1]]/m, [[OP2_HI]].s, [[OP1_HI]].s
 ; VBITS_EQ_2048-NEXT: uunpklo [[OP2_LO:z[0-9]+]].s, [[OP2]].h
 ; VBITS_EQ_2048-NEXT: uunpklo [[OP1_LO:z[0-9]+]].s, [[OP1]].h
 ; VBITS_EQ_2048-NEXT: movprfx [[RES_LO:z[0-9]+]], [[OP1_LO]]
 ; VBITS_EQ_2048-NEXT: udiv    [[RES_LO:z[0-9]+]].s, [[PG1]]/m, [[RES_LO]].s, [[OP2_LO]].s
 ; VBITS_EQ_2048-NEXT: uzp1 [[ZIP:z[0-9]+]].h, [[RES_LO]].h, [[RES_HI]].h
-; VBITS_EQ_2048-NEXT: mul [[MUL:z[0-9]+]].h, [[PG]]/m, [[OP2]].h, [[ZIP]].h
-; VBITS_EQ_2048-NEXT: sub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[MUL]].h
-; VBITS_EQ_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_EQ_2048-NEXT: mls [[OP1]].h, [[PG]]/m, [[ZIP]].h, [[OP2]].h
+; VBITS_EQ_2048-NEXT: st1h { [[OP1]].h }, [[PG]], [x0]
 ; VBITS_EQ_2048-NEXT: ret
   %op1 = load <128 x i16>, <128 x i16>* %a
   %op2 = load <128 x i16>, <128 x i16>* %b
   %res = urem <128 x i16> %op1, %op2
   store <128 x i16> %res, <128 x i16>* %a
   ret void
 }
 

 
 define void @urem_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
 ; CHECK-LABEL: urem_v8i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
 ; CHECK-NEXT: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-NEXT: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
 ; CHECK-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; CHECK-NEXT: udiv [[DIV:z[0-9]+]].s, [[PG]]/m, [[PFX]].s, [[OP2]].s
-; CHECK-NEXT: mul [[MUL:z[0-9]+]].s, [[PG]]/m, [[OP2]].s, [[DIV]].s
-; CHECK-NEXT: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[MUL]].s
-; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK-NEXT: mls [[OP1]].s, [[PG]]/m, [[DIV]].s, [[OP2]].s
+; CHECK-NEXT: st1w { [[OP1]].s }, [[PG]], [x0]
 ; CHECK-NEXT: ret
   %op1 = load <8 x i32>, <8 x i32>* %a
   %op2 = load <8 x i32>, <8 x i32>* %b
   %res = urem <8 x i32> %op1, %op2
   store <8 x i32> %res, <8 x i32>* %a
   ret void
 }
 
 define void @urem_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 {
 ; CHECK-LABEL: urem_v16i32:
 ; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
 ; VBITS_GE_512-NEXT: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_512-NEXT: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
 ; VBITS_GE_512-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_512-NEXT: udiv [[DIV:z[0-9]+]].s, [[PG]]/m, [[PFX]].s, [[OP2]].s
-; VBITS_GE_512-NEXT: mul [[MUL:z[0-9]+]].s, [[PG]]/m, [[OP2]].s, [[DIV]].s
-; VBITS_GE_512-NEXT: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[MUL]].s
-; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: mls [[OP1]].s, [[PG]]/m, [[DIV]].s, [[OP2]].s
+; VBITS_GE_512-NEXT: st1w { [[OP1]].s }, [[PG]], [x0]
 ; VBITS_GE_512-NEXT: ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = urem <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 define void @urem_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 {
 ; CHECK-LABEL: urem_v32i32:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
 ; VBITS_GE_1024-NEXT: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_1024-NEXT: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
 ; VBITS_GE_1024-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_1024-NEXT: udiv [[DIV:z[0-9]+]].s, [[PG]]/m, [[PFX]].s, [[OP2]].s
-; VBITS_GE_1024-NEXT: mul [[MUL:z[0-9]+]].s, [[PG]]/m, [[OP2]].s, [[DIV]].s
-; VBITS_GE_1024-NEXT: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[MUL]].s
-; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: mls [[OP1]].s, [[PG]]/m, [[DIV]].s, [[OP2]].s
+; VBITS_GE_1024-NEXT: st1w { [[OP1]].s }, [[PG]], [x0]
 ; VBITS_GE_1024-NEXT: ret
   %op1 = load <32 x i32>, <32 x i32>* %a
   %op2 = load <32 x i32>, <32 x i32>* %b
   %res = urem <32 x i32> %op1, %op2
   store <32 x i32> %res, <32 x i32>* %a
   ret void
 }
 
 define void @urem_v64i32(<64 x i32>* %a, <64 x i32>* %b) #0 {
 ; CHECK-LABEL: urem_v64i32:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
 ; VBITS_GE_2048-NEXT: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_2048-NEXT: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
 ; VBITS_GE_2048-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_2048-NEXT: udiv [[DIV:z[0-9]+]].s, [[PG]]/m, [[PFX]].s, [[OP2]].s
-; VBITS_GE_2048-NEXT: mul [[MUL:z[0-9]+]].s, [[PG]]/m, [[OP2]].s, [[DIV]].s
-; VBITS_GE_2048-NEXT: sub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[MUL]].s
-; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: mls [[OP1]].s, [[PG]]/m, [[DIV]].s, [[OP2]].s
+; VBITS_GE_2048-NEXT: st1w { [[OP1]].s }, [[PG]], [x0]
 ; VBITS_GE_2048-NEXT: ret
   %op1 = load <64 x i32>, <64 x i32>* %a
   %op2 = load <64 x i32>, <64 x i32>* %b
   %res = urem <64 x i32> %op1, %op2
   store <64 x i32> %res, <64 x i32>* %a
   ret void
 }
 

 
 define void @urem_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 {
 ; CHECK-LABEL: urem_v4i64:
 ; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
 ; CHECK-NEXT: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; CHECK-NEXT: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
 ; CHECK-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; CHECK-NEXT: udiv [[DIV:z[0-9]+]].d, [[PG]]/m, [[PFX]].d, [[OP2]].d
-; CHECK-NEXT: mul [[MUL:z[0-9]+]].d, [[PG]]/m, [[OP2]].d, [[DIV]].d
-; CHECK-NEXT: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[MUL]].d
-; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK-NEXT: mls [[OP1]].d, [[PG]]/m, [[DIV]].d, [[OP2]].d
+; CHECK-NEXT: st1d { [[OP1]].d }, [[PG]], [x0]
 ; CHECK-NEXT: ret
   %op1 = load <4 x i64>, <4 x i64>* %a
   %op2 = load <4 x i64>, <4 x i64>* %b
   %res = urem <4 x i64> %op1, %op2
   store <4 x i64> %res, <4 x i64>* %a
   ret void
 }
 
 define void @urem_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 {
 ; CHECK-LABEL: urem_v8i64:
 ; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
 ; VBITS_GE_512-NEXT: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_512-NEXT: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
 ; VBITS_GE_512-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_512-NEXT: udiv [[DIV:z[0-9]+]].d, [[PG]]/m, [[PFX]].d, [[OP2]].d
-; VBITS_GE_512-NEXT: mul [[MUL:z[0-9]+]].d, [[PG]]/m, [[OP2]].d, [[DIV]].d
-; VBITS_GE_512-NEXT: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[MUL]].d
-; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: mls [[OP1]].d, [[PG]]/m, [[DIV]].d, [[OP2]].d
+; VBITS_GE_512-NEXT: st1d { [[OP1]].d }, [[PG]], [x0]
 ; VBITS_GE_512-NEXT: ret
   %op1 = load <8 x i64>, <8 x i64>* %a
   %op2 = load <8 x i64>, <8 x i64>* %b
   %res = urem <8 x i64> %op1, %op2
   store <8 x i64> %res, <8 x i64>* %a
   ret void
 }
 
 define void @urem_v16i64(<16 x i64>* %a, <16 x i64>* %b) #0 {
 ; CHECK-LABEL: urem_v16i64:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
 ; VBITS_GE_1024-NEXT: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_1024-NEXT: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
 ; VBITS_GE_1024-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_1024-NEXT: udiv [[DIV:z[0-9]+]].d, [[PG]]/m, [[PFX]].d, [[OP2]].d
-; VBITS_GE_1024-NEXT: mul [[MUL:z[0-9]+]].d, [[PG]]/m, [[OP2]].d, [[DIV]].d
-; VBITS_GE_1024-NEXT: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[MUL]].d
-; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: mls [[OP1]].d, [[PG]]/m, [[DIV]].d, [[OP2]].d
+; VBITS_GE_1024-NEXT: st1d { [[OP1]].d }, [[PG]], [x0]
 ; VBITS_GE_1024-NEXT: ret
   %op1 = load <16 x i64>, <16 x i64>* %a
   %op2 = load <16 x i64>, <16 x i64>* %b
   %res = urem <16 x i64> %op1, %op2
   store <16 x i64> %res, <16 x i64>* %a
   ret void
 }
 
 define void @urem_v32i64(<32 x i64>* %a, <32 x i64>* %b) #0 {
 ; CHECK-LABEL: urem_v32i64:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
 ; VBITS_GE_2048-NEXT: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_2048-NEXT: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
 ; VBITS_GE_2048-NEXT: movprfx [[PFX:z[0-9]+]], [[OP1]]
 ; VBITS_GE_2048-NEXT: udiv [[DIV:z[0-9]+]].d, [[PG]]/m, [[PFX]].d, [[OP2]].d
-; VBITS_GE_2048-NEXT: mul [[MUL:z[0-9]+]].d, [[PG]]/m, [[OP2]].d, [[DIV]].d
-; VBITS_GE_2048-NEXT: sub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[MUL]].d
-; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: mls [[OP1]].d, [[PG]]/m, [[DIV]].d, [[OP2]].d
+; VBITS_GE_2048-NEXT: st1d { [[OP1]].d }, [[PG]], [x0]
 ; VBITS_GE_2048-NEXT: ret
   %op1 = load <32 x i64>, <32 x i64>* %a
   %op2 = load <32 x i64>, <32 x i64>* %b
   %res = urem <32 x i64> %op1, %op2
   store <32 x i64> %res, <32 x i64>* %a
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-fixed-length-limit-duplane.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
 define <4 x i32> @test(<16 x i32>* %arg1, <16 x i32>* %arg2) {
 ; CHECK-LABEL: test:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    mov x8, #8
 ; CHECK-NEXT:    ptrue p0.s, vl8
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0]
 ; CHECK-NEXT:    mov z0.d, z1.d
-; CHECK-NEXT:    add z2.s, p0/m, z2.s, z2.s
+; CHECK-NEXT:    add z2.s, z2.s, z2.s
 ; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #16
-; CHECK-NEXT:    add z1.s, p0/m, z1.s, z1.s
+; CHECK-NEXT:    add z1.s, z1.s, z1.s
 ; CHECK-NEXT:    dup v0.4s, v0.s[2]
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0, x8, lsl #2]
 ; CHECK-NEXT:    st1w { z2.s }, p0, [x0]
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <16 x i32>, <16 x i32>* %arg1, align 256
   %1 = load <16 x i32>, <16 x i32>* %arg2, align 256
   %shvec = shufflevector <16 x i32> %0, <16 x i32> %1, <4 x i32> <i32 14, i32 14, i32 14, i32 14>
   %2 = add <16 x i32> %0, %0
   store <16 x i32> %2, <16 x i32>* %arg1, align 256
   ret <4 x i32> %shvec
 }
 
 define <2 x i32> @test2(<16 x i32>* %arg1, <16 x i32>* %arg2) {
 ; CHECK-LABEL: test2:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    mov x8, #8
 ; CHECK-NEXT:    ptrue p0.s, vl8
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0]
 ; CHECK-NEXT:    mov z0.d, z1.d
-; CHECK-NEXT:    add z2.s, p0/m, z2.s, z2.s
+; CHECK-NEXT:    add z2.s, z2.s, z2.s
 ; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #24
-; CHECK-NEXT:    add z1.s, p0/m, z1.s, z1.s
+; CHECK-NEXT:    add z1.s, z1.s, z1.s
 ; CHECK-NEXT:    dup v0.2s, v0.s[0]
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0, x8, lsl #2]
 ; CHECK-NEXT:    st1w { z2.s }, p0, [x0]
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <16 x i32>, <16 x i32>* %arg1, align 256
   %1 = load <16 x i32>, <16 x i32>* %arg2, align 256
   %shvec = shufflevector <16 x i32> %0, <16 x i32> %1, <2 x i32> <i32 14, i32 14>
   %2 = add <16 x i32> %0, %0
   store <16 x i32> %2, <16 x i32>* %arg1, align 256
   ret <2 x i32> %shvec
 }

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll

 define void @masked_gather_vec_plus_reg(<32 x float>* %a, <32 x i8*>* %b, i64 %off) #0 {
 ; VBITS_GE_2048-LABEL: masked_gather_vec_plus_reg:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.s, vl32
 ; VBITS_GE_2048-NEXT:    ptrue p1.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1d { z1.d }, p1/z, [x1]
 ; VBITS_GE_2048-NEXT:    mov z2.d, x2
-; VBITS_GE_2048-NEXT:    fcmeq p2.s, p0/z, z0.s, #0.0
-; VBITS_GE_2048-NEXT:    movprfx z0, z1
-; VBITS_GE_2048-NEXT:    add z0.d, p1/m, z0.d, z2.d
-; VBITS_GE_2048-NEXT:    punpklo p1.h, p2.b
+; VBITS_GE_2048-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
+; VBITS_GE_2048-NEXT:    add z0.d, z1.d, z2.d
+; VBITS_GE_2048-NEXT:    punpklo p1.h, p1.b
 ; VBITS_GE_2048-NEXT:    ld1w { z0.d }, p1/z, [z0.d]
 ; VBITS_GE_2048-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    ret
   %cvals = load <32 x float>, <32 x float>* %a
   %bases = load <32 x i8*>, <32 x i8*>* %b
   %byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 %off
   %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*>
   %mask = fcmp oeq <32 x float> %cvals, zeroinitializer
   %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x float*> %ptrs, i32 8, <32 x i1> %mask, <32 x float> undef)
   store <32 x float> %vals, <32 x float>* %a
   ret void
 }
 
 ; FIXME: This case does not yet codegen well due to deficiencies in opcode selection
 define void @masked_gather_vec_plus_imm(<32 x float>* %a, <32 x i8*>* %b) #0 {
 ; VBITS_GE_2048-LABEL: masked_gather_vec_plus_imm:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.s, vl32
 ; VBITS_GE_2048-NEXT:    ptrue p1.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1d { z1.d }, p1/z, [x1]
-; VBITS_GE_2048-NEXT:    mov z2.d, #4 // =0x4
-; VBITS_GE_2048-NEXT:    fcmeq p2.s, p0/z, z0.s, #0.0
-; VBITS_GE_2048-NEXT:    movprfx z0, z1
-; VBITS_GE_2048-NEXT:    add z0.d, p1/m, z0.d, z2.d
-; VBITS_GE_2048-NEXT:    punpklo p1.h, p2.b
-; VBITS_GE_2048-NEXT:    ld1w { z0.d }, p1/z, [z0.d]
+; VBITS_GE_2048-NEXT:    fcmeq p1.s, p0/z, z0.s, #0.0
+; VBITS_GE_2048-NEXT:    add z1.d, z1.d, #4
+; VBITS_GE_2048-NEXT:    punpklo p1.h, p1.b
+; VBITS_GE_2048-NEXT:    ld1w { z0.d }, p1/z, [z1.d]
 ; VBITS_GE_2048-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    ret
   %cvals = load <32 x float>, <32 x float>* %a
   %bases = load <32 x i8*>, <32 x i8*>* %b
   %byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 4
   %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*>
   %mask = fcmp oeq <32 x float> %cvals, zeroinitializer

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll

 ; VBITS_GE_2048-LABEL: masked_scatter_vec_plus_reg:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.s, vl32
 ; VBITS_GE_2048-NEXT:    ptrue p1.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1d { z1.d }, p1/z, [x1]
 ; VBITS_GE_2048-NEXT:    mov z2.d, x2
 ; VBITS_GE_2048-NEXT:    fcmeq p0.s, p0/z, z0.s, #0.0
-; VBITS_GE_2048-NEXT:    add z1.d, p1/m, z1.d, z2.d
+; VBITS_GE_2048-NEXT:    add z1.d, z1.d, z2.d
 ; VBITS_GE_2048-NEXT:    uunpklo z0.d, z0.s
 ; VBITS_GE_2048-NEXT:    punpklo p0.h, p0.b
 ; VBITS_GE_2048-NEXT:    st1w { z0.d }, p0, [z1.d]
 ; VBITS_GE_2048-NEXT:    ret
   %vals = load <32 x float>, <32 x float>* %a
   %bases = load <32 x i8*>, <32 x i8*>* %b
   %byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 %off
   %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*>
   %mask = fcmp oeq <32 x float> %vals, zeroinitializer
   call void @llvm.masked.scatter.v32f32(<32 x float> %vals, <32 x float*> %ptrs, i32 8, <32 x i1> %mask)
   ret void
 }
 
 ; FIXME: This case does not yet codegen well due to deficiencies in opcode selection
 define void @masked_scatter_vec_plus_imm(<32 x float>* %a, <32 x i8*>* %b) #0 {
 ; VBITS_GE_2048-LABEL: masked_scatter_vec_plus_imm:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.s, vl32
 ; VBITS_GE_2048-NEXT:    ptrue p1.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1d { z1.d }, p1/z, [x1]
-; VBITS_GE_2048-NEXT:    mov z2.d, #4 // =0x4
 ; VBITS_GE_2048-NEXT:    fcmeq p0.s, p0/z, z0.s, #0.0
-; VBITS_GE_2048-NEXT:    add z1.d, p1/m, z1.d, z2.d
+; VBITS_GE_2048-NEXT:    add z1.d, z1.d, #4
 ; VBITS_GE_2048-NEXT:    uunpklo z0.d, z0.s
 ; VBITS_GE_2048-NEXT:    punpklo p0.h, p0.b
 ; VBITS_GE_2048-NEXT:    st1w { z0.d }, p0, [z1.d]
 ; VBITS_GE_2048-NEXT:    ret
   %vals = load <32 x float>, <32 x float>* %a
   %bases = load <32 x i8*>, <32 x i8*>* %b
   %byte_ptrs = getelementptr i8, <32 x i8*> %bases, i64 4
   %ptrs = bitcast <32 x i8*> %byte_ptrs to <32 x float*>

llvm/test/CodeGen/AArch64/sve-fixed-length-optimize-ptrue.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -aarch64-sve-vector-bits-min=512 -aarch64-sve-vector-bits-max=512  < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
 define void @add_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 {
 ; CHECK-LABEL: add_v64i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
-; CHECK-NEXT:    add z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT:    add z0.b, z0.b, z1.b
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %res = add <64 x i8> %op1, %op2
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 
 define void @add_v32i16(<32 x i16>* %a, <32 x i16>* %b, <32 x i16>* %c) #0 {
 ; CHECK-LABEL: add_v32i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; CHECK-NEXT:    add z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %res = add <32 x i16> %op1, %op2
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }

llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll

 ; VBITS_EQ_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
 ; VBITS_EQ_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    ld1h { z2.h }, p0/z, [x1, x8, lsl #1]
 ; VBITS_EQ_256-NEXT:    ld1h { z3.h }, p0/z, [x1]
 ; VBITS_EQ_256-NEXT:    zip2 z4.h, z1.h, z3.h
 ; VBITS_EQ_256-NEXT:    zip1 z1.h, z1.h, z3.h
 ; VBITS_EQ_256-NEXT:    zip2 z3.h, z0.h, z2.h
 ; VBITS_EQ_256-NEXT:    zip1 z0.h, z0.h, z2.h
-; VBITS_EQ_256-NEXT:    add z0.h, p0/m, z0.h, z1.h
-; VBITS_EQ_256-NEXT:    movprfx z1, z4
-; VBITS_EQ_256-NEXT:    add z1.h, p0/m, z1.h, z3.h
+; VBITS_EQ_256-NEXT:    add z0.h, z1.h, z0.h
+; VBITS_EQ_256-NEXT:    add z1.h, z4.h, z3.h
 ; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0, x8, lsl #1]
 ; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_EQ_512-LABEL: zip_v32i16:
 ; VBITS_EQ_512:       // %bb.0:
 ; VBITS_EQ_512-NEXT:    ptrue p0.h
 ; VBITS_EQ_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_EQ_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_EQ_512-NEXT:    zip1 z2.h, z0.h, z1.h
 ; VBITS_EQ_512-NEXT:    zip2 z0.h, z0.h, z1.h
-; VBITS_EQ_512-NEXT:    add z0.h, p0/m, z0.h, z2.h
+; VBITS_EQ_512-NEXT:    add z0.h, z2.h, z0.h
 ; VBITS_EQ_512-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_EQ_512-NEXT:    ret
   %tmp1 = load <32 x i16>, <32 x i16>* %a
   %tmp2 = load <32 x i16>, <32 x i16>* %b
   %tmp3 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47>
   %tmp4 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   %tmp5 = add <32 x i16> %tmp3, %tmp4
   store <32 x i16> %tmp5, <32 x i16>* %a

 define void @trn_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 {
 ; VBITS_EQ_256-LABEL: trn_v32i8:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    ptrue p0.b
 ; VBITS_EQ_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    ld1b { z1.b }, p0/z, [x1]
 ; VBITS_EQ_256-NEXT:    trn1 z2.b, z0.b, z1.b
 ; VBITS_EQ_256-NEXT:    trn2 z0.b, z0.b, z1.b
-; VBITS_EQ_256-NEXT:    add z0.b, p0/m, z0.b, z2.b
+; VBITS_EQ_256-NEXT:    add z0.b, z2.b, z0.b
 ; VBITS_EQ_256-NEXT:    st1b { z0.b }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_EQ_512-LABEL: trn_v32i8:
 ; VBITS_EQ_512:       // %bb.0:
 ; VBITS_EQ_512-NEXT:    ptrue p0.b, vl32
 ; VBITS_EQ_512-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; VBITS_EQ_512-NEXT:    ld1b { z1.b }, p0/z, [x1]
 ; VBITS_EQ_512-NEXT:    trn1 z2.b, z0.b, z1.b
 ; VBITS_EQ_512-NEXT:    trn2 z0.b, z0.b, z1.b
-; VBITS_EQ_512-NEXT:    add z0.b, p0/m, z0.b, z2.b
+; VBITS_EQ_512-NEXT:    add z0.b, z2.b, z0.b
 ; VBITS_EQ_512-NEXT:    st1b { z0.b }, p0, [x0]
 ; VBITS_EQ_512-NEXT:    ret
   %tmp1 = load <32 x i8>, <32 x i8>* %a
   %tmp2 = load <32 x i8>, <32 x i8>* %b
   %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
   %tmp4 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 1, i32 33, i32 3, i32 35, i32 undef, i32 37, i32 7, i32 undef, i32 undef, i32 41, i32 11, i32 43, i32 13, i32 45, i32 15, i32 47, i32 17, i32 49, i32 19, i32 51, i32 21, i32 53, i32 23, i32 55, i32 25, i32 57, i32 27, i32 59, i32 29, i32 61, i32 31, i32 63>
   %tmp5 = add <32 x i8> %tmp3, %tmp4
   store <32 x i8> %tmp5, <32 x i8>* %a
   ret void
 }
 
 define void @trn_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 {
 ; VBITS_EQ_256-LABEL: trn_v32i16:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    mov x8, #16
 ; VBITS_EQ_256-NEXT:    ptrue p0.h
 ; VBITS_EQ_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
 ; VBITS_EQ_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    ld1h { z2.h }, p0/z, [x1, x8, lsl #1]
 ; VBITS_EQ_256-NEXT:    ld1h { z3.h }, p0/z, [x1]
 ; VBITS_EQ_256-NEXT:    trn1 z4.h, z0.h, z2.h
 ; VBITS_EQ_256-NEXT:    trn1 z5.h, z1.h, z3.h
 ; VBITS_EQ_256-NEXT:    trn2 z0.h, z0.h, z2.h
 ; VBITS_EQ_256-NEXT:    trn2 z1.h, z1.h, z3.h
-; VBITS_EQ_256-NEXT:    add z0.h, p0/m, z0.h, z4.h
-; VBITS_EQ_256-NEXT:    add z1.h, p0/m, z1.h, z5.h
+; VBITS_EQ_256-NEXT:    add z0.h, z4.h, z0.h
+; VBITS_EQ_256-NEXT:    add z1.h, z5.h, z1.h
 ; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
 ; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_EQ_512-LABEL: trn_v32i16:
 ; VBITS_EQ_512:       // %bb.0:
 ; VBITS_EQ_512-NEXT:    ptrue p0.h
 ; VBITS_EQ_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_EQ_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_EQ_512-NEXT:    trn1 z2.h, z0.h, z1.h
 ; VBITS_EQ_512-NEXT:    trn2 z0.h, z0.h, z1.h
-; VBITS_EQ_512-NEXT:    add z0.h, p0/m, z0.h, z2.h
+; VBITS_EQ_512-NEXT:    add z0.h, z2.h, z0.h
 ; VBITS_EQ_512-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_EQ_512-NEXT:    ret
   %tmp1 = load <32 x i16>, <32 x i16>* %a
   %tmp2 = load <32 x i16>, <32 x i16>* %b
   %tmp3 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
   %tmp4 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 1, i32 33, i32 3, i32 35, i32 undef, i32 37, i32 7, i32 undef, i32 undef, i32 41, i32 11, i32 43, i32 13, i32 45, i32 15, i32 47, i32 17, i32 49, i32 19, i32 51, i32 21, i32 53, i32 23, i32 55, i32 25, i32 57, i32 27, i32 59, i32 29, i32 61, i32 31, i32 63>
   %tmp5 = add <32 x i16> %tmp3, %tmp4
   store <32 x i16> %tmp5, <32 x i16>* %a
   ret void
 }
 
 define void @trn_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 {
 ; VBITS_EQ_256-LABEL: trn_v16i16:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    ptrue p0.h
 ; VBITS_EQ_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_EQ_256-NEXT:    trn1 z2.h, z0.h, z1.h
 ; VBITS_EQ_256-NEXT:    trn2 z0.h, z0.h, z1.h
-; VBITS_EQ_256-NEXT:    add z0.h, p0/m, z0.h, z2.h
+; VBITS_EQ_256-NEXT:    add z0.h, z2.h, z0.h
 ; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_EQ_512-LABEL: trn_v16i16:
 ; VBITS_EQ_512:       // %bb.0:
 ; VBITS_EQ_512-NEXT:    ptrue p0.h, vl16
 ; VBITS_EQ_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_EQ_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_EQ_512-NEXT:    trn1 z2.h, z0.h, z1.h
 ; VBITS_EQ_512-NEXT:    trn2 z0.h, z0.h, z1.h
-; VBITS_EQ_512-NEXT:    add z0.h, p0/m, z0.h, z2.h
+; VBITS_EQ_512-NEXT:    add z0.h, z2.h, z0.h
 ; VBITS_EQ_512-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_EQ_512-NEXT:    ret
   %tmp1 = load <16 x i16>, <16 x i16>* %a
   %tmp2 = load <16 x i16>, <16 x i16>* %b
   %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
   %tmp4 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>
   %tmp5 = add <16 x i16> %tmp3, %tmp4
   store <16 x i16> %tmp5, <16 x i16>* %a
   ret void
 }
 
 define void @trn_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
 ; VBITS_EQ_256-LABEL: trn_v8i32:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    ptrue p0.s
 ; VBITS_EQ_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; VBITS_EQ_256-NEXT:    trn1 z2.s, z0.s, z1.s
 ; VBITS_EQ_256-NEXT:    trn2 z0.s, z0.s, z1.s
-; VBITS_EQ_256-NEXT:    add z0.s, p0/m, z0.s, z2.s
+; VBITS_EQ_256-NEXT:    add z0.s, z2.s, z0.s
 ; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_EQ_512-LABEL: trn_v8i32:
 ; VBITS_EQ_512:       // %bb.0:
 ; VBITS_EQ_512-NEXT:    ptrue p0.s, vl8
 ; VBITS_EQ_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_EQ_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; VBITS_EQ_512-NEXT:    trn1 z2.s, z0.s, z1.s
 ; VBITS_EQ_512-NEXT:    trn2 z0.s, z0.s, z1.s
-; VBITS_EQ_512-NEXT:    add z0.s, p0/m, z0.s, z2.s
+; VBITS_EQ_512-NEXT:    add z0.s, z2.s, z0.s
 ; VBITS_EQ_512-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_EQ_512-NEXT:    ret
   %tmp1 = load <8 x i32>, <8 x i32>* %a
   %tmp2 = load <8 x i32>, <8 x i32>* %b
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 6, i32 14>
   %tmp4 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 1, i32 undef, i32 3, i32 11, i32 5, i32 13, i32 undef, i32 undef>
   %tmp5 = add <8 x i32> %tmp3, %tmp4
   store <8 x i32> %tmp5, <8 x i32>* %a

 
 define void @trn_v8i32_undef(<8 x i32>* %a) #0 {
 ; VBITS_EQ_256-LABEL: trn_v8i32_undef:
 ; VBITS_EQ_256:       // %bb.0:
 ; VBITS_EQ_256-NEXT:    ptrue p0.s
 ; VBITS_EQ_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_EQ_256-NEXT:    trn1 z1.s, z0.s, z0.s
 ; VBITS_EQ_256-NEXT:    trn2 z0.s, z0.s, z0.s
-; VBITS_EQ_256-NEXT:    add z0.s, p0/m, z0.s, z1.s
+; VBITS_EQ_256-NEXT:    add z0.s, z1.s, z0.s
 ; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_EQ_512-LABEL: trn_v8i32_undef:
 ; VBITS_EQ_512:       // %bb.0:
 ; VBITS_EQ_512-NEXT:    ptrue p0.s, vl8
 ; VBITS_EQ_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_EQ_512-NEXT:    trn1 z1.s, z0.s, z0.s
 ; VBITS_EQ_512-NEXT:    trn2 z0.s, z0.s, z0.s
-; VBITS_EQ_512-NEXT:    add z0.s, p0/m, z0.s, z1.s
+; VBITS_EQ_512-NEXT:    add z0.s, z1.s, z0.s
 ; VBITS_EQ_512-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_EQ_512-NEXT:    ret
   %tmp1 = load <8 x i32>, <8 x i32>* %a
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   %tmp4 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
   %tmp5 = add <8 x i32> %tmp3, %tmp4
   store <8 x i32> %tmp5, <8 x i32>* %a
   ret void

 define void @uzp_v32i8(<32 x i8>* %a, <32 x i8>* %b) #1 {
 ; CHECK-LABEL: uzp_v32i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
 ; CHECK-NEXT:    uzp1 z2.b, z0.b, z1.b
 ; CHECK-NEXT:    uzp2 z0.b, z0.b, z1.b
-; CHECK-NEXT:    add z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT:    add z0.b, z2.b, z0.b
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    ret
   %tmp1 = load <32 x i8>, <32 x i8>* %a
   %tmp2 = load <32 x i8>, <32 x i8>* %b
   %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
   %tmp4 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 1, i32 3, i32 5, i32 undef, i32 9, i32 11, i32 13, i32 undef, i32 undef, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
   %tmp5 = add <32 x i8> %tmp3, %tmp4
   store <32 x i8> %tmp5, <32 x i8>* %a

 ; CHECK-NEXT:    mov x8, #16
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x1, x8, lsl #1]
 ; CHECK-NEXT:    ld1h { z3.h }, p0/z, [x1]
 ; CHECK-NEXT:    uzp1 z5.h, z1.h, z0.h
 ; CHECK-NEXT:    uzp2 z0.h, z1.h, z0.h
-; CHECK-NEXT:    add z0.h, p0/m, z0.h, z5.h
 ; CHECK-NEXT:    uzp1 z4.h, z3.h, z2.h
 ; CHECK-NEXT:    uzp2 z2.h, z3.h, z2.h
-; CHECK-NEXT:    movprfx z1, z4
-; CHECK-NEXT:    add z1.h, p0/m, z1.h, z2.h
+; CHECK-NEXT:    add z0.h, z5.h, z0.h
+; CHECK-NEXT:    add z1.h, z4.h, z2.h
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x0, x8, lsl #1]
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %tmp1 = load <32 x i16>, <32 x i16>* %a
   %tmp2 = load <32 x i16>, <32 x i16>* %b
   %tmp3 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
   %tmp4 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 1, i32 3, i32 5, i32 undef, i32 9, i32 11, i32 13, i32 undef, i32 undef, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
   %tmp5 = add <32 x i16> %tmp3, %tmp4
   store <32 x i16> %tmp5, <32 x i16>* %a
   ret void
 }
 
 ; Emit uzp1/2 instruction for v16i16 shuffle with vscale_range(2,2),
 ; since the size of v16i16 is the same as the runtime vector length.
 define void @uzp_v16i16(<16 x i16>* %a, <16 x i16>* %b) #1 {
 ; CHECK-LABEL: uzp_v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; CHECK-NEXT:    uzp1 z2.h, z0.h, z1.h
 ; CHECK-NEXT:    uzp2 z0.h, z0.h, z1.h
-; CHECK-NEXT:    add z0.h, p0/m, z0.h, z2.h
+; CHECK-NEXT:    add z0.h, z2.h, z0.h
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %tmp1 = load <16 x i16>, <16 x i16>* %a
   %tmp2 = load <16 x i16>, <16 x i16>* %b
   %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %tmp4 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %tmp5 = add <16 x i16> %tmp3, %tmp4
   store <16 x i16> %tmp5, <16 x i16>* %a

 define void @uzp_v4i64(<4 x i64>* %a, <4 x i64>* %b) #1 {
 ; CHECK-LABEL: uzp_v4i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    uzp1 z2.d, z0.d, z1.d
 ; CHECK-NEXT:    uzp2 z0.d, z0.d, z1.d
-; CHECK-NEXT:    add z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT:    add z0.d, z2.d, z0.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
   %tmp1 = load <4 x i64>, <4 x i64>* %a
   %tmp2 = load <4 x i64>, <4 x i64>* %b
   %tmp3 = shufflevector <4 x i64> %tmp1, <4 x i64> %tmp2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %tmp4 = shufflevector <4 x i64> %tmp1, <4 x i64> %tmp2, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
   %tmp5 = add <4 x i64> %tmp3, %tmp4
   store <4 x i64> %tmp5, <4 x i64>* %a

 ; Emit uzp1/2 instruction for v8i32 and undef shuffle with vscale_range(2,2)
 define void @uzp_v8i32_undef(<8 x i32>* %a) #1 {
 ; CHECK-LABEL: uzp_v8i32_undef:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
 ; CHECK-NEXT:    uzp2 z0.s, z0.s, z0.s
-; CHECK-NEXT:    add z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    add z0.s, z1.s, z0.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %tmp1 = load <8 x i32>, <8 x i32>* %a
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
   %tmp4 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 1, i32 3, i32 5, i32 7>
   %tmp5 = add <8 x i32> %tmp3, %tmp4
   store <8 x i32> %tmp5, <8 x i32>* %a
   ret void

llvm/test/CodeGen/AArch64/sve-fixed-length-trunc.ll

 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v32i16_v32i8(<32 x i16>* %in, <32 x i8>* %out) #0 {
 ; CHECK-LABEL: trunc_v32i16_v32i8:
 ; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
 ; VBITS_GE_512: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; VBITS_GE_512: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
-; VBITS_GE_512: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+; VBITS_GE_512: add [[A_BYTES]].b, [[A_BYTES]].b, [[A_BYTES]].b
   %a = load <32 x i16>, <32 x i16>* %in
   %b = trunc <32 x i16> %a to <32 x i8>
   %c = add <32 x i8> %b, %b
   store <32 x i8> %c, <32 x i8>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v64i16_v64i8(<64 x i16>* %in, <64 x i8>* %out) #0 {
 ; CHECK-LABEL: trunc_v64i16_v64i8:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
 ; VBITS_GE_1024: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; VBITS_GE_1024: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
-; VBITS_GE_1024: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+; VBITS_GE_1024: add [[A_BYTES]].b, [[A_BYTES]].b, [[A_BYTES]].b
   %a = load <64 x i16>, <64 x i16>* %in
   %b = trunc <64 x i16> %a to <64 x i8>
   %c = add <64 x i8> %b, %b
   store <64 x i8> %c, <64 x i8>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v128i16_v128i8(<128 x i16>* %in, <128 x i8>* %out) #0 {
 ; CHECK-LABEL: trunc_v128i16_v128i8:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
 ; VBITS_GE_2048: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
 ; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
-; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[A_BYTES]].b, [[A_BYTES]].b
   %a = load <128 x i16>, <128 x i16>* %in
   %b = trunc <128 x i16> %a to <128 x i8>
   %c = add <128 x i8> %b, %b
   store <128 x i8> %c, <128 x i8>* %out
   ret void
 }
 
 ;

 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v32i32_v32i8(<32 x i32>* %in, <32 x i8>* %out) #0 {
 ; CHECK-LABEL: trunc_v32i32_v32i8:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
 ; VBITS_GE_1024: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
 ; VBITS_GE_1024: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
-; VBITS_GE_1024: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+; VBITS_GE_1024: add [[A_BYTES]].b, [[A_BYTES]].b, [[A_BYTES]].b
   %a = load <32 x i32>, <32 x i32>* %in
   %b = trunc <32 x i32> %a to <32 x i8>
   %c = add <32 x i8> %b, %b
   store <32 x i8> %c, <32 x i8>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v64i32_v64i8(<64 x i32>* %in, <64 x i8>* %out) #0 {
 ; CHECK-LABEL: trunc_v64i32_v64i8:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
 ; VBITS_GE_2048: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
 ; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
-; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[A_BYTES]].b, [[A_BYTES]].b
   %a = load <64 x i32>, <64 x i32>* %in
   %b = trunc <64 x i32> %a to <64 x i8>
   %c = add <64 x i8> %b, %b
   store <64 x i8> %c, <64 x i8>* %out
   ret void
 }
 
 ;

 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v16i32_v16i16(<16 x i32>* %in, <16 x i16>* %out) #0 {
 ; CHECK-LABEL: trunc_v16i32_v16i16:
 ; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
 ; VBITS_GE_512: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_512: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
-; VBITS_GE_512: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+; VBITS_GE_512: add [[A_HALFS]].h, [[A_HALFS]].h, [[A_HALFS]].h
   %a = load <16 x i32>, <16 x i32>* %in
   %b = trunc <16 x i32> %a to <16 x i16>
   %c = add <16 x i16> %b, %b
   store <16 x i16> %c, <16 x i16>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v32i32_v32i16(<32 x i32>* %in, <32 x i16>* %out) #0 {
 ; CHECK-LABEL: trunc_v32i32_v32i16:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
 ; VBITS_GE_1024: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
-; VBITS_GE_1024: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+; VBITS_GE_1024: add [[A_HALFS]].h, [[A_HALFS]].h, [[A_HALFS]].h
   %a = load <32 x i32>, <32 x i32>* %in
   %b = trunc <32 x i32> %a to <32 x i16>
   %c = add <32 x i16> %b, %b
   store <32 x i16> %c, <32 x i16>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v64i32_v64i16(<64 x i32>* %in, <64 x i16>* %out) #0 {
 ; CHECK-LABEL: trunc_v64i32_v64i16:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
 ; VBITS_GE_2048: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
-; VBITS_GE_2048: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+; VBITS_GE_2048: add [[A_HALFS]].h, [[A_HALFS]].h, [[A_HALFS]].h
   %a = load <64 x i32>, <64 x i32>* %in
   %b = trunc <64 x i32> %a to <64 x i16>
   %c = add <64 x i16> %b, %b
   store <64 x i16> %c, <64 x i16>* %out
   ret void
 }
 
 ;

 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v32i64_v32i8(<32 x i64>* %in, <32 x i8>* %out) #0 {
 ; CHECK-LABEL: trunc_v32i64_v32i8:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
 ; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
 ; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
 ; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
-; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[A_BYTES]].b, [[A_BYTES]].b
   %a = load <32 x i64>, <32 x i64>* %in
   %b = trunc <32 x i64> %a to <32 x i8>
   %c = add <32 x i8> %b, %b
   store <32 x i8> %c, <32 x i8>* %out
   ret void
 }
 
 ;

 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v16i64_v16i16(<16 x i64>* %in, <16 x i16>* %out) #0 {
 ; CHECK-LABEL: trunc_v16i64_v16i16:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
 ; VBITS_GE_1024: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_1024: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
 ; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
-; VBITS_GE_1024: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+; VBITS_GE_1024: add [[A_HALFS]].h, [[A_HALFS]].h, [[A_HALFS]].h
   %a = load <16 x i64>, <16 x i64>* %in
   %b = trunc <16 x i64> %a to <16 x i16>
   %c = add <16 x i16> %b, %b
   store <16 x i16> %c, <16 x i16>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v32i64_v32i16(<32 x i64>* %in, <32 x i16>* %out) #0 {
 ; CHECK-LABEL: trunc_v32i64_v32i16:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
 ; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
 ; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
-; VBITS_GE_2048: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+; VBITS_GE_2048: add [[A_HALFS]].h, [[A_HALFS]].h, [[A_HALFS]].h
   %a = load <32 x i64>, <32 x i64>* %in
   %b = trunc <32 x i64> %a to <32 x i16>
   %c = add <32 x i16> %b, %b
   store <32 x i16> %c, <32 x i16>* %out
   ret void
 }
 
 ;

 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v8i64_v8i32(<8 x i64>* %in, <8 x i32>* %out) #0 {
 ; CHECK-LABEL: trunc_v8i64_v8i32:
 ; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
 ; VBITS_GE_512: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_512: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
-; VBITS_GE_512: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+; VBITS_GE_512: add [[A_WORDS]].s, [[A_WORDS]].s, [[A_WORDS]].s
   %a = load <8 x i64>, <8 x i64>* %in
   %b = trunc <8 x i64> %a to <8 x i32>
   %c = add <8 x i32> %b, %b
   store <8 x i32> %c, <8 x i32>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v16i64_v16i32(<16 x i64>* %in, <16 x i32>* %out) #0 {
 ; CHECK-LABEL: trunc_v16i64_v16i32:
 ; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
 ; VBITS_GE_1024: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_1024: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
-; VBITS_GE_1024: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+; VBITS_GE_1024: add [[A_WORDS]].s, [[A_WORDS]].s, [[A_WORDS]].s
   %a = load <16 x i64>, <16 x i64>* %in
   %b = trunc <16 x i64> %a to <16 x i32>
   %c = add <16 x i32> %b, %b
   store <16 x i32> %c, <16 x i32>* %out
   ret void
 }
 
 ; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
 define void @trunc_v32i64_v32i32(<32 x i64>* %in, <32 x i32>* %out) #0 {
 ; CHECK-LABEL: trunc_v32i64_v32i32:
 ; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
 ; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
 ; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
-; VBITS_GE_2048: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+; VBITS_GE_2048: add [[A_WORDS]].s, [[A_WORDS]].s, [[A_WORDS]].s
   %a = load <32 x i64>, <32 x i64>* %in
   %b = trunc <32 x i64> %a to <32 x i32>
   %c = add <32 x i32> %b, %b
   store <32 x i32> %c, <32 x i32>* %out
   ret void
 }
 
 attributes #0 = { nounwind "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll

 ; CHECK-NEXT:    ldr x11, [x0]
 ; CHECK-NEXT:    index z3.d, #0, #1
 ; CHECK-NEXT:    mov z2.d, x9
 ; CHECK-NEXT:    ptrue p1.d, vl1
 ; CHECK-NEXT:    cmpeq p2.d, p0/z, z3.d, z2.d
 ; CHECK-NEXT:    mov z1.d, z0.d
 ; CHECK-NEXT:    mov z0.d, p2/m, x10
 ; CHECK-NEXT:    mov z1.d, p1/m, x11
-; CHECK-NEXT:    add z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    add z0.d, z1.d, z0.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
 ; CHECK-NEXT:    ret
   %A = load <4 x i64>, <4 x i64>* %addr
   %ld1 = load i64, i64* %data
   %vec1 = insertelement <4 x i64> %A, i64 %ld1, i32 0
   %gep = getelementptr i64, i64* %data, i64 %idx
   %ld2 = load i64, i64* %gep
   %vec2 = insertelement <4 x i64> %A, i64 %ld2, i32 2

llvm/test/CodeGen/AArch64/sve-vscale-attr.ll

 ; CHECK-NOARG-NEXT:    stp q2, q3, [x0]
 ; CHECK-NOARG-NEXT:    ret
 ;
 ; CHECK-ARG-LABEL: func_vscale_none:
 ; CHECK-ARG:       // %bb.0:
 ; CHECK-ARG-NEXT:    ptrue p0.s, vl16
 ; CHECK-ARG-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-ARG-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; CHECK-ARG-NEXT:    add z0.s, p0/m, z0.s, z1.s
+; CHECK-ARG-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-ARG-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-ARG-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = add <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }

 ; CHECK-LABEL: func_vscale2_2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov x8, #8
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z3.s }, p0/z, [x1]
-; CHECK-NEXT:    add z0.s, p0/m, z0.s, z2.s
-; CHECK-NEXT:    add z1.s, p0/m, z1.s, z3.s
+; CHECK-NEXT:    add z0.s, z0.s, z2.s
+; CHECK-NEXT:    add z1.s, z1.s, z3.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = add <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 attributes #2 = { "target-features"="+sve" vscale_range(2,2) }
 
 define void @func_vscale2_4(<16 x i32>* %a, <16 x i32>* %b) #3 {
 ; CHECK-LABEL: func_vscale2_4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov x8, #8
 ; CHECK-NEXT:    ptrue p0.s, vl8
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z3.s }, p0/z, [x1]
-; CHECK-NEXT:    add z0.s, p0/m, z0.s, z2.s
-; CHECK-NEXT:    add z1.s, p0/m, z1.s, z3.s
+; CHECK-NEXT:    add z0.s, z0.s, z2.s
+; CHECK-NEXT:    add z1.s, z1.s, z3.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = add <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 attributes #3 = { "target-features"="+sve" vscale_range(2,4) }
 
 define void @func_vscale4_4(<16 x i32>* %a, <16 x i32>* %b) #4 {
 ; CHECK-LABEL: func_vscale4_4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; CHECK-NEXT:    add z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = add <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 attributes #4 = { "target-features"="+sve" vscale_range(4,4) }
 
 define void @func_vscale8_8(<16 x i32>* %a, <16 x i32>* %b) #5 {
 ; CHECK-LABEL: func_vscale8_8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl16
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; CHECK-NEXT:    add z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %res = add <16 x i32> %op1, %op2
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 attributes #5 = { "target-features"="+sve" vscale_range(8,8) }