[SVE][InstrFormats] Explcitly set hasSideEffects for all SVE instructions.

[paulwalker-arm]

Summary

The instruction property hasSideEffects relies on the presence of
tablegen isel patterns when constructing its value, unless
specifically overriden. Since adding SVE scheduling information
we've noticed this property flip-flop as isel patterns have been
updated. To make things consistent (and correct) this patch
explicitly sets the property for all SVE instructions.

This has resulted in the following notable changes:

• Normal load and store instructions no longer report having side effects.
• All prefetch instructions correctly report having side effects.
• FFR related instructions continue to report having side effects. This is likely overkill but I've chosen to remain cautious here.
• Most all integer instructions no longer report having side effects.
• Most all floating point instructions no longer report having side effects, but do now report their potential for raising FP exceptions. I do not know how to test the latter so I've again took a caution route of taging all floating point instructions except for DUPs.
• The conflict detection intrinsics now report they don't touch memory.

NOTE: SVE isel makes significant use of psuedo instructions but this patch makes no effort to update them.

NOTE: We'll need a similar patch for SME but without a scheduling model it'll be harder to verify the results.

[paulwalker-arm]

I cannot say I've double checked all the instructions but the output from the mca tests looks sensible and if nothing else I'm hoping this is a better starting point.

[peterwaller-arm]

I've been down the list of MCA output changes and agree those look good, and your reasoning seems good to me. Accepting as-is, since we think represents an improvement.

I believe the following command (adjusting relative paths) counts the number of instructions with hasSideEffects unset.

bin/llvm-tblgen -I ../../llvm/include -I ../../llvm/lib/Target/AArch64 ../../llvm/lib/Target/AArch64/AArch64.td --dump-json --class=AArch64Inst | \
  jq 'with_entries(select(.value | (type == "object" and has("hasSideEffects") and .hasSideEffects == null))) | length'

With this patch applied (on 5a7f47cc021b) I see the number drop from 4667 to 2198.

(and if you replace | length with | keys it shows the instructions). I do see a fair number of instructions remaining that don't have it set which you might have intended to capture, but I haven't dug in. These are the top ten, aggregating by three-char prefix:

 46   "ADD",
 48   "FMI",
 51   "FMA",
 64   "SML",
 64   "UML",
 70   "BFM",
 86   "MOV",
 95   "FCV",
 96   "CPY",
102   "FML",

[paulwalker-arm]

Thanks for the top tip @peterwaller-arm . I've analysed the output from a local run and observed:
392 belong to SVE, with all being pseudo instructions (These are the instructions that contain _UNDEF or _ZERO).
896 belong to SME.
757 look like scalar or NEON related.

I'll create a follow on patch for the SVE pseudo instructions, but given there's no easy way to test it I think I'll leave SME until we're further alone with its code generation support.

llvm/include/llvm/IR/IntrinsicsAArch64.td

class SVE2_2VectorArg_Imm_Narrowing_Intrinsic
       : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
                   [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty,
                    llvm_i32_ty],
                   [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 
   class SVE2_CONFLICT_DETECT_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                 [LLVMAnyPointerType<llvm_any_ty>,
-                 LLVMMatchType<1>]>;
+                 LLVMMatchType<1>],
+                [IntrNoMem]>;
 
   class SVE2_3VectorArg_Indexed_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                 [LLVMMatchType<0>,
                  LLVMSubdivide2VectorType<0>,
                  LLVMSubdivide2VectorType<0>,
                  llvm_i32_ty],
                 [IntrNoMem, ImmArg<ArgIndex<3>>]>;

llvm/lib/Target/AArch64/SVEInstrFormats.td

: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern),
   let Inst{16}    = opc{0};
   let Inst{15-10} = 0b111000;
   let Inst{9-5}   = pattern;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pd;
 
   let Defs = !if(!eq (opc{0}, 1), [NZCV], []);
   let ElementSize = pprty.ElementSize;
+  let hasSideEffects = 0;
   let isReMaterializable = 1;
 }
 
 multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>;
   def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>;
   def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>;
   def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>;
: I<(outs PPR8:$Pd), (ins),
   let Inst{23-22} = opc{5-4};
   let Inst{21-19} = 0b011;
   let Inst{18-16} = opc{3-1};
   let Inst{15-10} = 0b111001;
   let Inst{9}     = opc{0};
   let Inst{8-4}   = 0b00000;
   let Inst{3-0}   = Pd;
 
+  let hasSideEffects = 0;
   let isReMaterializable = 1;
 }
 
 multiclass sve_int_pfalse<bits<6> opc, string asm> {
   def NAME : sve_int_pfalse<opc, asm>;
 
   def : InstAlias<"pfalse\t$Pd", (!cast<Instruction>(NAME) PNR8:$Pd), 0>;
 
: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn),
   let Inst{18-16} = opc{3-1};
   let Inst{15-14} = 0b11;
   let Inst{13-10} = Pg;
   let Inst{9}     = opc{0};
   let Inst{8-5}   = Pn;
   let Inst{4-0}   = 0b00000;
 
   let Defs = [NZCV];
+  let hasSideEffects = 0;
   let isCompare = 1;
 }
 
 multiclass sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op,
                          SDPatternOperator op_any> {
   def NAME : sve_int_ptest<opc, asm, op>;
 
   let hasNoSchedulingInfo = 1, isCompare = 1, Defs = [NZCV] in {
: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn),
   let Inst{15-11} = 0b11000;
   let Inst{10-9}  = opc{1-0};
   let Inst{8-5}   = Pg;
   let Inst{4}     = 0;
   let Inst{3-0}   = Pdn;
 
   let Constraints = "$Pdn = $_Pdn";
   let Defs = [NZCV];
-  let isPTestLike = 1;
   let ElementSize = pprty.ElementSize;
+  let hasSideEffects = 0;
+  let isPTestLike = 1;
 }
 
 multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>;
 
   def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;
 }
 
: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn),
   let Inst{8-5}   = Pg;
   let Inst{4-0}   = Rdn;
 
   // Signed 32bit forms require their GPR operand printed.
   let AsmString = !if(!eq(opc{4,2-0}, 0b0000),
                       !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"),
                       !strconcat(asm, "\t$Rdn, $Pg"));
   let Constraints = "$Rdn = $_Rdn";
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_count_r_s32<bits<5> opc, string asm,
                                SDPatternOperator op> {
   def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>;
   def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>;
   def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>;
   def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>;
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm),
   let Inst{15-11} = 0b10000;
   let Inst{10-9}  = opc{1-0};
   let Inst{8-5}   = Pm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_count_v<bits<5> opc, string asm,
                            SDPatternOperator op = null_frag> {
   def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>;
   def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>;
   def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>;
 
: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn),
   let Inst{23-22} = sz8_64;
   let Inst{21-19} = 0b100;
   let Inst{18-16} = opc{3-1};
   let Inst{15-14} = 0b10;
   let Inst{13-10} = Pg;
   let Inst{9}     = opc{0};
   let Inst{8-5}   = Pn;
   let Inst{4-0}   = Rd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_pcount_pred<bits<4> opc, string asm,
                                SDPatternOperator int_op> {
   def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>;
   def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>;
   def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>;
   def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>;
: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
   let Inst{23-22} = opc{2-1};
   let Inst{21-20} = 0b10;
   let Inst{19-16} = imm4;
   let Inst{15-11} = 0b11100;
   let Inst{10}    = opc{0};
   let Inst{9-5}   = pattern;
   let Inst{4-0}   = Rd;
 
+  let hasSideEffects = 0;
   let isReMaterializable = 1;
 }
 
 multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> {
   def NAME : sve_int_count<opc, asm>;
 
   def : InstAlias<asm # "\t$Rd, $pattern",
                   (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>;
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
   let Inst{15-12} = 0b1100;
   let Inst{11-10} = opc{1-0};
   let Inst{9-5}   = pattern;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty,
                             SDPatternOperator op = null_frag,
                             ValueType vt = OtherVT> {
   def NAME : sve_int_countvlv<opc, asm, zprty>;
 
   def : InstAlias<asm # "\t$Zdn, $pattern",
: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
   let Inst{21-20} = 0b11;
   let Inst{19-16} = imm4;
   let Inst{15-11} = 0b11100;
   let Inst{10}    = opc{0};
   let Inst{9-5}   = pattern;
   let Inst{4-0}   = Rdn;
 
   let Constraints = "$Rdn = $_Rdn";
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_pred_pattern_a<bits<3> opc, string asm,
                                   SDPatternOperator op,
                                   SDPatternOperator opcnt> {
   let Predicates = [HasSVEorSME] in {
     def NAME : sve_int_pred_pattern_a<opc, asm>;
 
: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
   let Inst{4-0}   = Rdn;
 
   // Signed 32bit forms require their GPR operand printed.
   let AsmString = !if(!eq(opc{2,0}, 0b00),
                       !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"),
                       !strconcat(asm, "\t$Rdn, $pattern, mul $imm4"));
 
   let Constraints = "$Rdn = $_Rdn";
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm,
                                       SDPatternOperator op> {
   def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>;
 
   def : InstAlias<asm # "\t$Rd, $Rn, $pattern",
                   (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>;
: I<(outs zprty:$Zd), (ins srcRegType:$Rn),
   [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> {
   bits<5> Rn;
   bits<5> Zd;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21-10} = 0b100000001110;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>;
   def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>;
   def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>;
   def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>;
 
: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx),
   bits<7> idx;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = {?,?}; // imm3h
   let Inst{21}    = 0b1;
   let Inst{20-16} = tsz;
   let Inst{15-10} = 0b001000;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_dup_i<string asm> {
   def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> {
     let Inst{23-22} = idx{5-4};
     let Inst{20-17} = idx{3-0};
   }
   def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> {
: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm),
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b001;
   let Inst{12-11} = opc;
   let Inst{10}    = 0b0;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8,  Z_b>;
   def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>;
   def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>;
   def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>;
 
: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm),
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b001;
   let Inst{12-11} = opc;
   let Inst{10}    = 0b1;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_perm_tbx<string asm, bits<2> opc, SDPatternOperator op> {
   def _B : sve2_int_perm_tbx<0b00, opc, asm, ZPR8>;
   def _H : sve2_int_perm_tbx<0b01, opc, asm, ZPR16>;
   def _S : sve2_int_perm_tbx<0b10, opc, asm, ZPR32>;
   def _D : sve2_int_perm_tbx<0b11, opc, asm, ZPR64>;
 
: I<(outs zprty:$Zd), (ins zprty:$Zn),
   []>, Sched<[]> {
   bits<5> Zd;
   bits<5> Zn;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21-10} = 0b111000001110;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>;
   def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>;
   def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>;
   def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>;
 
: I<(outs pprty:$Pd), (ins pprty:$Pn),
   bits<4> Pd;
   bits<4> Pn;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21-9}  = 0b1101000100000;
   let Inst{8-5}   = Pn;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>;
   def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>;
   def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>;
   def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>;
 
: I<(outs zprty1:$Zd), (ins zprty2:$Zn),
   bits<5> Zn;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz16_64;
   let Inst{21-18} = 0b1100;
   let Inst{17-16} = opc;
   let Inst{15-10} = 0b001110;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> {
   def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>;
   def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>;
   def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>;
 
   def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>;
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm),
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21-10} = 0b100100001110;
   let Inst{9-5}   = Rm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>;
   def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>;
   def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>;
   def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>;
 
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm),
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21-10} = 0b110100001110;
   let Inst{9-5}   = Vm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>;
   def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>;
   def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>;
   def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>;
 
: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8),
   let Inst{15-13} = 0b000;
   let Inst{12-10} = imm8{2-0};
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> {
   def NAME : sve_int_perm_extract_i<asm>;
 
   def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255,
                          !cast<Instruction>(NAME)>;
 }
 
 class sve2_int_perm_extract_i_cons<string asm>
 : I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8),
   asm, "\t$Zd, $Zn, $imm8",
   "", []>, Sched<[]> {
   bits<5> Zd;
   bits<5> Zn;
   bits<8> imm8;
   let Inst{31-21} = 0b00000101011;
   let Inst{20-16} = imm8{7-3};
   let Inst{15-13} = 0b000;
   let Inst{12-10} = imm8{2-0};
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 //===----------------------------------------------------------------------===//
 // SVE Vector Select Group
 //===----------------------------------------------------------------------===//
 
 class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty>
 : I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm),
   asm, "\t$Zd, $Pg, $Zn, $Zm",
   "",
   []>, Sched<[]> {
   bits<4> Pg;
   bits<5> Zd;
   bits<5> Zm;
   bits<5> Zn;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-14} = 0b11;
   let Inst{13-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> {
   def _B : sve_int_sel_vvv<0b00, asm, ZPR8>;
   def _H : sve_int_sel_vvv<0b01, asm, ZPR16>;
   def _S : sve_int_sel_vvv<0b10, asm, ZPR32>;
   def _D : sve_int_sel_vvv<0b11, asm, ZPR64>;
 
: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm),
   let Inst{3-0}   = Pd;
 
   // SEL has no predication qualifier.
   let AsmString = !if(!eq(opc, 0b0011),
                       !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"),
                       !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm"));
 
   let Defs = !if(!eq (opc{2}, 1), [NZCV], []);
-
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op,
                             SDPatternOperator op_nopred = null_frag> {
   def NAME : sve_int_pred_log<opc, asm>;
 
   def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
   def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>;
: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13),
   let Inst{21-18} = 0b0000;
   let Inst{17-5}  = imms13;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DecoderMethod = "DecodeSVELogicalImmInstruction";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> {
   def NAME : sve_int_log_imm<opc, asm>;
 
   def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8,  i32, SVELogicalImm8Pat,  !cast<Instruction>(NAME)>;
   def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>;
   def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>;
: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms),
   "",
   []>, Sched<[]> {
   bits<5> Zd;
   bits<13> imms;
   let Inst{31-18} = 0b00000101110000;
   let Inst{17-5} = imms;
   let Inst{4-0} = Zd;
 
-  let isReMaterializable = 1;
   let DecoderMethod = "DecodeSVELogicalImmInstruction";
+  let hasSideEffects = 0;
+  let isReMaterializable = 1;
 }
 
 multiclass sve_int_dup_mask_imm<string asm> {
   def NAME : sve_int_dup_mask_imm<asm>;
 
   def : InstAlias<"dupm $Zd, $imm",
                   (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>;
   def : InstAlias<"dupm $Zd, $imm",
: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b000;
   let Inst{12-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>;
   def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>;
   def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>;
   def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>;
 
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1),
   let Inst{12-10} = Pg;
   let Inst{9-6}   = 0b0000;
   let Inst{5}     = i1;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, string Ps, Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> {
   let DestructiveInstType = DestructiveBinaryImm in {
   def _H : SVEPseudo2Instr<Ps # _H, 1>, sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>;
   def _S : SVEPseudo2Instr<Ps # _S, 1>, sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>;
   def _D : SVEPseudo2Instr<Ps # _D, 1>, sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>;
   }
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps,
                             SDPatternOperator op, DestructiveInstTypeEnum flags,
                             string revname="", bit isReverseInstr=0> {
   let DestructiveInstType = flags in {
   def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>,
            SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, timm32_0_7:$imm3),
   let Inst{18-16} = imm3;
   let Inst{15-10} = 0b100000;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_ftmad<string asm, SDPatternOperator op> {
   def _H : sve_fp_ftmad<0b01, asm, ZPR16>;
   def _S : sve_fp_ftmad<0b10, asm, ZPR32>;
   def _D : sve_fp_ftmad<0b11, asm, ZPR64>;
 
   def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 timm32_0_7:$imm))),
: I<(outs zprty:$Zd), (ins  zprty:$Zn, zprty:$Zm),
   let Inst{31-24} = 0b01100101;
   let Inst{23-22} = sz;
   let Inst{21}    = 0b0;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b000;
   let Inst{12-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op,
                            SDPatternOperator predicated_op = null_frag> {
   def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>;
   def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>;
   def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>;
 
: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm),
   let Inst{14-13} = opc;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let ElementSize = zprty.ElementSize;
   let DestructiveInstType = DestructiveTernaryCommWithRev;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps,
                               SDPatternOperator op, string revname,
                               bit isReverseInstr=0> {
   def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>,
            SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
   def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>,
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za),
   let Inst{14-13} = opc;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op,
                               string revname, bit isReverseInstr> {
   def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>,
            SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
   def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>,
            SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop),
   let Inst{15-12} = 0b0000;
   let Inst{11-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve2p1_fp_bfma_by_indexed_elem<string asm, bits<2> opc> {
   def NAME : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16,
                                          VectorIndexH32b> {
     bits<3> Zm;
     bits<3> iop;
     let Inst{22} = iop{2};
: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop),
   let Inst{31-24} = 0b01100100;
   let Inst{23-22} = sz;
   let Inst{21}    = 0b1;
   let Inst{15-12} = 0b0010;
   let Inst{11}    = o2;
   let Inst{10}    = 0b0;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve2p1_fp_bfmul_by_indexed_elem<string asm> {
   def NAME : sve_fp_fmul_by_indexed_elem<{0, ?}, 0b1, asm, ZPR16, ZPR3b16, VectorIndexH32b> {
     bits<3> Zm;
     bits<3> iop;
     let Inst{22} = iop{2};
     let Inst{20-19} = iop{1-0};
: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm,
   let Inst{14-13} = imm;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_fcmla<string asm, SDPatternOperator op> {
   def _H : sve_fp_fcmla<0b01, asm, ZPR16>;
   def _S : sve_fp_fcmla<0b10, asm, ZPR32>;
   def _D : sve_fp_fcmla<0b11, asm, ZPR64>;
 
   def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))),
: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop,
   let Inst{15-12} = 0b0001;
   let Inst{11-10} = imm;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> {
   def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> {
     bits<3> Zm;
     bits<2> iop;
     let Inst{20-19} = iop;
     let Inst{18-16} = Zm;
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm,
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_fcadd<string asm, SDPatternOperator op> {
   def _H : sve_fp_fcadd<0b01, asm, ZPR16>;
   def _S : sve_fp_fcadd<0b10, asm, ZPR32>;
   def _D : sve_fp_fcadd<0b11, asm, ZPR64>;
 
   def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))),
: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn),
   let Inst{21-18} = 0b0010;
   let Inst{17-16} = opc{1-0};
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve2_fp_convert_down_narrow<string asm, string op> {
   def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>;
   def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>;
 
   def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>;
   def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm,
                                  SDPatternOperator op> {
   def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>;
   def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>;
   def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>;
 
: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm,
   let Inst{11}    = iop{0};
   let Inst{10}    = opc{0};
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve2_fp_mla_long_by_indexed_elem<bits<3> opc, string asm,
                                             ValueType OutVT, ValueType InVT,
                                             SDPatternOperator op> {
   def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>;
   def : SVE_4_Op_Imm_Pat<OutVT, op, OutVT, InVT, InVT, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>;
 }
: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
   let Inst{12-11} = 0b00;
   let Inst{10}    = opc{0};
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve2_fp_mla_long<bits<3> opc, string asm, ValueType OutVT,
                             ValueType InVT, SDPatternOperator op> {
   def NAME : sve2_fp_mla_long<opc, asm>;
   def : SVE_3_Op_Pat<OutVT, op, OutVT, InVT, InVT, !cast<Instruction>(NAME)>;
 }
 
: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6),
   let Inst{31-23} = 0b000001000;
   let Inst{22}    = opc;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Rn;
   let Inst{15-12} = 0b0101;
   let Inst{11}    = streaming_sve;
   let Inst{10-5}  = imm6;
   let Inst{4-0}   = Rd;
+
+  let hasSideEffects = 0;
 }
 
 class sve_int_read_vl_a<bit op, bits<5> opc2, string asm, bit streaming_sve = 0b0>
 : I<(outs GPR64:$Rd), (ins simm6_32b:$imm6),
   asm, "\t$Rd, $imm6",
   "",
   []>, Sched<[]> {
   bits<5> Rd;
   bits<6> imm6;
   let Inst{31-23} = 0b000001001;
   let Inst{22}    = op;
   let Inst{21}    = 0b1;
   let Inst{20-16} = opc2{4-0};
   let Inst{15-12} = 0b0101;
   let Inst{11}    = streaming_sve;
   let Inst{10-5}  = imm6;
   let Inst{4-0}   = Rd;
 
+  let hasSideEffects = 0;
   let isReMaterializable = 1;
 }
 
 //===----------------------------------------------------------------------===//
 // SVE Permute - In Lane Group
 //===----------------------------------------------------------------------===//
 
 class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm,
                                ZPRRegOp zprty>
 : I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
   asm, "\t$Zd, $Zn, $Zm",
   "",
   []>, Sched<[]> {
   bits<5> Zd;
   bits<5> Zm;
   bits<5> Zn;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b011;
   let Inst{12-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm,
                                     SDPatternOperator op> {
   def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>;
   def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>;
   def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>;
   def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>;
: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn),
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveUnaryPassthru;
   let ElementSize = Sz;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_2op_p_zd<bits<7> opc, string asm,
                            RegisterOperand i_zprtype,
                            RegisterOperand o_zprtype,
                            SDPatternOperator int_op,
                            SDPatternOperator ir_op, ValueType vt1,
                            ValueType vt2, ValueType vt3, ElementSizeEnum Sz> {
: I<(outs zprty:$Zd), (ins zprty:$Zn),
   bits<5> Zn;
   let Inst{31-24} = 0b01100101;
   let Inst{23-22} = sz;
   let Inst{21-19} = 0b001;
   let Inst{18-16} = opc;
   let Inst{15-10} = 0b001100;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> {
   def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>;
   def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>;
   def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>;
 
   def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>;
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
   let Inst{15-13} = 0b000;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_pred_log<bits<3> opc, string asm, string Ps,
                                 SDPatternOperator op,
                                 DestructiveInstTypeEnum flags> {
   let DestructiveInstType = flags in {
   def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>,
              SVEPseudo2Instr<Ps # _B, 1>;
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za),
   let Inst{13}    = opc;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Za;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>;
   def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>;
   def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>;
   def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>;
 
: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm),
   let Inst{13}    = opc;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>;
   def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>;
   def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>;
   def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>;
 
: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm),
   let Inst{15}    = 0b0;
   let Inst{14-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> {
   def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>;
   def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>;
   def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>;
   def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>;
 
: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop),
   let Inst{21}    = 0b1;
   let Inst{15-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm,
                                         SDPatternOperator op> {
   def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {
     bits<3> Zm;
     bits<3> iop;
     let Inst{22} = iop{2};
: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm,
   let Inst{20-16} = Zm;
   let Inst{15-11} = 0;
   let Inst{10}    = U;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> {
   def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>;
   def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>;
 
   def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32,  nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>;
   def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>;
: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop),
   let Inst{21}    = 0b1;
   let Inst{15-11} = 0;
   let Inst{10}    = U;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm,
                                         SDPatternOperator op> {
   def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> {
     bits<2> iop;
     bits<3> Zm;
     let Inst{20-19} = iop;
: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm,
   let Inst{15-12} = opc;
   let Inst{11-10} = rot;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_cintx_dot<string asm, SDPatternOperator op> {
   def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>;
   def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>;
 
   def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3),
                          (i32 complexrotateop:$imm))),
: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop,
   let Inst{15-12} = opc;
   let Inst{11-10} = rot;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> {
   def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> {
     bits<2> iop;
     bits<3> Zm;
     let Inst{20-19} = iop;
     let Inst{18-16} = Zm;
: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = sz;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b011;
   let Inst{12-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op,
                         SDPatternOperator op_pred = null_frag> {
   def _B : sve2_int_mul<0b00, opc, asm, ZPR8>;
   def _H : sve2_int_mul<0b01, opc, asm, ZPR16>;
   def _S : sve2_int_mul<0b10, opc, asm, ZPR32>;
   def _D : sve2_int_mul<0b11, opc, asm, ZPR64>;
: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop),
   bits<5> Zn;
   let Inst{31-24} = 0b01000100;
   let Inst{23-22} = sz;
   let Inst{21}    = 0b1;
   let Inst{15-14} = 0b11;
   let Inst{13-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm,
                                         SDPatternOperator op> {
   def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {
     bits<3> Zm;
     bits<3> iop;
     let Inst{22} = iop{2};
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
   let Inst{13}    = opc{0};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op,
                                string Ps = "",
                                DestructiveInstTypeEnum flags=DestructiveOther,
                                string revname="", bit isReverseInstr=0> {
   let DestructiveInstType = flags in {
   def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>,
: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn),
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty1.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> {
   def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>;
   def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>;
   def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>;
 
   def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>;
: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveUnaryPassthru;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm,
                                    SDPatternOperator op> {
   def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>,
            SVEPseudo2Instr<NAME # _S, 1>;
 
   def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm),
   let Inst{31-24} = 0b01000101;
   let Inst{23-22} = sz;
   let Inst{21}    = 0b0;
   let Inst{20-16} = Zm;
   let Inst{15}    = 0b0;
   let Inst{14-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_wide_int_arith_long<bits<5> opc, string asm,
                                     SDPatternOperator op> {
   def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>;
   def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>;
   def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>;
 
: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm),
   let Inst{31-24} = 0b01000101;
   let Inst{23-22} = sz;
   let Inst{21}    = 0b0;
   let Inst{20-16} = Zm;
   let Inst{15-14} = 0b10;
   let Inst{13-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> {
   def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>;
   def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>;
   def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>;
   def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>;
 
: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm),
   let Inst{15-11} = 0b10010;
   let Inst{10}    = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_bitwise_xor_interleaved<bit opc, string asm,
                                         SDPatternOperator op> {
   def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8,  ZPR8>;
   def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>;
   def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>;
   def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>;
: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm),
   let Inst{22}    = tsz8_64{2};
   let Inst{21}    = 0b0;
   let Inst{20-19} = tsz8_64{1-0};
   let Inst{18-16} = imm{2-0}; // imm3
   let Inst{15-12} = 0b1010;
   let Inst{11-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm,
                                         SDPatternOperator op> {
   def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm,
                                         ZPR16, ZPR8, vecshiftL8>;
   def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm,
                                         ZPR32, ZPR16, vecshiftL16> {
: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm),
   let Inst{20-19} = tsz8_64{1-0};
   let Inst{18-16} = imm{2-0}; // imm3
   let Inst{15-11} = 0b11110;
   let Inst{10}    = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_bin_shift_imm_left<bit opc, string asm,
                                        SDPatternOperator op> {
   def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;
   def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {
     let Inst{19} = imm{3};
   }
: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm),
   let Inst{15-12} = 0b1110;
   let Inst{11-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm,
                                               SDPatternOperator op,
                                               SDPatternOperator shift_op = null_frag> {
   def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;
   def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {
     let Inst{19} = imm{3};
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot),
   let Inst{15-11} = 0b11011;
   let Inst{10}    = rot;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> {
   def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>;
   def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>;
   def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>;
   def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>;
 
: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm),
   let Inst{15-14} = 0b11;
   let Inst{13-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> {
   def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>;
   def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>;
   def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>;
   def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>;
 
: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm),
   let Inst{21}    = 0b1;
   let Inst{20-19} = tsz8_64{1-0};
   let Inst{18-16} = imm{2-0}; // imm3
   let Inst{15-14} = 0b00;
   let Inst{13-11} = opc;
   let Inst{10}    = 0b0;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm,
                                                       SDPatternOperator op> {
   def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16,
                                                 tvecshiftR8>;
   def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32,
                                                 tvecshiftR16> {
: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm),
   let Inst{18-16} = imm{2-0}; // imm3
   let Inst{15-14} = 0b00;
   let Inst{13-11} = opc;
   let Inst{10}    = 0b1;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm,
                                                    SDPatternOperator op> {
   def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16,
                                              tvecshiftR8>;
   def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32,
                                              tvecshiftR16> {
: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm),
   let Inst{23-22} = sz;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b011;
   let Inst{12-11} = opc; // S, R
   let Inst{10}    = 0b0; // Top
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm,
                                               SDPatternOperator op> {
   def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>;
   def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>;
   def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>;
 
: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm),
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b011;
   let Inst{12-11} = opc; // S, R
   let Inst{10}    = 0b1; // Top
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm,
                                            SDPatternOperator op> {
   def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>;
   def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>;
   def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>;
 
: I<(outs zprty1:$Zd), (ins zprty2:$Zn),
   let Inst{22}    = tsz8_64{2};
   let Inst{21}    = 0b1;
   let Inst{20-19} = tsz8_64{1-0};
   let Inst{18-13} = 0b000010;
   let Inst{12-11} = opc;
   let Inst{10}    = 0b0;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm,
                                               SDPatternOperator op> {
   def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>;
   def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>;
   def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>;
 
: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn),
   let Inst{20-19} = tsz8_64{1-0};
   let Inst{18-13} = 0b000010;
   let Inst{12-11} = opc;
   let Inst{10}    = 0b1;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm,
                                            SDPatternOperator op> {
   def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>;
   def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>;
   def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>;
 
: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveUnaryPassthru;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm,
                                   SDPatternOperator op> {
   def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>,
            SVEPseudo2Instr<NAME # _B, 1>;
   def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>,
            SVEPseudo2Instr<NAME # _H, 1>;
: I<(outs zprty:$Zd), (ins immtype:$imm),
   bits<9> imm;
   let Inst{31-24} = 0b00100101;
   let Inst{23-22} = sz8_64;
   let Inst{21-14} = 0b11100011;
   let Inst{13}    = imm{8};   // sh
   let Inst{12-5}  = imm{7-0}; // imm8
   let Inst{4-0}   = Zd;
 
+  let hasSideEffects = 0;
   let isReMaterializable = 1;
 }
 
 multiclass sve_int_dup_imm<string asm> {
   def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>;
   def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>;
   def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>;
   def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>;
: I<(outs zprty:$Zd), (ins fpimmtype:$imm8),
   bits<8> imm8;
   let Inst{31-24} = 0b00100101;
   let Inst{23-22} = sz8_64;
   let Inst{21-14} = 0b11100111;
   let Inst{13}    = 0b0;
   let Inst{12-5}  = imm8;
   let Inst{4-0}   = Zd;
 
+  let hasSideEffects = 0;
   let isReMaterializable = 1;
 }
 
 multiclass sve_int_dup_fpimm<string asm> {
   def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>;
   def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>;
   def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>;
 
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),
   let Inst{15-14} = 0b11;
   let Inst{13}    = imm{8};   // sh
   let Inst{12-5}  = imm{7-0}; // imm8
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8,  addsub_imm8_opt_lsl_i8>;
   def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>;
   def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>;
   def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>;
 
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),
   let Inst{21-16} = opc;
   let Inst{15-13} = 0b110;
   let Inst{12-5} = imm;
   let Inst{4-0} = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8_32b>;
   def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8_32b>;
   def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8_32b>;
   def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8_32b>;
 
: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm),
   bits<5> Zn;
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = opc{1-0};
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b001100;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> {
   def NAME : sve_int_bin_cons_log<opc, asm>;
 
   def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>;
: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk),
   let Inst{15-11} = 0b00111;
   let Inst{10}    = opc{0};
   let Inst{9-5}   = Zk;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op,
                                        SDPatternOperator ir_op = null_frag> {
   def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>;
 
   def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",
                   (!cast<Instruction>(NAME) ZPR8:$Zdn,  ZPR8:$Zm,  ZPR8:$Zk),  1>;
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm),
   let Inst{18-16} = imm{2-0}; // imm3
   let Inst{15-10} = 0b001101;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> {
   def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>;
   def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> {
     let Inst{19} = imm{3};
   }
   def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> {
: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8),
   let Inst{19-16} = Pg;
   let Inst{15-13} = 0b110;
   let Inst{12-5}  = imm8;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_dup_fpimm_pred<string asm> {
   def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>;
   def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>;
   def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>;
 
   def : InstAlias<"fmov $Zd, $Pg/m, $imm8",
: I<(outs zprty:$Zd), iops,
   let Inst{15}    = 0b0;
   let Inst{14}    = m;
   let Inst{13}    = imm{8};   // sh
   let Inst{12-5}  = imm{7-0}; // imm8
   let Inst{4-0}   = Zd;
 
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_dup_imm_pred_merge_inst<
     bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm,
     ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> {
   let Constraints = "$Zd = $_Zd" in
   def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty,  "/m",
                                   (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>;
: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm),
   let Inst{13}    = opc{1};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
   let ElementSize = pprty.ElementSize;
+  let hasSideEffects = 0;
   let isPTestLike = 1;
 }
 
 multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt,
                          ValueType intvt, Instruction cmp> {
   def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)),
             (cmp $Op1, $Op2, $Op3)>;
   def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)),
: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5),
   let Inst{13}    = opc{1};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
   let ElementSize = pprty.ElementSize;
+  let hasSideEffects = 0;
   let isPTestLike = 1;
 }
 
 multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc,
                              ValueType predvt, ValueType intvt,
                              Operand immtype, Instruction cmp> {
   def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg),
                                     (intvt ZPR:$Zs1),
: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7),
   let Inst{13}    = opc{1};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
   let ElementSize = pprty.ElementSize;
+  let hasSideEffects = 0;
   let isPTestLike = 1;
 }
 
 multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc,
                            CondCode commuted_cc> {
   def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>;
   def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>;
   def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>;
: I<(outs), (ins rt:$Rn, rt:$Rm),
   let Inst{21}    = 0b1;
   let Inst{20-16} = Rm;
   let Inst{15-10} = 0b001000;
   let Inst{9-5}   = Rn;
   let Inst{4}     = opc;
   let Inst{3-0}   = 0b0000;
 
   let Defs = [NZCV];
+  let hasSideEffects = 0;
 }
 
 class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm,
                        RegisterClass gprty, PPRRegOp pprty>
 : I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm),
   asm, "\t$Pd, $Rn, $Rm",
   "", []>, Sched<[]> {
   bits<4> Pd;
   bits<5> Rm;
   bits<5> Rn;
   let Inst{31-24} = 0b00100101;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b000;
   let Inst{12-10} = opc{3-1};
   let Inst{9-5}   = Rn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
   let ElementSize = pprty.ElementSize;
+  let hasSideEffects = 0;
   let isWhile = 1;
 }
 
 multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>;
   def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>;
   def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>;
   def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>;
: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm),
   let Inst{20-16} = Rm;
   let Inst{15-10} = 0b001100;
   let Inst{9-5}   = Rn;
   let Inst{4}     = rw;
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
   let ElementSize = pprty.ElementSize;
+  let hasSideEffects = 0;
   let isWhile = 1;
 }
 
 multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> {
   def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>;
   def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>;
   def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>;
   def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>;
: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),
   let Inst{31-24} = 0b01100101;
   let Inst{23-22} = sz;
   let Inst{21-19} = 0b000;
   let Inst{18-16} = opc;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Vd;
+
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> {
   def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>;
   def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>;
   def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>;
 
   def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;
: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm),
   let Inst{21-19} = 0b011;
   let Inst{18-16} = opc;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Vdn;
 
   let Constraints = "$Vdn = $_Vdn";
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> {
   def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>;
   def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>;
   def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>;
 
   def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>;
: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),
   let Inst{20-16} = Zm;
   let Inst{15}    = opc{2};
   let Inst{14}    = 0b1;
   let Inst{13}    = opc{1};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
+
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> {
   def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>;
   def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>;
   def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>;
 
   def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn),
   let Inst{23-22} = sz;
   let Inst{21-18} = 0b0100;
   let Inst{17-16} = opc{2-1};
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
+
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_2op_p_pd<bits<3> opc, string asm,
                            CondCode cc1, CondCode cc2,
                            CondCode invcc1, CondCode invcc2> {
   def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>;
   def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>;
   def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>;
: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b),
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = imm5b;
   let Inst{15-10} = 0b010000;
   let Inst{9-5}   = imm5;
   let Inst{4-0}   = Zd;
 
+  let hasSideEffects = 0;
   let isReMaterializable = 1;
 }
 
 multiclass sve_int_index_ii<string asm> {
   def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>;
   def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>;
   def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>;
   def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>;
: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm),
   bits<5> imm5;
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Rm;
   let Inst{15-10} = 0b010010;
   let Inst{9-5}   = imm5;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_index_ir<string asm, SDPatternOperator mulop, SDPatternOperator muloneuseop> {
   def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>;
   def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>;
   def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>;
   def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>;
 
: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5),
   bits<5> imm5;
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = imm5;
   let Inst{15-10} = 0b010001;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_index_ri<string asm> {
   def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>;
   def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>;
   def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>;
   def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>;
 
: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm),
   bits<5> Rn;
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Rm;
   let Inst{15-10} = 0b010011;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_index_rr<string asm, SDPatternOperator mulop> {
   def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>;
   def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>;
   def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>;
   def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>;
 
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm),
   let Inst{12-10} = Pg;
   let Inst{9-8}   = tsz8_64{1-0};
   let Inst{7-5}   = imm{2-0}; // imm3
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveBinaryImm;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps,
                                            SDPatternOperator op = null_frag> {
   def _B : SVEPseudo2Instr<Ps # _B, 1>,
            sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;
   def _H : SVEPseudo2Instr<Ps # _H, 1>,
            sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm),
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps,
                                   SDPatternOperator op, string revname, bit isReverseInstr = 0> {
   let DestructiveInstType = DestructiveBinaryCommWithRev in {
   def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>,
            SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;
   def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>,
: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm),
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = sz8_64;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-12} = 0b1000;
   let Inst{11-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm, SDPatternOperator op> {
   def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>;
   def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>;
   def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>;
 
   def : SVE_2_Op_Pred_All_Active<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;
: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm),
   let Inst{23-22} = tsz8_64{3-2};
   let Inst{21}    = 0b1;
   let Inst{20-19} = tsz8_64{1-0};
   let Inst{18-16} = imm{2-0}; // imm3
   let Inst{15-12} = 0b1001;
   let Inst{11-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm,
                                            SDPatternOperator op> {
   def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;
   def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {
     let Inst{19} = imm{3};
   }
: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
   let Inst{22-21} = esz;
   let Inst{20}    = 0;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b111;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm,
                           RegisterOperand listty, ZPRRegOp zprty>
 {
   def NAME : sve_mem_cst_si<msz, esz, asm, listty>;
 
: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),
   let Inst{22-21} = nregs;
   let Inst{20}    = 1;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b111;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
                           string asm, Operand immtype> {
   def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>;
 
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
class sve_mem_128b_est_si<bits<2> nregs, RegisterOperand VecList,
   let Inst{23-22} = nregs;
   let Inst{21-20} = 0b00;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b000;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_128b_est_si<bits<2> nregs, RegisterOperand VecList,
                                string asm, Operand immtype> {
   def NAME : sve_mem_128b_est_si<nregs, VecList, asm, immtype>;
 
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   let Inst{24-23} = sz;
   let Inst{22-21} = nregs;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b011;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 
 // SVE store multiple structures (quadwords, scalar plus scalar)
 class sve_mem_128b_est_ss<bits<2> nregs, RegisterOperand VecList,
                           string asm, RegisterOperand gprty>
     : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
         asm, "\t$Zt, $Pg, [$Rn, $Rm]",
         "", []>, Sched<[]> {
   bits<5> Zt;
   bits<5> Rn;
   bits<3> Pg;
   bits<5> Rm;
   let Inst{31-24} = 0b11100100;
   let Inst{23-22} = nregs;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b000;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 
 class sve_mem_cst_ss_base<bits<4> dtype, string asm,
                           RegisterOperand listty, RegisterOperand gprty>
 : I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   asm, "\t$Zt, $Pg, [$Rn, $Rm]",
   "",
   []>, Sched<[]> {
   bits<3> Pg;
   bits<5> Rm;
   bits<5> Rn;
   bits<5> Zt;
   let Inst{31-25} = 0b1110010;
   let Inst{24-21} = dtype;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b010;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_cst_ss<bits<4> dtype, string asm,
                           RegisterOperand listty, ZPRRegOp zprty,
                           RegisterOperand gprty> {
   def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>;
 
: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
   let Inst{24-23} = msz;
   let Inst{22-20} = 0b001;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b111;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty,
                             ZPRRegOp zprty> {
   def NAME : sve_mem_cstnt_si<msz, asm, listty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   let Inst{24-23} = msz;
   let Inst{22-21} = 0b00;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b011;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty,
                             ZPRRegOp zprty, RegisterOperand gprty> {
   def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]",
: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm),
   let Inst{24-22} = opc;
   let Inst{21}    = 0b0;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm,
                              SDPatternOperator op,
                              ValueType vt> {
   def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>;
 
: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
   let Inst{20-16} = Zm;
   let Inst{15}    = 0b1;
   let Inst{14}    = xs;
   let Inst{13}    = 0;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm,
                                     SDPatternOperator sxtw_op,
                                     SDPatternOperator uxtw_op,
                                     RegisterOperand sxtw_opnd,
                                     RegisterOperand uxtw_opnd,
: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
   let Inst{22}    = 0b0;
   let Inst{21}    = scaled;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm,
                                     SDPatternOperator op,
                                     RegisterOperand zprext,
                                     ValueType vt> {
   def _SCALED : sve_mem_sst_sv2<msz, 1, asm, zprext>;
: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5),
   let Inst{22}    = 0b1;
   let Inst{21}    = opc{0};
   let Inst{20-16} = imm5;
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm,
                                    Operand imm_ty,
                                    SDPatternOperator op,
                                    ValueType vt> {
   def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>;
: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9),
   bits<9> imm9;
   let Inst{31-22} = 0b1110010110;
   let Inst{21-16} = imm9{8-3};
   let Inst{15-13} = 0b010;
   let Inst{12-10} = imm9{2-0};
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_z_spill<string asm> {
   def NAME : sve_mem_z_spill<asm>;
 
   def : InstAlias<asm # "\t$Zt, [$Rn]",
                   (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>;
: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9),
   let Inst{31-22} = 0b1110010110;
   let Inst{21-16} = imm9{8-3};
   let Inst{15-13} = 0b000;
   let Inst{12-10} = imm9{2-0};
   let Inst{9-5}   = Rn;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 multiclass sve_mem_p_spill<string asm> {
   def NAME : sve_mem_p_spill<asm>;
 
   def : InstAlias<asm # "\t$Pt, [$Rn]",
                   (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>;
: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm),
   let Inst{21-20} = 0b10;
   let Inst{19-16} = Pm;
   let Inst{15-13} = 0b010;
   let Inst{12-10} = opc;
   let Inst{9}     = 0b0;
   let Inst{8-5}   = Pn;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm,
                                     SDPatternOperator op> {
   def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>;
   def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>;
   def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>;
   def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>;
: I<(outs PPR16:$Pd), (ins PPR8:$Pn),
   bits<4> Pd;
   bits<4> Pn;
   let Inst{31-17} = 0b000001010011000;
   let Inst{16}    = opc;
   let Inst{15-9}  = 0b0100000;
   let Inst{8-5}   = Pn;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> {
   def NAME : sve_int_perm_punpk<opc, asm>;
 
   def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>;
   def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1,  !cast<Instruction>(NAME)>;
   def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1,  !cast<Instruction>(NAME)>;
: I<(outs PPR8:$Pd), (ins PPRAny:$Pg),
   let Inst{22}    = s;
   let Inst{21-9}  = 0b0110001111000;
   let Inst{8-5}   = Pg;
   let Inst{4}     = 0;
   let Inst{3-0}   = Pd;
 
   let Defs = !if(s, [NZCV], []);
   let Uses = [FFR];
+  let hasSideEffects = 1;
 }
 
 multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> {
   def _REAL : sve_int_rdffr_pred<s, asm>;
 
   // We need a layer of indirection because early machine code passes balk at
   // physical register (i.e. FFR) uses that have no previous definition.
   let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
   def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>,
            PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>;
   }
 }
 
 class sve_int_rdffr_unpred<string asm> : I<
   (outs PPR8:$Pd), (ins),
   asm, "\t$Pd",
   "",
   []>, Sched<[]> {
   bits<4> Pd;
   let Inst{31-4} = 0b0010010100011001111100000000;
   let Inst{3-0}   = Pd;
 
   let Uses = [FFR];
+  let hasSideEffects = 1;
 }
 
 multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> {
   def _REAL : sve_int_rdffr_unpred<asm>;
 
   // We need a layer of indirection because early machine code passes balk at
   // physical register (i.e. FFR) uses that have no previous definition.
   let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
   def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>,
            PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>;
   }
 }
 
 class sve_int_wrffr<string asm, SDPatternOperator op>
 : I<(outs), (ins PPR8:$Pn),
   asm, "\t$Pn",
   "",
   [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> {
   bits<4> Pn;
   let Inst{31-9} = 0b00100101001010001001000;
   let Inst{8-5}  = Pn;
   let Inst{4-0}  = 0b00000;
 
-  let hasSideEffects = 1;
   let Defs = [FFR];
+  let hasSideEffects = 1;
 }
 
 class sve_int_setffr<string asm, SDPatternOperator op>
 : I<(outs), (ins),
   asm, "",
   "",
   [(op)]>, Sched<[]> {
   let Inst{31-0} = 0b00100101001011001001000000000000;
 
-  let hasSideEffects = 1;
   let Defs = [FFR];
+  let hasSideEffects = 1;
 }
 
 //===----------------------------------------------------------------------===//
 // SVE Permute Vector - Predicated Group
 //===----------------------------------------------------------------------===//
 
 class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm,
                             ZPRRegOp zprty, RegisterClass rt>
: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm),
   let Inst{21-17} = 0b11000;
   let Inst{16}    = ab;
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Rdn;
 
   let Constraints = "$Rdn = $_Rdn";
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> {
   def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>;
   def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>;
   def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>;
   def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>;
 
: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm),
   let Inst{21-17} = 0b10101;
   let Inst{16}    = ab;
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Vdn;
 
   let Constraints = "$Vdn = $_Vdn";
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> {
   def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>;
   def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>;
   def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>;
   def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>;
 
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> {
   def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>;
   def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>;
   def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>;
   def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>;
 
: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn),
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21-17} = 0b10000;
   let Inst{16}    = ab;
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Rd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> {
   def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>;
   def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>;
   def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>;
   def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>;
 
: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21-17} = 0b10001;
   let Inst{16}    = ab;
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Vd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> {
   def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>;
   def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>;
   def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>;
   def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>;
 
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
   let Inst{21-13} = 0b101100100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeNone;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_splice<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_splice<0b00, asm, ZPR8>;
   def _H : sve_int_perm_splice<0b01, asm, ZPR16>;
   def _S : sve_int_perm_splice<0b10, asm, ZPR32>;
   def _D : sve_int_perm_splice<0b11, asm, ZPR64>;
 
: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn),
   bits<5> Zn;
   bits<5> Zd;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = sz8_64;
   let Inst{21-13} = 0b101101100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_int_perm_splice_cons<string asm> {
   def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8,  ZZ_b>;
   def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>;
   def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>;
   def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>;
 }
: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>;
   def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>;
   def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>;
   def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>;
 
: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn),
   let Inst{21-13} = 0b101000101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>;
   def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>;
   def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>;
   def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>;
 
: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn),
   let Inst{21-13} = 0b100000100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Vn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> {
   def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>;
   def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>;
   def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>;
   def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>;
 
: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),
   bits<5> Zd;
   bits<5> Zn;
   let Inst{31-23} = 0b000001011;
   let Inst{22}    = sz;
   let Inst{21-13} = 0b100001100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_compact<string asm, SDPatternOperator op> {
   def _S : sve_int_perm_compact<0b0, asm, ZPR32>;
   def _D : sve_int_perm_compact<0b1, asm, ZPR64>;
 
   def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
   def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
   let Inst{24-21} = dtype;
   let Inst{20}    = nf;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
-  let mayLoad = 1;
-  let Uses = !if(nf, [FFR], []);
   let Defs = !if(nf, [FFR], []);
+  let Uses = !if(nf, [FFR], []);
+  let hasSideEffects = nf;
+  let mayLoad = 1;
 }
 
 multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm,
                                RegisterOperand listty, ZPRRegOp zprty> {
   def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
                   (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
   let Inst{24-23} = msz;
   let Inst{22-20} = 0b000;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b111;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty,
                             ZPRRegOp zprty> {
   def NAME : sve_mem_cldnt_si_base<msz, asm, listty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   let Inst{24-23} = msz;
   let Inst{22-21} = 0b00;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b110;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty,
                             ZPRRegOp zprty, RegisterOperand gprty> {
   def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4),
   let Inst{24-23} = sz;
   let Inst{22-20} = 0;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty,
                            ZPRRegOp zprty> {
   def NAME : sve_mem_ldqr_si<sz, asm, listty>;
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
                   (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   let Inst{24-23} = sz;
   let Inst{22-21} = 0;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty,
                            ZPRRegOp zprty, RegisterOperand gprty> {
   def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6),
   let Inst{22}    = 1;
   let Inst{21-16} = imm6;
   let Inst{15}    = 0b1;
   let Inst{14-13} = dtypel;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm,
                           RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> {
   def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
: I<(outs VecList:$Zt), iops,
   let Inst{24-21} = dtype;
   let Inst{20-16} = Rm;
   let Inst{15-14} = 0b01;
   let Inst{13}    = ff;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
-  let mayLoad = 1;
-  let Uses = !if(ff, [FFR], []);
   let Defs = !if(ff, [FFR], []);
+  let Uses = !if(ff, [FFR], []);
+  let hasSideEffects = ff;
+  let mayLoad = 1;
 }
 
 multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty,
                           ZPRRegOp zprty, RegisterOperand gprty> {
   def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
                                asm, listty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),
   let Inst{22-21} = nregs{1-0};
   let Inst{20}    = nregs{2};
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b111;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_eld_si<bits<2> sz, bits<3> nregs, RegisterOperand VecList,
                           string asm, Operand immtype> {
   def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   let Inst{20-16} = Rm;
   let Inst{15}    = 0b1;
   let Inst{14}    = nregs{2};
   let Inst{13}    = 0b0;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 //===----------------------------------------------------------------------===//
 // SVE Memory - 32-bit Gather and Unsized Contiguous Group
 //===----------------------------------------------------------------------===//
 
 // bit xs      is '1' if offsets are signed
: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
   let Inst{21}    = scaled;
   let Inst{20-16} = Zm;
   let Inst{15}    = 0b0;
   let Inst{14-13} = opc{1-0};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
-  let mayLoad = 1;
+
   let Defs = !if(!eq(opc{0}, 1), [FFR], []);
   let Uses = !if(!eq(opc{0}, 1), [FFR], []);
+  let hasSideEffects = opc{0};
+  let mayLoad = 1;
 }
 
 multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm,
                                         SDPatternOperator sxtw_op,
                                         SDPatternOperator uxtw_op,
                                         RegisterOperand sxtw_opnd,
                                         RegisterOperand uxtw_opnd,
                                         ValueType vt> {
: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5),
   let Inst{22-21} = 0b01;
   let Inst{20-16} = imm5;
   let Inst{15}    = 0b1;
   let Inst{14-13} = opc{1-0};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zt;
 
-  let mayLoad = 1;
+
   let Defs = !if(!eq(opc{0}, 1), [FFR], []);
   let Uses = !if(!eq(opc{0}, 1), [FFR], []);
+  let hasSideEffects = opc{0};
+  let mayLoad = 1;
 }
 
 multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty,
                                       SDPatternOperator op, ValueType vt> {
   def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
                   (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>;
: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5),
   let Inst{24-23} = msz;
   let Inst{22-21} = 0b00;
   let Inst{20-16} = imm5;
   let Inst{15-13} = 0b111;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = prfop;
+
+  let hasSideEffects = 1;
 }
 
 multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> {
   def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>;
 
   def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]",
                   (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>;
 
: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9),
   bits<9> imm9;
   let Inst{31-22} = 0b1000010110;
   let Inst{21-16} = imm9{8-3};
   let Inst{15-13} = 0b010;
   let Inst{12-10} = imm9{2-0};
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_z_fill<string asm> {
   def NAME : sve_mem_z_fill<asm>;
 
   def : InstAlias<asm # "\t$Zt, [$Rn]",
                   (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>;
: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9),
   let Inst{31-22} = 0b1000010110;
   let Inst{21-16} = imm9{8-3};
   let Inst{15-13} = 0b000;
   let Inst{12-10} = imm9{2-0};
   let Inst{9-5}   = Rn;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_p_fill<string asm> {
   def NAME : sve_mem_p_fill<asm>;
 
   def : InstAlias<asm # "\t$Pt, [$Rn]",
                   (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>;
: I<(outs VecList:$Zt), iops,
   let Inst{22-21} = 0b00;
   let Inst{20-16} = Rm;
   let Inst{15}    = 0b1;
   let Inst{14-13} = opc{1-0};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm,
                                   SDPatternOperator op,
                                   ValueType vt> {
   def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm),
                                      asm, Z_s>;
: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
   let Inst{21}    = scaled;
   let Inst{20-16} = Zm;
   let Inst{15}    = lsl;
   let Inst{14-13} = opc{1-0};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
-  let mayLoad = 1;
+
   let Defs = !if(!eq(opc{0}, 1), [FFR], []);
   let Uses = !if(!eq(opc{0}, 1), [FFR], []);
+  let hasSideEffects = opc{0};
+  let mayLoad = 1;
 }
 
 multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm,
                                         SDPatternOperator sxtw_op,
                                         SDPatternOperator uxtw_op,
                                         RegisterOperand sxtw_opnd,
                                         RegisterOperand uxtw_opnd,
                                         ValueType vt> {
: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5),
   let Inst{22-21} = 0b01;
   let Inst{20-16} = imm5;
   let Inst{15}    = 0b1;
   let Inst{14-13} = opc{1-0};
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zt;
 
-  let mayLoad = 1;
   let Defs = !if(!eq(opc{0}, 1), [FFR], []);
   let Uses = !if(!eq(opc{0}, 1), [FFR], []);
+  let hasSideEffects = opc{0};
+  let mayLoad = 1;
 }
 
 multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty,
                                       SDPatternOperator op, ValueType vt> {
   def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>;
 
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
                   (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>;
: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm),
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = opc;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-12} = 0b1010;
   let Inst{11-10} = msz;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> {
   def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>;
   def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>;
   def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>;
   def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>;
 }
: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
   bits<5> Zn;
   let Inst{31-24} = 0b00000100;
   let Inst{23-22} = sz;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b101100;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> {
   def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>;
   def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>;
   def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>;
 
   def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),
   let Inst{23-22} = sz8_32;
   let Inst{21}    = 0b0;
   let Inst{20-19} = fmt;
   let Inst{18-16} = opc;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Vd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm,
                                   SDPatternOperator op> {
   def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>;
   def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>;
   def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>;
 
: I<(outs zprty:$Zd), iops,
   let Inst{21-19} = 0b010;
   let Inst{18-16} = opc;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> {
 let Constraints = "$Zd = $_Zd" in {
   def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m",
                                 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>;
   def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m",
                                 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>;
: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm),
   let Inst{15-14} = 0b11;
   let Inst{13-10} = Pg;
   let Inst{9}     = 0b0;
   let Inst{8-5}   = Pn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Defs = !if(!eq (opc{1}, 1), [NZCV], []);
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> {
   def NAME : sve_int_brkp<opc, asm>;
 
   def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
 }
 
: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm),
   let Inst{9}     = 0b0;
   let Inst{8-5}   = Pn;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pdm;
 
   let Constraints = "$Pdm = $_Pdm";
   let Defs = !if(S, [NZCV], []);
   let ElementSize = ElementSizeB;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> {
   def NAME : sve_int_brkn<opc, asm>;
 
   def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
 }
 
: I<(outs PPR8:$Pd), iops,
   let Inst{13-10} = Pg;
   let Inst{9}     = 0b0;
   let Inst{8-5}   = Pn;
   let Inst{4}     = opc{0};
   let Inst{3-0}   = Pd;
 
   let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", "");
   let Defs = !if(!eq (opc{1}, 1), [NZCV], []);
-
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> {
   def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>;
 
   def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
 }
 
: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4}     = opc;
   let Inst{3-0}   = Pd;
 
   let Defs = [NZCV];
   let ElementSize = pprty.ElementSize;
+  let hasSideEffects = 0;
   let isPTestLike = 1;
 }
 
 multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> {
   def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>;
   def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>;
 
   def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm),
   bits<5> Zd;
   bits<5> Zn;
   bits<5> Zm;
   let Inst{31-21} = 0b01000101001;
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b101000;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 //===----------------------------------------------------------------------===//
 // SVE2 Histogram Computation - Vector Group
 //===----------------------------------------------------------------------===//
 
 class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty>
 : I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),
   asm, "\t$Zd, $Pg/z, $Zn, $Zm",
   "",
   []>, Sched<[]> {
   bits<5> Zd;
   bits<5> Zn;
   bits<3> Pg;
   bits<5> Zm;
   let Inst{31-23} = 0b010001011;
   let Inst{22}    = sz;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b110;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> {
   def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>;
   def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>;
 
   def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
   def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
   bits<5> Zn;
   bits<5> Zm;
   let Inst{31-21} = 0b01000101001;
   let Inst{20-16} = Zm;
   let Inst{15-11} = 0b11110;
   let Inst{10}    = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty,
                                    SDPatternOperator op, ValueType vt> {
   def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>;
   def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>;
 }
 
 class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty>
 : I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm),
   asm, "\t$Zdn, $_Zdn, $Zm",
   "",
   []>, Sched<[]> {
   bits<5> Zdn;
   bits<5> Zm;
   let Inst{31-17} = 0b010001010010001;
   let Inst{16}    = opc{1};
   let Inst{15-11} = 0b11100;
   let Inst{10}    = opc{0};
   let Inst{9-5}   = Zm;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty,
                                   SDPatternOperator op, ValueType vt> {
   def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>;
   def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>;
 }
 
 class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty>
 : I<(outs zprty:$Zdn), (ins zprty:$_Zdn),
   asm, "\t$Zdn, $_Zdn",
   "",
   []>, Sched<[]> {
   bits<5> Zdn;
   let Inst{31-11} = 0b010001010010000011100;
   let Inst{10}    = opc;
   let Inst{9-5}   = 0b00000;
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
+  let hasSideEffects = 0;
 }
 
 multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> {
   def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>;
   def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>;
 }
 
 //===----------------------------------------------------------------------===//
: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b100000;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_float_dot<bit bf, string asm, ValueType InVT, SDPatternOperator op> {
   def NAME : sve_float_dot<bf, asm>;
   def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, InVT, InVT, !cast<Instruction>(NAME)>;
 }
 
 class sve_float_dot_indexed<bit bf, string asm>
: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS32b:$iop),
   let Inst{20-19} = iop;
   let Inst{18-16} = Zm;
   let Inst{15-10} = 0b010000;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_float_dot_indexed<bit bf, string asm, ValueType InVT, SDPatternOperator op> {
   def NAME : sve_float_dot_indexed<bf, asm>;
   def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, InVT, InVT, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;
 }
 
 class sve_bfloat_matmul<string asm>
 : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
   asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
   bits<5> Zm;
   bits<5> Zda;
   bits<5> Zn;
   let Inst{31-21} = 0b01100100011;
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b111001;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ElementSizeH;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> {
   def NAME : sve_bfloat_matmul<asm>;
   def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>;
 }
 
 class sve_bfloat_convert<bit N, string asm>
 : I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn),
   asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> {
   bits<5> Zd;
   bits<3> Pg;
   bits<5> Zn;
   let Inst{31-25} = 0b0110010;
   let Inst{24}    = N;
   let Inst{23-13} = 0b10001010101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveOther;
-  let hasSideEffects = 1;
   let ElementSize = ElementSizeS;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> {
   def NAME : sve_bfloat_convert<N, asm>;
   def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>;
 }
 
 //===----------------------------------------------------------------------===//
: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm,
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b100110;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ZPR32.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> {
   def NAME : sve_int_matmul<uns, asm>;
 
   def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;
 }
 
: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm,
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b011110;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ZPR32.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> {
   def NAME : sve_int_dot_mixed<asm>;
 
   def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;
 }
 
: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx),
   let Inst{15-11} = 0b00011;
   let Inst{10}    = U;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ZPR32.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> {
   def NAME : sve_int_dot_mixed_indexed<U, asm>;
 
   def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;
 }
 
: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm),
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b111001;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zprty.ElementSize;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> {
   def NAME : sve_fp_matrix_mla<sz, asm, zprty>;
 
   def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>;
 }
 
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4),
   let Inst{24-23} = sz;
   let Inst{22-20} = 0b010;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty,
                            ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> {
   def NAME : sve_mem_ldor_si<sz, asm, listty>;
   def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
                   (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
   let Inst{24-23} = sz;
   let Inst{22-21} = 0b01;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty,
                            ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty,
                            ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> {
   def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>;
 
: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm),
   bits<5> Zn;
   let Inst{31-21} = 0b00000101101;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b000;
   let Inst{12-11} = opc;
   let Inst{10}    = P;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> {
   def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>;
 
   def : SVE_2_Op_Pat<nxv16i8,  op, nxv16i8,  nxv16i8,  !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv8i16,  op, nxv8i16,  nxv8i16,  !cast<Instruction>(NAME)>;
   def : SVE_2_Op_Pat<nxv8f16,  op, nxv8f16,  nxv8f16,  !cast<Instruction>(NAME)>;
class sve2p1_fclamp<string asm, bits<2> sz, ZPRRegOp zpr_ty>
   let Inst{20-16} = Zm;
   let Inst{15-10} = 0b001001;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = zpr_ty.ElementSize;
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_fclamp<string asm, SDPatternOperator op> {
   def _H : sve2p1_fclamp<asm, 0b01, ZPR16>;
   def _S : sve2p1_fclamp<asm, 0b10, ZPR32>;
   def _D : sve2p1_fclamp<asm, 0b11, ZPR64>;
 
   def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
class sve2p1_two_way_dot_vv<string mnemonic, bit u>
   let Inst{20-16} = Zm;
   let Inst{15-11} = 0b11001;
   let Inst{10}    = u;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
 }
 
 
 // SVE two-way dot product (indexed)
 class sve2p1_two_way_dot_vvi<string mnemonic, bit u>
     : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS32b:$i2),
         mnemonic, "\t$Zda, $Zn, $Zm$i2",
         "", []>, Sched<[]> {
   bits<5> Zda;
   bits<5> Zn;
   bits<3> Zm;
   bits<2> i2;
   let Inst{31-21} = 0b01000100100;
   let Inst{20-19} = i2;
   let Inst{18-16} = Zm;
   let Inst{15-11} = 0b11001;
   let Inst{10}    = u;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
 }
 
 
 class sve2p1_ptrue_pn<string mnemonic, bits<2> sz, PNRP8to15RegOp pnrty>
     : I<(outs pnrty:$PNd), (ins ), mnemonic, "\t$PNd",
         "", []>, Sched<[]> {
   bits<3> PNd;
   let Inst{31-24}  = 0b00100101;
   let Inst{23-22} = sz;
   let Inst{21-3}  = 0b1000000111100000010;
   let Inst{2-0}   = PNd;
+
+  let hasSideEffects = 0;
 }
 
 
 multiclass sve2p1_ptrue_pn<string mnemonic> {
  def _B : sve2p1_ptrue_pn<mnemonic, 0b00, PNR8_p8to15>;
  def _H : sve2p1_ptrue_pn<mnemonic, 0b01, PNR16_p8to15>;
  def _S : sve2p1_ptrue_pn<mnemonic, 0b10, PNR32_p8to15>;
  def _D : sve2p1_ptrue_pn<mnemonic, 0b11, PNR64_p8to15>;
class sve2p1_pred_as_ctr_to_mask_base<string mnemonic, bits<2> sz, bits<3> opc,
   bits<2> imm2;
   let Inst{31-24} = 0b00100101;
   let Inst{23-22} = sz;
   let Inst{21-11} = 0b10000001110;
   let Inst{10-8}  = opc;
   let Inst{7-5}   = PNn;
   let Inst{4}     = 0b1;
   let Inst{3-0}   = Pd;
+
+  let hasSideEffects = 0;
 }
 
 class sve2p1_pred_as_ctr_to_mask<string mnemonic, bits<2> sz, PPRRegOp pprty>
     : sve2p1_pred_as_ctr_to_mask_base<mnemonic, sz, {0, ?, ?}, pprty, VectorIndexS> {
   bits<2> index;
   let Inst{9-8} = index;
 }
 
class sve2p1_multi_vec_extract_narrow<string mnemonic, bits<2> opc, bits<3> tsz>
   let Inst{21}    = 0b1;
   let Inst{20-19} = tsz{1-0};
   let Inst{18-13} = 0b001010;
   let Inst{12-11} = opc;
   let Inst{10}    = 0b0;
   let Inst{9-6}   = Zn;
   let Inst{5}     = 0b0;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_multi_vec_extract_narrow<string mnemonic, bits<2> opc, SDPatternOperator intrinsic> {
   def NAME : sve2p1_multi_vec_extract_narrow<mnemonic, opc, 0b010>;
   def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, nxv8i16, nxv4i32>;
 }
 
 // SVE2 multi-vec shift narrow
class sve2p1_multi_vec_shift_narrow<string mnemonic, bits<3> opc, bits<2> tsz>
   let Inst{20}    = tsz{0};
   let Inst{19-16} = imm4;
   let Inst{15-14} = 0b00;
   let Inst{13-11} = opc;
   let Inst{10}    = 0b0;
   let Inst{9-6}   = Zn;
   let Inst{5}     = 0b0;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_multi_vec_shift_narrow<string mnemonic, bits<3> opc> {
   def : sve2p1_multi_vec_shift_narrow<mnemonic, opc, 0b01>;
 }
 
 
 // SME2 multi-vec contiguous load (scalar plus scalar, two registers)
class sve2p1_mem_cld_ss_2z<string mnemonic, bits<2> msz, bit n,
   let Inst{20-16} = Rm;
   let Inst{15}    = 0b0;
   let Inst{14-13} = msz;
   let Inst{12-10} = PNg;
   let Inst{9-5} = Rn;
   let Inst{4-1} = Zt;
   let Inst{0}   = n;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 // SME2 multi-vec contiguous load (scalar plus immediate, two registers)
 class sve2p1_mem_cld_si_2z<string mnemonic, bits<2> msz, bit n,
                          RegisterOperand vector_ty>
     : I<(outs vector_ty:$Zt),
         (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s2:$imm4),
         mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]",
         "", []>, Sched<[]> {
   bits<4> Zt;
   bits<5> Rn;
   bits<3> PNg;
   bits<4> imm4;
   let Inst{31-20} = 0b101000000100;
   let Inst{19-16} = imm4;
   let Inst{15}    = 0b0;
   let Inst{14-13} = msz;
   let Inst{12-10} = PNg;
   let Inst{9-5}   = Rn;
   let Inst{4-1}   = Zt;
   let Inst{0}     = n;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve2p1_mem_cld_si_2z<string mnemonic, bits<2> msz, bit n,
                               RegisterOperand vector_ty> {
   def NAME : sve2p1_mem_cld_si_2z<mnemonic, msz, n, vector_ty>;
 
   def : InstAlias<mnemonic # " $Zt, $PNg/z, [$Rn]",
class sve2p1_mem_cld_ss_4z<string mnemonic, bits<2> msz, bit n,
   let Inst{15}    = 0b1;
   let Inst{14-13} = msz;
   let Inst{12-10} = PNg;
   let Inst{9-5} = Rn;
   let Inst{4-2} = Zt;
   let Inst{1}   = 0b0;
   let Inst{0}   = n;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 // SME2 multi-vec contiguous load (scalar plus immediate, four registers)
 class sve2p1_mem_cld_si_4z<string mnemonic, bits<2> msz, bit n,
                          RegisterOperand vector_ty>
     : I<(outs vector_ty:$Zt),
         (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s4:$imm4),
         mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]",
         "", []>, Sched<[]> {
   bits<3> Zt;
   bits<5> Rn;
   bits<3> PNg;
   bits<4> imm4;
   let Inst{31-20} = 0b101000000100;
   let Inst{19-16} = imm4;
   let Inst{15}    = 0b1;
   let Inst{14-13} = msz;
   let Inst{12-10} = PNg;
   let Inst{9-5}   = Rn;
   let Inst{4-2}   = Zt;
   let Inst{1}     = 0b0;
   let Inst{0}     = n;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve2p1_mem_cld_si_4z<string mnemonic, bits<2> msz, bit n,
                               RegisterOperand vector_ty> {
   def NAME : sve2p1_mem_cld_si_4z<mnemonic, msz, n, vector_ty>;
 
   def : InstAlias<mnemonic # " $Zt, $PNg/z, [$Rn]",
class sve2p1_mem_cst_ss_2z<string mnemonic, bits<2> msz, bit n,
   let Inst{20-16} = Rm;
   let Inst{15}    = 0b0;
   let Inst{14-13} = msz;
   let Inst{12-10} = PNg;
   let Inst{9-5} = Rn;
   let Inst{4-1} = Zt;
   let Inst{0}   = n;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 
 // SME2 multi-vec contiguous store (scalar plus immediate, two registers)
 class sve2p1_mem_cst_si_2z<string mnemonic, bits<2> msz, bit n,
                            RegisterOperand vector_ty>
     : I<(outs ),
         (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s2:$imm4),
         mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]",
         "", []>, Sched<[]> {
   bits<4> Zt;
   bits<5> Rn;
   bits<3> PNg;
   bits<4> imm4;
   let Inst{31-20} = 0b101000000110;
   let Inst{19-16} = imm4;
   let Inst{15}    = 0b0;
   let Inst{14-13} = msz;
   let Inst{12-10} = PNg;
   let Inst{9-5}   = Rn;
   let Inst{4-1}   = Zt;
   let Inst{0}     = n;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 
 multiclass sve2p1_mem_cst_si_2z<string mnemonic, bits<2> msz, bit n,
                               RegisterOperand vector_ty> {
   def NAME : sve2p1_mem_cst_si_2z<mnemonic, msz, n, vector_ty>;
 
class sve2p1_mem_cst_si_4z<string mnemonic, bits<2> msz, bit n,
   let Inst{15}    = 0b1;
   let Inst{14-13} = msz;
   let Inst{12-10} = PNg;
   let Inst{9-5}   = Rn;
   let Inst{4-2}   = Zt;
   let Inst{1}     = 0b0;
   let Inst{0}     = n;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 
 multiclass sve2p1_mem_cst_si_4z<string mnemonic, bits<2> msz, bit n,
                                 RegisterOperand vector_ty> {
   def NAME : sve2p1_mem_cst_si_4z<mnemonic, msz, n, vector_ty>;
 
class sve2p1_pcount_pn<string mnemonic, bits<3> opc, bits<2> sz, PNRRegOp pnrty>
   let Inst{23-22} = sz;
   let Inst{21-19} = 0b100;
   let Inst{18-16} = opc;
   let Inst{15-11} = 0b10000;
   let Inst{10}    = vl;
   let Inst{9}     = 0b1;
   let Inst{8-5}   = PNn;
   let Inst{4-0}   = Rd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_pcount_pn<string mnemonic, bits<3> opc> {
   def _B : sve2p1_pcount_pn<mnemonic, opc, 0b00, PNR8>;
   def _H : sve2p1_pcount_pn<mnemonic, opc, 0b01, PNR16>;
   def _S : sve2p1_pcount_pn<mnemonic, opc, 0b10, PNR32>;
   def _D : sve2p1_pcount_pn<mnemonic, opc, 0b11, PNR64>;
 }
class sve2p1_int_while_rr_pn<string mnemonic, bits<2> sz, bits<3> opc,
   let Inst{12}    = 0b0;
   let Inst{11-10} = opc{2-1};
   let Inst{9-5}   = Rn;
   let Inst{4}     = 0b1;
   let Inst{3}     = opc{0};
   let Inst{2-0}   = PNd;
 
   let Defs = [NZCV];
+  let hasSideEffects = 0;
 }
 
 
 multiclass sve2p1_int_while_rr_pn<string mnemonic, bits<3> opc> {
  def _B : sve2p1_int_while_rr_pn<mnemonic, 0b00, opc, PNR8_p8to15>;
  def _H : sve2p1_int_while_rr_pn<mnemonic, 0b01, opc, PNR16_p8to15>;
  def _S : sve2p1_int_while_rr_pn<mnemonic, 0b10, opc, PNR32_p8to15>;
  def _D : sve2p1_int_while_rr_pn<mnemonic, 0b11, opc, PNR64_p8to15>;
class sve2p1_int_while_rr_pair<string mnemonic, bits<2> sz, bits<3> opc,
   let Inst{15-12} = 0b0101;
   let Inst{11-10} = opc{2-1};
   let Inst{9-5}   = Rn;
   let Inst{4}     = 0b1;
   let Inst{3-1}   = Pd;
   let Inst{0}     = opc{0};
 
   let Defs = [NZCV];
+  let hasSideEffects = 0;
 }
 
 
 multiclass sve2p1_int_while_rr_pair<string mnemonic, bits<3> opc> {
  def _B : sve2p1_int_while_rr_pair<mnemonic, 0b00, opc, PP_b_mul_r>;
  def _H : sve2p1_int_while_rr_pair<mnemonic, 0b01, opc, PP_h_mul_r>;
  def _S : sve2p1_int_while_rr_pair<mnemonic, 0b10, opc, PP_s_mul_r>;
  def _D : sve2p1_int_while_rr_pair<mnemonic, 0b11, opc, PP_d_mul_r>;
class sve_mem_128b_gld_64_unscaled<string mnemonic>
   bits<5> Rm;
   let Inst{31-21} = 0b11000100000;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 
 multiclass sve_mem_128b_gld_64_unscaled<string mnemonic> {
   def NAME : sve_mem_128b_gld_64_unscaled<mnemonic>;
 
   def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Zn]",
class sve_mem_sst_128b_64_unscaled<string mnemonic>
   bits<5> Rm;
   let Inst{31-21} = 0b11100100001;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayStore = 1;
 }
 
 
 multiclass sve_mem_sst_128b_64_unscaled<string mnemonic> {
   def NAME : sve_mem_sst_128b_64_unscaled<mnemonic>;
 
   def : InstAlias<mnemonic # " $Zt, $Pg, [$Zn]",
class sve_mem_128b_cld_si<bits<2> dtype, string mnemonic>
   let Inst{24-23} = dtype;
   let Inst{22-20} = 0b001;
   let Inst{19-16} = imm4;
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_128b_cld_si<bits<2> dtype, string mnemonic> {
   def NAME : sve_mem_128b_cld_si<dtype, mnemonic>;
 
   def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn]",
                   (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
class sve_mem_128b_cld_ss<bits<2> dtype, string mnemonic, RegisterOperand gprsh_ty>
   let Inst{24-23} = dtype;
   let Inst{22-21} = 0b00;
   let Inst{20-16} = Rm;
   let Inst{15-13} = 0b100;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Rn;
   let Inst{4-0}   = Zt;
 
+  let hasSideEffects = 0;
   let mayLoad = 1;
 }
 
 multiclass sve_mem_128b_cld_ss<bits<2> dtype, string mnemonic, RegisterOperand gprsh_ty> {
   def NAME : sve_mem_128b_cld_ss<dtype, mnemonic, gprsh_ty>;
 
   def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn, $Rm]",
                  (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprsh_ty:$Rm), 0>;
class sve2p1_fp_reduction_q<bits<2> sz, bits<3> opc, string mnemonic,
   let Inst{31-24} = 0b01100100;
   let Inst{23-22} = sz;
   let Inst{21-19} = 0b010;
   let Inst{18-16} = opc;
   let Inst{15-13} = 0b101;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Vd;
+
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 multiclass sve2p1_fp_reduction_q<bits<3> opc, string mnemonic> {
   def _H : sve2p1_fp_reduction_q<0b01, opc, mnemonic, ZPR16, "8h">;
   def _S : sve2p1_fp_reduction_q<0b10, opc, mnemonic, ZPR32, "4s">;
   def _D : sve2p1_fp_reduction_q<0b11, opc, mnemonic, ZPR64, "2d">;
 }
 
 
 // SVE Permute Vector - Quadwords (DUPQ)
 class sve2p1_dupq<bits<5> ind_tsz, string mnemonic, ZPRRegOp zprty, Operand itype>
     : I<(outs zprty:$Zd), (ins zprty:$Zn, itype:$index),
         mnemonic, "\t$Zd, $Zn$index",
         "", []>, Sched<[]> {
   bits<5> Zd;
   bits<5> Zn;
   let Inst{31-21} = 0b00000101001;
   let Inst{20-16} = ind_tsz;
   let Inst{15-10} = 0b001001;
   let Inst{9-5} = Zn;
   let Inst{4-0} = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_dupq<string mnemonic> {
   def _B : sve2p1_dupq<{?, ?, ?, ?, 1}, mnemonic, ZPR8, VectorIndexB32b> {
     bits<4> index;
     let Inst{20-17} = index;
   }
   def _H : sve2p1_dupq<{?, ?, ?, 1, 0}, mnemonic, ZPR16, VectorIndexH32b> {
class sve2p1_extq<string mnemonic>
   let Inst{19-16} = imm4;
   let Inst{15-10} = 0b001001;
   let Inst{9-5} = Zm;
   let Inst{4-0} = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
   let DestructiveInstType = DestructiveOther;
   let ElementSize = ZPR8.ElementSize;
+  let hasSideEffects = 0;
 }
 
 
 // SVE move predicate from vector
 class sve2p1_vector_to_pred<bits<4> opc, string mnemonic,
                             PPRRegOp ppr_ty, Operand itype>
     : I<(outs ppr_ty:$Pd), (ins ZPRAny:$Zn, itype:$index),
         mnemonic, "\t$Pd, $Zn$index",
         "", []>, Sched<[]> {
   bits<4> Pd;
   bits<5> Zn;
   let Inst{31-24} = 0b00000101;
   let Inst{23-22} = opc{3-2};
   let Inst{21-19} = 0b101;
   let Inst{18-17} = opc{1-0};
   let Inst{16-10} = 0b0001110;
   let Inst{9-5}   = Zn;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_vector_to_pred<string mnemonic> {
   def _B : sve2p1_vector_to_pred<{0, 0, 0, 1}, mnemonic, PPR8,  VectorIndex0>;
   def _H : sve2p1_vector_to_pred<{0, 0, 1, ?}, mnemonic, PPR16, VectorIndexD32b> {
     bits<1> index;
     let Inst{17} = index;
   }
class sve2p1_pred_to_vector<bits<4> opc, string mnemonic,
   let Inst{23-22} = opc{3-2};
   let Inst{21-19} = 0b101;
   let Inst{18-17} = opc{1-0};
   let Inst{16-9}  = 0b10011100;
   let Inst{8-5}   = Pn;
   let Inst{4-0}   = Zd;
 
   let Constraints = "$Zd = $_Zd";
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_pred_to_vector<string mnemonic> {
   def _B : sve2p1_pred_to_vector<{0, 0, 0, 1}, mnemonic, PPR8,  VectorIndex0>;
   def _H : sve2p1_pred_to_vector<{0, 0, 1, ?}, mnemonic, PPR16, VectorIndexD32b> {
     bits<1> index;
     let Inst{17} = index;
   }
class sve2p1_int_reduce_q<bits<2> sz, bits<4> opc, string mnemonic,
   let Inst{21}    = 0b0;
   let Inst{20-19} = opc{3-2};
   let Inst{18}    = 0b1;
   let Inst{17-16} = opc{1-0};
   let Inst{15-13} = 0b001;
   let Inst{12-10} = Pg;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Vd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_int_reduce_q<bits<4> opc, string mnemonic> {
   def _B : sve2p1_int_reduce_q<0b00, opc, mnemonic, ZPR8,  "16b">;
   def _H : sve2p1_int_reduce_q<0b01, opc, mnemonic, ZPR16, "8h">;
   def _S : sve2p1_int_reduce_q<0b10, opc, mnemonic, ZPR32, "4s">;
   def _D : sve2p1_int_reduce_q<0b11, opc, mnemonic, ZPR64, "2d">;
 }
class sve2p1_permute_vec_elems_q<bits<2> sz, bits<3> opc, string mnemonic,
   let Inst{31-24} = 0b01000100;
   let Inst{23-22} = sz;
   let Inst{21}    = 0b0;
   let Inst{20-16} = Zm;
   let Inst{15-13} = 0b111;
   let Inst{12-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+
+  let hasSideEffects = 0;
 }
 
 multiclass sve2p1_permute_vec_elems_q<bits<3> opc, string mnemonic> {
   def _B : sve2p1_permute_vec_elems_q<0b00, opc, mnemonic, ZPR8,  ZPR8>;
   def _H : sve2p1_permute_vec_elems_q<0b01, opc, mnemonic, ZPR16, ZPR16>;
   def _S : sve2p1_permute_vec_elems_q<0b10, opc, mnemonic, ZPR32, ZPR32>;
   def _D : sve2p1_permute_vec_elems_q<0b11, opc, mnemonic, ZPR64, ZPR64>;
 }
 
 multiclass sve2p1_tblq<string mnemonic> {
   def _B : sve2p1_permute_vec_elems_q<0b00, 0b110, mnemonic, ZPR8,  Z_b>;
   def _H : sve2p1_permute_vec_elems_q<0b01, 0b110, mnemonic, ZPR16, Z_h>;
   def _S : sve2p1_permute_vec_elems_q<0b10, 0b110, mnemonic, ZPR32, Z_s>;
   def _D : sve2p1_permute_vec_elems_q<0b11, 0b110, mnemonic, ZPR64, Z_d>;
 }

llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    mov x19, x1
 ; CHECK-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
 ; CHECK-NEXT:    str z0, [x8] // 16-byte Folded Spill
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    bl __extenddftf2
 ; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    ldr z1, [x8] // 16-byte Folded Reload
 ; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr z1, [x8] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov d1, v1.d[1]
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    bl __extenddftf2
 ; CHECK-NEXT:    add x8, sp, #48
 ; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr z0, [x8, #1, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    bl __extenddftf2
 ; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    ldr z1, [x8, #1, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr z1, [x8, #1, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov d1, v1.d[1]
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    bl __extenddftf2
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    stp q1, q0, [x19]
 ; CHECK-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    stp q0, q2, [x19, #32]

llvm/test/CodeGen/AArch64/sve-insert-vector.ll

 }
 
 define <vscale x 4 x bfloat> @insert_nxv4bf16_v4bf16(<vscale x 4 x bfloat> %sv0, <4 x bfloat> %v1) nounwind {
 ; CHECK-LABEL: insert_nxv4bf16_v4bf16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    st1h { z0.s }, p0, [sp, #1, mul vl]
 ; CHECK-NEXT:    addpl x8, sp, #4
+; CHECK-NEXT:    st1h { z0.s }, p0, [sp, #1, mul vl]
 ; CHECK-NEXT:    str d1, [x8]
 ; CHECK-NEXT:    ld1h { z0.s }, p0/z, [sp, #1, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %v0 = call <vscale x 4 x bfloat> @llvm.vector.insert.nxv4bf16.v4bf16(<vscale x 4 x bfloat> %sv0, <4 x bfloat> %v1, i64 0)
   ret <vscale x 4 x bfloat> %v0
 }

llvm/test/tools/llvm-mca/AArch64/A64FX/A64FX-sve-instructions.s

 # CHECK-NEXT:  1      4     0.50                        add z31.d, z31.d, #65280
 # CHECK-NEXT:  1      4     0.50                        add z31.d, z31.d, z31.d
 # CHECK-NEXT:  1      4     0.50                        add z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        add z31.h, z31.h, #65280
 # CHECK-NEXT:  1      4     0.50                        add z31.h, z31.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        add z31.s, p7/m, z31.s, z31.s
 # CHECK-NEXT:  1      4     0.50                        add z31.s, z31.s, #65280
 # CHECK-NEXT:  1      4     0.50                        add z31.s, z31.s, z31.s
-# CHECK-NEXT:  1      1     0.50                  U     addpl sp, sp, #31
-# CHECK-NEXT:  1      1     0.50                  U     addpl x0, x0, #-32
-# CHECK-NEXT:  1      1     0.50                  U     addpl x21, x21, #0
-# CHECK-NEXT:  1      1     0.50                  U     addpl x23, x8, #-1
+# CHECK-NEXT:  1      1     0.50                        addpl sp, sp, #31
+# CHECK-NEXT:  1      1     0.50                        addpl x0, x0, #-32
+# CHECK-NEXT:  1      1     0.50                        addpl x21, x21, #0
+# CHECK-NEXT:  1      1     0.50                        addpl x23, x8, #-1
 # CHECK-NEXT:  1      1     0.50                        addvl sp, sp, #31
 # CHECK-NEXT:  1      1     0.50                        addvl x0, x0, #-32
 # CHECK-NEXT:  1      1     0.50                        addvl x21, x21, #0
 # CHECK-NEXT:  1      1     0.50                        addvl x23, x8, #-1
 # CHECK-NEXT:  2      5     2.00                        adr z0.d, [z0.d, z0.d, lsl #1]
 # CHECK-NEXT:  2      5     2.00                        adr z0.d, [z0.d, z0.d, lsl #2]
 # CHECK-NEXT:  2      5     2.00                        adr z0.d, [z0.d, z0.d, lsl #3]
 # CHECK-NEXT:  1      4     0.50                        adr z0.d, [z0.d, z0.d, sxtw #1]

 # CHECK-NEXT:  1      4     1.00                        and z23.h, z23.h, #0x6
 # CHECK-NEXT:  1      4     1.00                        and z23.h, z23.h, #0xfff9
 # CHECK-NEXT:  1      4     0.50                        and z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  1      4     0.50                        and z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  1      4     0.50                        and z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        and z31.s, p7/m, z31.s, z31.s
 # CHECK-NEXT:  1      4     1.00                        and z5.b, z5.b, #0x6
 # CHECK-NEXT:  1      4     1.00                        and z5.b, z5.b, #0xf9
-# CHECK-NEXT:  1      3     1.00                  U     ands  p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     1.00                        ands  p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  10     46    5.00                        andv  b0, p7, z31.b
 # CHECK-NEXT:  7      34    3.50                        andv  d0, p7, z31.d
 # CHECK-NEXT:  9      42    4.50                        andv  h0, p7, z31.h
 # CHECK-NEXT:  8      38    4.00                        andv  s0, p7, z31.s
 # CHECK-NEXT:  1      4     0.50                        asr z0.b, p0/m, z0.b, #1
 # CHECK-NEXT:  1      4     0.50                        asr z0.b, p0/m, z0.b, z0.b
 # CHECK-NEXT:  1      4     0.50                        asr z0.b, p0/m, z0.b, z1.d
 # CHECK-NEXT:  1      4     0.50                        asr z0.b, z0.b, #1

 # CHECK-NEXT:  2      8     1.00                        asrd  z0.b, p0/m, z0.b, #1
 # CHECK-NEXT:  2      8     1.00                        asrd  z0.d, p0/m, z0.d, #1
 # CHECK-NEXT:  2      8     1.00                        asrd  z0.h, p0/m, z0.h, #1
 # CHECK-NEXT:  2      8     1.00                        asrd  z0.s, p0/m, z0.s, #1
 # CHECK-NEXT:  2      8     1.00                        asrd  z31.b, p0/m, z31.b, #8
 # CHECK-NEXT:  2      8     1.00                        asrd  z31.d, p0/m, z31.d, #64
 # CHECK-NEXT:  2      8     1.00                        asrd  z31.h, p0/m, z31.h, #16
 # CHECK-NEXT:  2      8     1.00                        asrd  z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  1      4     0.50                  U     asrr  z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  1      4     0.50                  U     asrr  z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  1      4     0.50                  U     asrr  z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  1      4     0.50                  U     asrr  z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        asrr  z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        asrr  z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        asrr  z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        asrr  z0.s, p0/m, z0.s, z0.s
 # CHECK-NEXT:  1      3     1.00                        bic p0.b, p0/z, p0.b, p0.b
 # CHECK-NEXT:  1      3     1.00                        bic p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      4     0.50                        bic z0.d, z0.d, z0.d
 # CHECK-NEXT:  1      4     0.50                        bic z23.d, z13.d, z8.d
 # CHECK-NEXT:  1      4     0.50                        bic z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  1      4     0.50                        bic z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  1      4     0.50                        bic z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        bic z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  1      3     1.00                  U     bics  p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  1      3     1.00                  U     bics  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        bics  p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      3     1.00                        bics  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      3     1.00                        brka  p0.b, p15/m, p15.b
 # CHECK-NEXT:  1      3     1.00                        brka  p0.b, p15/z, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     brkas p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     1.00                        brkas p0.b, p15/z, p15.b
 # CHECK-NEXT:  1      3     1.00                        brkb  p0.b, p15/m, p15.b
 # CHECK-NEXT:  1      3     1.00                        brkb  p0.b, p15/z, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     brkbs p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     1.00                        brkbs p0.b, p15/z, p15.b
 # CHECK-NEXT:  1      3     1.00                        brkn  p0.b, p15/z, p1.b, p0.b
 # CHECK-NEXT:  1      3     1.00                        brkn  p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     brkns p0.b, p15/z, p1.b, p0.b
-# CHECK-NEXT:  1      3     1.00                  U     brkns p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        brkns p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  1      3     1.00                        brkns p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      3     1.00                        brkpa p0.b, p15/z, p1.b, p2.b
 # CHECK-NEXT:  1      3     1.00                        brkpa p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     brkpas  p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  1      3     1.00                  U     brkpas  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        brkpas  p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      3     1.00                        brkpas  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      3     1.00                        brkpb p0.b, p15/z, p1.b, p2.b
 # CHECK-NEXT:  1      3     1.00                        brkpb p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     brkpbs  p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  1      3     1.00                  U     brkpbs  p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  1      6     1.00                  U     clasta  b0, p7, b0, z31.b
+# CHECK-NEXT:  1      3     1.00                        brkpbs  p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      3     1.00                        brkpbs  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        clasta  b0, p7, b0, z31.b
 # CHECK-NEXT:  1      6     1.00                        clasta  d0, p7, d0, z31.d
 # CHECK-NEXT:  1      6     1.00                        clasta  h0, p7, h0, z31.h
 # CHECK-NEXT:  1      6     1.00                        clasta  s0, p7, s0, z31.s
 # CHECK-NEXT:  1      29    1.00                        clasta  w0, p7, w0, z31.b
 # CHECK-NEXT:  1      29    1.00                        clasta  w0, p7, w0, z31.h
 # CHECK-NEXT:  1      29    1.00                        clasta  w0, p7, w0, z31.s
 # CHECK-NEXT:  1      29    1.00                        clasta  x0, p7, x0, z31.d
 # CHECK-NEXT:  1      6     1.00                        clasta  z0.b, p7, z0.b, z31.b
 # CHECK-NEXT:  1      6     1.00                        clasta  z0.d, p7, z0.d, z31.d
 # CHECK-NEXT:  1      6     1.00                        clasta  z0.h, p7, z0.h, z31.h
 # CHECK-NEXT:  1      6     1.00                        clasta  z0.s, p7, z0.s, z31.s
-# CHECK-NEXT:  1      6     1.00                  U     clastb  b0, p7, b0, z31.b
+# CHECK-NEXT:  1      6     1.00                        clastb  b0, p7, b0, z31.b
 # CHECK-NEXT:  1      6     1.00                        clastb  d0, p7, d0, z31.d
 # CHECK-NEXT:  1      6     1.00                        clastb  h0, p7, h0, z31.h
 # CHECK-NEXT:  1      6     1.00                        clastb  s0, p7, s0, z31.s
 # CHECK-NEXT:  1      29    1.00                        clastb  w0, p7, w0, z31.b
 # CHECK-NEXT:  1      29    1.00                        clastb  w0, p7, w0, z31.h
 # CHECK-NEXT:  1      29    1.00                        clastb  w0, p7, w0, z31.s
 # CHECK-NEXT:  1      29    1.00                        clastb  x0, p7, x0, z31.d
 # CHECK-NEXT:  1      6     1.00                        clastb  z0.b, p7, z0.b, z31.b

 # CHECK-NEXT:  1      6     1.00                        cntp  x0, p15, p0.h
 # CHECK-NEXT:  1      6     1.00                        cntp  x0, p15, p0.s
 # CHECK-NEXT:  1      1     0.50                        cntw  x0
 # CHECK-NEXT:  1      1     0.50                        cntw  x0, #28
 # CHECK-NEXT:  1      1     0.50                        cntw  x0, all, mul #16
 # CHECK-NEXT:  1      1     0.50                        cntw  x0, pow2
 # CHECK-NEXT:  1      6     1.00                        compact z31.d, p7, z31.d
 # CHECK-NEXT:  1      6     1.00                        compact z31.s, p7, z31.s
-# CHECK-NEXT:  1      2     1.00                  U     ctermeq w30, wzr
-# CHECK-NEXT:  1      2     1.00                  U     ctermeq wzr, w30
-# CHECK-NEXT:  1      2     1.00                  U     ctermeq x30, xzr
-# CHECK-NEXT:  1      2     1.00                  U     ctermeq xzr, x30
-# CHECK-NEXT:  1      2     1.00                  U     ctermne w30, wzr
-# CHECK-NEXT:  1      2     1.00                  U     ctermne wzr, w30
-# CHECK-NEXT:  1      2     1.00                  U     ctermne x30, xzr
-# CHECK-NEXT:  1      2     1.00                  U     ctermne xzr, x30
+# CHECK-NEXT:  1      2     1.00                        ctermeq w30, wzr
+# CHECK-NEXT:  1      2     1.00                        ctermeq wzr, w30
+# CHECK-NEXT:  1      2     1.00                        ctermeq x30, xzr
+# CHECK-NEXT:  1      2     1.00                        ctermeq xzr, x30
+# CHECK-NEXT:  1      2     1.00                        ctermne w30, wzr
+# CHECK-NEXT:  1      2     1.00                        ctermne wzr, w30
+# CHECK-NEXT:  1      2     1.00                        ctermne x30, xzr
+# CHECK-NEXT:  1      2     1.00                        ctermne xzr, x30
 # CHECK-NEXT:  1      1     0.50                        decb  x0
 # CHECK-NEXT:  1      1     0.50                        decb  x0, #14
 # CHECK-NEXT:  1      1     0.50                        decb  x0, all, mul #16
 # CHECK-NEXT:  1      1     0.50                        decb  x0, pow2
 # CHECK-NEXT:  1      1     0.50                        decb  x0, vl1
 # CHECK-NEXT:  1      1     0.50                        decd  x0
 # CHECK-NEXT:  1      1     0.50                        decd  x0, #14
 # CHECK-NEXT:  1      1     0.50                        decd  x0, all, mul #16
 # CHECK-NEXT:  1      1     0.50                        decd  x0, pow2
 # CHECK-NEXT:  1      1     0.50                        decd  x0, vl1
 # CHECK-NEXT:  1      1     0.50                        dech  x0
 # CHECK-NEXT:  1      1     0.50                        dech  x0, #14
 # CHECK-NEXT:  1      1     0.50                        dech  x0, all, mul #16
 # CHECK-NEXT:  1      1     0.50                        dech  x0, pow2
 # CHECK-NEXT:  1      1     0.50                        dech  x0, vl1
 # CHECK-NEXT:  2      7     1.00                        decp  x0, p0.b
 # CHECK-NEXT:  2      7     1.00                        decp  x0, p0.d
 # CHECK-NEXT:  2      7     1.00                        decp  x0, p0.h
 # CHECK-NEXT:  2      7     1.00                        decp  x0, p0.s
 # CHECK-NEXT:  2      7     1.00                        decp  xzr, p15.b
 # CHECK-NEXT:  2      7     1.00                        decp  xzr, p15.d
 # CHECK-NEXT:  2      7     1.00                        decp  xzr, p15.h
 # CHECK-NEXT:  2      7     1.00                        decp  xzr, p15.s
-# CHECK-NEXT:  1      12    1.00                  U     decp  z31.d, p15.d
-# CHECK-NEXT:  1      12    1.00                  U     decp  z31.h, p15.h
-# CHECK-NEXT:  1      12    1.00                  U     decp  z31.s, p15.s
+# CHECK-NEXT:  1      12    1.00                        decp  z31.d, p15.d
+# CHECK-NEXT:  1      12    1.00                        decp  z31.h, p15.h
+# CHECK-NEXT:  1      12    1.00                        decp  z31.s, p15.s
 # CHECK-NEXT:  1      1     0.50                        decw  x0
 # CHECK-NEXT:  1      1     0.50                        decw  x0, #14
 # CHECK-NEXT:  1      1     0.50                        decw  x0, all, mul #16
 # CHECK-NEXT:  1      1     0.50                        decw  x0, pow2
 # CHECK-NEXT:  1      1     0.50                        decw  x0, vl1
 # CHECK-NEXT:  1      4     1.00                        dupm  z0.d, #0xfffffffffffffff9
 # CHECK-NEXT:  1      4     1.00                        dupm  z0.s, #0xfffffff9
 # CHECK-NEXT:  1      4     1.00                        dupm  z23.h, #0xfff9
 # CHECK-NEXT:  1      4     1.00                        dupm  z5.b, #0xf9
 # CHECK-NEXT:  1      3     1.00                        eor p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  1      4     1.00                        eor z0.d, z0.d, #0x6
 # CHECK-NEXT:  1      4     1.00                        eor z0.d, z0.d, #0xfffffffffffffff9
 # CHECK-NEXT:  1      4     0.50                        eor z0.d, z0.d, z0.d
 # CHECK-NEXT:  1      4     1.00                        eor z0.s, z0.s, #0x6
 # CHECK-NEXT:  1      4     1.00                        eor z0.s, z0.s, #0xfffffff9
 # CHECK-NEXT:  1      4     0.50                        eor z23.d, z13.d, z8.d
 # CHECK-NEXT:  1      4     1.00                        eor z23.h, z23.h, #0x6
 # CHECK-NEXT:  1      4     1.00                        eor z23.h, z23.h, #0xfff9
 # CHECK-NEXT:  1      4     0.50                        eor z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  1      4     0.50                        eor z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  1      4     0.50                        eor z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        eor z31.s, p7/m, z31.s, z31.s
 # CHECK-NEXT:  1      4     1.00                        eor z5.b, z5.b, #0x6
 # CHECK-NEXT:  1      4     1.00                        eor z5.b, z5.b, #0xf9
-# CHECK-NEXT:  1      3     1.00                  U     eors  p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     1.00                        eors  p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  10     46    5.00                        eorv  b0, p7, z31.b
 # CHECK-NEXT:  7      34    3.50                        eorv  d0, p7, z31.d
 # CHECK-NEXT:  9      42    4.50                        eorv  h0, p7, z31.h
 # CHECK-NEXT:  8      38    4.00                        eorv  s0, p7, z31.s
 # CHECK-NEXT:  1      6     1.00                        ext z31.b, z31.b, z0.b, #0
 # CHECK-NEXT:  1      6     1.00                        ext z31.b, z31.b, z0.b, #255
 # CHECK-NEXT:  1      9     0.50                        fabd  z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  1      9     0.50                        fabd  z0.h, p7/m, z0.h, z31.h

 # CHECK-NEXT:  1      9     0.50                        fmls  z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  2      15    1.00                        fmls  z0.d, z1.d, z7.d[1]
 # CHECK-NEXT:  1      9     0.50                        fmls  z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  2      15    1.00                        fmls  z0.h, z1.h, z7.h[7]
 # CHECK-NEXT:  1      9     0.50                        fmls  z0.s, p7/m, z1.s, z31.s
 # CHECK-NEXT:  2      15    1.00                        fmls  z0.s, z1.s, z7.s[3]
 # CHECK-NEXT:  1      6     1.00                        fmov  z0.d, #-10.00000000
 # CHECK-NEXT:  1      6     1.00                        fmov  z0.d, #0.12500000
-# CHECK-NEXT:  1      4     1.00                  U     fmov  z0.d, p0/m, #-10.00000000
-# CHECK-NEXT:  1      4     1.00                  U     fmov  z0.d, p0/m, #0.12500000
+# CHECK-NEXT:  1      4     1.00                        fmov  z0.d, p0/m, #-10.00000000
+# CHECK-NEXT:  1      4     1.00                        fmov  z0.d, p0/m, #0.12500000
 # CHECK-NEXT:  1      6     1.00                        fmov  z0.h, #-0.12500000
-# CHECK-NEXT:  1      4     1.00                  U     fmov  z0.h, p0/m, #-0.12500000
+# CHECK-NEXT:  1      4     1.00                        fmov  z0.h, p0/m, #-0.12500000
 # CHECK-NEXT:  1      6     1.00                        fmov  z0.s, #-0.12500000
-# CHECK-NEXT:  1      4     1.00                  U     fmov  z0.s, p0/m, #-0.12500000
+# CHECK-NEXT:  1      4     1.00                        fmov  z0.s, p0/m, #-0.12500000
 # CHECK-NEXT:  1      9     0.50                        fmsb  z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      9     0.50                        fmsb  z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      9     0.50                        fmsb  z0.s, p7/m, z1.s, z31.s
 # CHECK-NEXT:  1      9     0.50                        fmul  z0.d, p0/m, z0.d, #0.5
 # CHECK-NEXT:  1      9     0.50                        fmul  z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  2      15    1.00                        fmul  z0.d, z0.d, z0.d[0]
 # CHECK-NEXT:  1      9     0.50                        fmul  z0.d, z1.d, z31.d
 # CHECK-NEXT:  1      9     0.50                        fmul  z0.h, p0/m, z0.h, #0.5

 # CHECK-NEXT:  2      7     1.00                        incp  x0, p0.b
 # CHECK-NEXT:  2      7     1.00                        incp  x0, p0.d
 # CHECK-NEXT:  2      7     1.00                        incp  x0, p0.h
 # CHECK-NEXT:  2      7     1.00                        incp  x0, p0.s
 # CHECK-NEXT:  2      7     1.00                        incp  xzr, p15.b
 # CHECK-NEXT:  2      7     1.00                        incp  xzr, p15.d
 # CHECK-NEXT:  2      7     1.00                        incp  xzr, p15.h
 # CHECK-NEXT:  2      7     1.00                        incp  xzr, p15.s
-# CHECK-NEXT:  1      12    1.00                  U     incp  z31.d, p15.d
-# CHECK-NEXT:  1      12    1.00                  U     incp  z31.h, p15.h
-# CHECK-NEXT:  1      12    1.00                  U     incp  z31.s, p15.s
+# CHECK-NEXT:  1      12    1.00                        incp  z31.d, p15.d
+# CHECK-NEXT:  1      12    1.00                        incp  z31.h, p15.h
+# CHECK-NEXT:  1      12    1.00                        incp  z31.s, p15.s
 # CHECK-NEXT:  1      1     0.50                        incw  x0
 # CHECK-NEXT:  1      1     0.50                        incw  x0, #14
 # CHECK-NEXT:  1      1     0.50                        incw  x0, all, mul #16
 # CHECK-NEXT:  1      1     0.50                        incw  x0, pow2
 # CHECK-NEXT:  1      1     0.50                        incw  x0, vl1
 # CHECK-NEXT:  1      4     0.50                        incw  z0.s
 # CHECK-NEXT:  1      4     0.50                        incw  z0.s, all, mul #16
 # CHECK-NEXT:  2      13    2.00                        index z0.b, #0, #0

 # CHECK-NEXT:  1      6     1.00                        insr  z31.b, b31
 # CHECK-NEXT:  1      10    1.00                        insr  z31.b, wzr
 # CHECK-NEXT:  1      6     1.00                        insr  z31.d, d31
 # CHECK-NEXT:  1      10    1.00                        insr  z31.d, xzr
 # CHECK-NEXT:  1      6     1.00                        insr  z31.h, h31
 # CHECK-NEXT:  1      10    1.00                        insr  z31.h, wzr
 # CHECK-NEXT:  1      6     1.00                        insr  z31.s, s31
 # CHECK-NEXT:  1      10    1.00                        insr  z31.s, wzr
-# CHECK-NEXT:  1      6     1.00                  U     lasta b0, p7, z31.b
+# CHECK-NEXT:  1      6     1.00                        lasta b0, p7, z31.b
 # CHECK-NEXT:  1      6     1.00                        lasta d0, p7, z31.d
 # CHECK-NEXT:  1      6     1.00                        lasta h0, p7, z31.h
 # CHECK-NEXT:  1      6     1.00                        lasta s0, p7, z31.s
 # CHECK-NEXT:  1      29    1.00                        lasta w0, p7, z31.b
 # CHECK-NEXT:  1      29    1.00                        lasta w0, p7, z31.h
 # CHECK-NEXT:  1      29    1.00                        lasta w0, p7, z31.s
 # CHECK-NEXT:  1      29    1.00                        lasta x0, p7, z31.d
-# CHECK-NEXT:  1      6     1.00                  U     lastb b0, p7, z31.b
+# CHECK-NEXT:  1      6     1.00                        lastb b0, p7, z31.b
 # CHECK-NEXT:  1      6     1.00                        lastb d0, p7, z31.d
 # CHECK-NEXT:  1      6     1.00                        lastb h0, p7, z31.h
 # CHECK-NEXT:  1      6     1.00                        lastb s0, p7, z31.s
 # CHECK-NEXT:  1      29    1.00                        lastb w0, p7, z31.b
 # CHECK-NEXT:  1      29    1.00                        lastb w0, p7, z31.h
 # CHECK-NEXT:  1      29    1.00                        lastb w0, p7, z31.s
 # CHECK-NEXT:  1      29    1.00                        lastb x0, p7, z31.d
 # CHECK-NEXT:  1      11    0.50    *                   ld1b  { z0.b }, p0/z, [sp, x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1b  { z0.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z0.b }, p0/z, [x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1b  { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1b  { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1b  { z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1b  { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z21.b }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1b  { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1b  { z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      16    2.00    *                   ld1b  { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      23    4.00    *                   ld1b  { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      23    4.00    *                   ld1b  { z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      19    4.00    *                   ld1b  { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1b  { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1b  { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      11    0.50    *                   ld1b  { z21.s }, p5/z, [x10, x21]
 # CHECK-NEXT:  1      11    0.50    *                   ld1b  { z23.d }, p3/z, [x13, x8]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z31.b }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1b  { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1b  { z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1b  { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1b  { z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1b  { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      16    2.00    *                   ld1b  { z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1b  { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      19    4.00    *                   ld1b  { z31.s }, p7/z, [z31.s, #31]
 # CHECK-NEXT:  1      11    0.50    *                   ld1b  { z5.h }, p3/z, [x17, x16]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1d  { z0.d }, p0/z, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1d  { z0.d }, p0/z, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1d  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1d  { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1d  { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1d  { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1d  { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1d  { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      20    2.00    *                   ld1d  { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      11    0.50    *                   ld1d  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      16    2.00    *                   ld1d  { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      11    0.50    *                   ld1d  { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1d  { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1d  { z21.d }, p5/z, [x10, z21.d, uxtw]
 # CHECK-NEXT:  1      11    0.50    *                   ld1d  { z23.d }, p3/z, [sp, x8, lsl #3]
 # CHECK-NEXT:  1      11    0.50    *                   ld1d  { z23.d }, p3/z, [x13, x8, lsl #3]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1d  { z23.d }, p3/z, [x13, z8.d, lsl #3]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1d  { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1d  { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1d  { z31.d }, p7/z, [z31.d, #248]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1h  { z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1h  { z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1h  { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1h  { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1h  { z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1h  { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1h  { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1h  { z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1d  { z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  1      11    0.50    *                   ld1d  { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1d  { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      16    2.00    *                   ld1d  { z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  1      20    2.00    *                   ld1h  { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      20    2.00    *                   ld1h  { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      16    2.00    *                   ld1h  { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      23    4.00    *                   ld1h  { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      23    4.00    *                   ld1h  { z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      19    4.00    *                   ld1h  { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1h  { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1h  { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      11    0.50    *                   ld1h  { z21.s }, p5/z, [x10, x21, lsl #1]
 # CHECK-NEXT:  1      11    0.50    *                   ld1h  { z23.d }, p3/z, [x13, x8, lsl #1]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1h  { z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1h  { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1h  { z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1h  { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1h  { z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1h  { z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1h  { z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      20    2.00    *                   ld1h  { z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1h  { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      16    2.00    *                   ld1h  { z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1h  { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      23    4.00    *                   ld1h  { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      23    4.00    *                   ld1h  { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      19    4.00    *                   ld1h  { z31.s }, p7/z, [z31.s, #62]
 # CHECK-NEXT:  1      11    0.50    *                   ld1h  { z5.h }, p3/z, [sp, x16, lsl #1]
 # CHECK-NEXT:  1      11    0.50    *                   ld1h  { z5.h }, p3/z, [x17, x16, lsl #1]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rb { z0.b }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rb { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rb { z0.h }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rb { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rb { z31.b }, p7/z, [sp, #63]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rb { z31.d }, p7/z, [sp, #63]

 # CHECK-NEXT:  1      11    0.50    *                   ld1rsh  { z31.d }, p7/z, [sp, #126]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rsh  { z31.s }, p7/z, [sp, #126]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rsw  { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rsw  { z31.d }, p7/z, [sp, #252]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rw { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rw { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rw { z31.d }, p7/z, [sp, #252]
 # CHECK-NEXT:  1      11    0.50    *                   ld1rw { z31.s }, p7/z, [sp, #252]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1sb { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      16    2.00    *                   ld1sb { z0.d }, p0/z, [z0.d]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sb { z0.h }, p0/z, [sp, x0]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sb { z0.h }, p0/z, [x0, x0]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1sb { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sb { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sb { z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      23    4.00    *                   ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      19    4.00    *                   ld1sb { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sb { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sb { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sb { z21.s }, p5/z, [x10, x21]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sb { z23.d }, p3/z, [x13, x8]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sb { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1sb { z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sb { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1sb { z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sh { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1sh { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sh { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1sh { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sh { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sh { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sh { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sb { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      16    2.00    *                   ld1sb { z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sb { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      19    4.00    *                   ld1sb { z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sh { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      16    2.00    *                   ld1sh { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      23    4.00    *                   ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      23    4.00    *                   ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sh { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      19    4.00    *                   ld1sh { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sh { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sh { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sh { z21.d }, p5/z, [x10, z21.d, uxtw]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sh { z21.s }, p5/z, [sp, x21, lsl #1]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sh { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sh { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sh { z21.s }, p5/z, [x10, x21, lsl #1]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sh { z23.d }, p3/z, [x13, x8, lsl #1]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sh { z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sh { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sh { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1sh { z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sh { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1sh { z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sw { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1sw { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sw { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sw { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sw { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sh { z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sh { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sh { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      16    2.00    *                   ld1sh { z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sh { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      23    4.00    *                   ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      23    4.00    *                   ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      19    4.00    *                   ld1sh { z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sw { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      16    2.00    *                   ld1sw { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sw { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sw { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sw { z21.d }, p5/z, [x10, z21.d, uxtw]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sw { z23.d }, p3/z, [sp, x8, lsl #2]
 # CHECK-NEXT:  1      11    0.50    *                   ld1sw { z23.d }, p3/z, [x13, x8, lsl #2]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sw { z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1sw { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1sw { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1sw { z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1w  { z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1w  { z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1w  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1w  { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1w  { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1w  { z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1w  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1w  { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1w  { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1w  { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1w  { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sw { z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      11    0.50    *                   ld1sw { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1sw { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      16    2.00    *                   ld1sw { z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      20    2.00    *                   ld1w  { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      20    2.00    *                   ld1w  { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      11    0.50    *                   ld1w  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      16    2.00    *                   ld1w  { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      23    4.00    *                   ld1w  { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      23    4.00    *                   ld1w  { z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      11    0.50    *                   ld1w  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      19    4.00    *                   ld1w  { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      11    0.50    *                   ld1w  { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1w  { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      20    2.00    *                   ld1w  { z21.d }, p5/z, [x10, z21.d, uxtw]
 # CHECK-NEXT:  1      11    0.50    *                   ld1w  { z21.s }, p5/z, [sp, x21, lsl #2]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1w  { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      11    0.50    *                   ld1w  { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      11    0.50    *                   ld1w  { z21.s }, p5/z, [x10, x21, lsl #2]
 # CHECK-NEXT:  1      11    0.50    *                   ld1w  { z23.d }, p3/z, [x13, x8, lsl #2]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1w  { z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1w  { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      20    2.00    *             U     ld1w  { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  1      16    2.00    *             U     ld1w  { z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  1      11    0.50    *             U     ld1w  { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1w  { z31.s }, p7/z, [sp, z31.s, sxtw #2]
-# CHECK-NEXT:  1      23    4.00    *             U     ld1w  { z31.s }, p7/z, [sp, z31.s, uxtw #2]
-# CHECK-NEXT:  1      19    4.00    *             U     ld1w  { z31.s }, p7/z, [z31.s, #124]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2b  { z0.b, z1.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2b  { z0.b, z1.b }, p0/z, [x0]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2b  { z21.b, z22.b }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2b  { z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2b  { z5.b, z6.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  3      12    1.50    *             U     ld2d  { z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  2      11    1.00    *             U     ld2d  { z0.d, z1.d }, p0/z, [x0]
-# CHECK-NEXT:  2      11    1.00    *             U     ld2d  { z21.d, z22.d }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  2      11    1.00    *             U     ld2d  { z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  3      12    1.50    *             U     ld2d  { z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2h  { z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2h  { z0.h, z1.h }, p0/z, [x0]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2h  { z21.h, z22.h }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2h  { z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  3      15    4.50    *             U     ld2h  { z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  3      12    1.50    *             U     ld2w  { z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  2      11    1.00    *             U     ld2w  { z0.s, z1.s }, p0/z, [x0]
-# CHECK-NEXT:  2      11    1.00    *             U     ld2w  { z21.s, z22.s }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  2      11    1.00    *             U     ld2w  { z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  3      12    1.50    *             U     ld2w  { z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3b  { z0.b - z2.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3b  { z0.b - z2.b }, p0/z, [x0]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3b  { z21.b - z23.b }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3b  { z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3b  { z5.b - z7.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  4      12    2.00    *             U     ld3d  { z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  3      11    1.50    *             U     ld3d  { z0.d - z2.d }, p0/z, [x0]
-# CHECK-NEXT:  3      11    1.50    *             U     ld3d  { z21.d - z23.d }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  3      11    1.50    *             U     ld3d  { z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  4      12    2.00    *             U     ld3d  { z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3h  { z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3h  { z0.h - z2.h }, p0/z, [x0]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3h  { z21.h - z23.h }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3h  { z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  4      15    6.50    *             U     ld3h  { z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  4      12    2.00    *             U     ld3w  { z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  3      11    1.50    *             U     ld3w  { z0.s - z2.s }, p0/z, [x0]
-# CHECK-NEXT:  3      11    1.50    *             U     ld3w  { z21.s - z23.s }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  3      11    1.50    *             U     ld3w  { z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  4      12    2.00    *             U     ld3w  { z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4b  { z0.b - z3.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4b  { z0.b - z3.b }, p0/z, [x0]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4b  { z21.b - z24.b }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4b  { z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4b  { z5.b - z8.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  5      12    2.50    *             U     ld4d  { z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  4      11    2.00    *             U     ld4d  { z0.d - z3.d }, p0/z, [x0]
-# CHECK-NEXT:  4      11    2.00    *             U     ld4d  { z21.d - z24.d }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  4      11    2.00    *             U     ld4d  { z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  5      12    2.50    *             U     ld4d  { z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4h  { z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4h  { z0.h - z3.h }, p0/z, [x0]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4h  { z21.h - z24.h }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4h  { z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  5      15    8.50    *             U     ld4h  { z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  5      12    2.50    *             U     ld4w  { z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  4      11    2.00    *             U     ld4w  { z0.s - z3.s }, p0/z, [x0]
-# CHECK-NEXT:  4      11    2.00    *             U     ld4w  { z21.s - z24.s }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  4      11    2.00    *             U     ld4w  { z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  5      12    2.50    *             U     ld4w  { z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      20    2.00    *                   ld1w  { z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      11    0.50    *                   ld1w  { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      20    2.00    *                   ld1w  { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      16    2.00    *                   ld1w  { z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      11    0.50    *                   ld1w  { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      23    4.00    *                   ld1w  { z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  1      23    4.00    *                   ld1w  { z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  1      19    4.00    *                   ld1w  { z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  3      15    4.50    *                   ld2b  { z0.b, z1.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  3      15    4.50    *                   ld2b  { z0.b, z1.b }, p0/z, [x0]
+# CHECK-NEXT:  3      15    4.50    *                   ld2b  { z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  3      15    4.50    *                   ld2b  { z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  3      15    4.50    *                   ld2b  { z5.b, z6.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  3      12    1.50    *                   ld2d  { z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  2      11    1.00    *                   ld2d  { z0.d, z1.d }, p0/z, [x0]
+# CHECK-NEXT:  2      11    1.00    *                   ld2d  { z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  2      11    1.00    *                   ld2d  { z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  3      12    1.50    *                   ld2d  { z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  3      15    4.50    *                   ld2h  { z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  3      15    4.50    *                   ld2h  { z0.h, z1.h }, p0/z, [x0]
+# CHECK-NEXT:  3      15    4.50    *                   ld2h  { z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  3      15    4.50    *                   ld2h  { z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  3      15    4.50    *                   ld2h  { z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  3      12    1.50    *                   ld2w  { z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  2      11    1.00    *                   ld2w  { z0.s, z1.s }, p0/z, [x0]
+# CHECK-NEXT:  2      11    1.00    *                   ld2w  { z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  2      11    1.00    *                   ld2w  { z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  3      12    1.50    *                   ld2w  { z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  4      15    6.50    *                   ld3b  { z0.b - z2.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  4      15    6.50    *                   ld3b  { z0.b - z2.b }, p0/z, [x0]
+# CHECK-NEXT:  4      15    6.50    *                   ld3b  { z21.b - z23.b }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  4      15    6.50    *                   ld3b  { z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  4      15    6.50    *                   ld3b  { z5.b - z7.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  4      12    2.00    *                   ld3d  { z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  3      11    1.50    *                   ld3d  { z0.d - z2.d }, p0/z, [x0]
+# CHECK-NEXT:  3      11    1.50    *                   ld3d  { z21.d - z23.d }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  3      11    1.50    *                   ld3d  { z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  4      12    2.00    *                   ld3d  { z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  4      15    6.50    *                   ld3h  { z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  4      15    6.50    *                   ld3h  { z0.h - z2.h }, p0/z, [x0]
+# CHECK-NEXT:  4      15    6.50    *                   ld3h  { z21.h - z23.h }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  4      15    6.50    *                   ld3h  { z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  4      15    6.50    *                   ld3h  { z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  4      12    2.00    *                   ld3w  { z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  3      11    1.50    *                   ld3w  { z0.s - z2.s }, p0/z, [x0]
+# CHECK-NEXT:  3      11    1.50    *                   ld3w  { z21.s - z23.s }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  3      11    1.50    *                   ld3w  { z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  4      12    2.00    *                   ld3w  { z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  5      15    8.50    *                   ld4b  { z0.b - z3.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  5      15    8.50    *                   ld4b  { z0.b - z3.b }, p0/z, [x0]
+# CHECK-NEXT:  5      15    8.50    *                   ld4b  { z21.b - z24.b }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  5      15    8.50    *                   ld4b  { z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  5      15    8.50    *                   ld4b  { z5.b - z8.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  5      12    2.50    *                   ld4d  { z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  4      11    2.00    *                   ld4d  { z0.d - z3.d }, p0/z, [x0]
+# CHECK-NEXT:  4      11    2.00    *                   ld4d  { z21.d - z24.d }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  4      11    2.00    *                   ld4d  { z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  5      12    2.50    *                   ld4d  { z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  5      15    8.50    *                   ld4h  { z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  5      15    8.50    *                   ld4h  { z0.h - z3.h }, p0/z, [x0]
+# CHECK-NEXT:  5      15    8.50    *                   ld4h  { z21.h - z24.h }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  5      15    8.50    *                   ld4h  { z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  5      15    8.50    *                   ld4h  { z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  5      12    2.50    *                   ld4w  { z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  4      11    2.00    *                   ld4w  { z0.s - z3.s }, p0/z, [x0]
+# CHECK-NEXT:  4      11    2.00    *                   ld4w  { z21.s - z24.s }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  4      11    2.00    *                   ld4w  { z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  5      12    2.50    *                   ld4w  { z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
 # CHECK-NEXT:  1      11    0.50    *             U     ldff1b  { z0.d }, p0/z, [x0, x0]
 # CHECK-NEXT:  1      16    2.00    *             U     ldff1b  { z0.d }, p0/z, [z0.d]
 # CHECK-NEXT:  1      11    0.50    *             U     ldff1b  { z0.h }, p0/z, [x0, x0]
 # CHECK-NEXT:  1      11    0.50    *             U     ldff1b  { z0.s }, p0/z, [x0, x0]
 # CHECK-NEXT:  1      23    4.00    *             U     ldff1b  { z0.s }, p0/z, [x0, z0.s, sxtw]
 # CHECK-NEXT:  1      23    4.00    *             U     ldff1b  { z0.s }, p0/z, [x0, z0.s, uxtw]
 # CHECK-NEXT:  1      19    4.00    *             U     ldff1b  { z0.s }, p0/z, [z0.s]
 # CHECK-NEXT:  1      20    2.00    *             U     ldff1b  { z21.d }, p5/z, [x10, z21.d, sxtw]

 # CHECK-NEXT:  1      11    0.50    *                   ldnt1h  { z23.h }, p3/z, [x13, #-8, mul vl]
 # CHECK-NEXT:  1      11    0.50    *                   ldnt1w  { z0.s }, p0/z, [x0, x0, lsl #2]
 # CHECK-NEXT:  1      11    0.50    *                   ldnt1w  { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  1      11    0.50    *                   ldnt1w  { z21.s }, p5/z, [x10, #7, mul vl]
 # CHECK-NEXT:  1      11    0.50    *                   ldnt1w  { z23.s }, p3/z, [x13, #-8, mul vl]
 # CHECK-NEXT:  1      11    1.00    *                   ldr p0, [x0]
 # CHECK-NEXT:  1      11    1.00    *                   ldr p5, [x10, #255, mul vl]
 # CHECK-NEXT:  1      11    1.00    *                   ldr p7, [x13, #-256, mul vl]
-# CHECK-NEXT:  1      11    1.00    *             U     ldr z0, [x0]
-# CHECK-NEXT:  1      11    1.00    *             U     ldr z23, [x13, #255, mul vl]
-# CHECK-NEXT:  1      11    1.00    *             U     ldr z31, [sp, #-256, mul vl]
+# CHECK-NEXT:  1      11    1.00    *                   ldr z0, [x0]
+# CHECK-NEXT:  1      11    1.00    *                   ldr z23, [x13, #255, mul vl]
+# CHECK-NEXT:  1      11    1.00    *                   ldr z31, [sp, #-256, mul vl]
 # CHECK-NEXT:  1      4     0.50                        lsl z0.b, p0/m, z0.b, #0
 # CHECK-NEXT:  1      4     0.50                        lsl z0.b, p0/m, z0.b, z0.b
 # CHECK-NEXT:  1      4     0.50                        lsl z0.b, p0/m, z0.b, z1.d
 # CHECK-NEXT:  1      4     0.50                        lsl z0.b, z0.b, #0
 # CHECK-NEXT:  1      4     0.50                        lsl z0.b, z1.b, z2.d
 # CHECK-NEXT:  1      4     0.50                        lsl z0.d, p0/m, z0.d, #0
 # CHECK-NEXT:  1      4     0.50                        lsl z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  1      4     0.50                        lsl z0.d, z0.d, #0

 # CHECK-NEXT:  1      4     0.50                        lsl z31.b, p0/m, z31.b, #7
 # CHECK-NEXT:  1      4     0.50                        lsl z31.b, z31.b, #7
 # CHECK-NEXT:  1      4     0.50                        lsl z31.d, p0/m, z31.d, #63
 # CHECK-NEXT:  1      4     0.50                        lsl z31.d, z31.d, #63
 # CHECK-NEXT:  1      4     0.50                        lsl z31.h, p0/m, z31.h, #15
 # CHECK-NEXT:  1      4     0.50                        lsl z31.h, z31.h, #15
 # CHECK-NEXT:  1      4     0.50                        lsl z31.s, p0/m, z31.s, #31
 # CHECK-NEXT:  1      4     0.50                        lsl z31.s, z31.s, #31
-# CHECK-NEXT:  1      4     0.50                  U     lslr  z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  1      4     0.50                  U     lslr  z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  1      4     0.50                  U     lslr  z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  1      4     0.50                  U     lslr  z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        lslr  z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        lslr  z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        lslr  z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        lslr  z0.s, p0/m, z0.s, z0.s
 # CHECK-NEXT:  1      4     0.50                        lsr z0.b, p0/m, z0.b, #1
 # CHECK-NEXT:  1      4     0.50                        lsr z0.b, p0/m, z0.b, z0.b
 # CHECK-NEXT:  1      4     0.50                        lsr z0.b, p0/m, z0.b, z1.d
 # CHECK-NEXT:  1      4     0.50                        lsr z0.b, z0.b, #1
 # CHECK-NEXT:  1      4     0.50                        lsr z0.b, z1.b, z2.d
 # CHECK-NEXT:  1      4     0.50                        lsr z0.d, p0/m, z0.d, #1
 # CHECK-NEXT:  1      4     0.50                        lsr z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  1      4     0.50                        lsr z0.d, z0.d, #1

 # CHECK-NEXT:  1      4     0.50                        lsr z31.b, p0/m, z31.b, #8
 # CHECK-NEXT:  1      4     0.50                        lsr z31.b, z31.b, #8
 # CHECK-NEXT:  1      4     0.50                        lsr z31.d, p0/m, z31.d, #64
 # CHECK-NEXT:  1      4     0.50                        lsr z31.d, z31.d, #64
 # CHECK-NEXT:  1      4     0.50                        lsr z31.h, p0/m, z31.h, #16
 # CHECK-NEXT:  1      4     0.50                        lsr z31.h, z31.h, #16
 # CHECK-NEXT:  1      4     0.50                        lsr z31.s, p0/m, z31.s, #32
 # CHECK-NEXT:  1      4     0.50                        lsr z31.s, z31.s, #32
-# CHECK-NEXT:  1      4     0.50                  U     lsrr  z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  1      4     0.50                  U     lsrr  z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  1      4     0.50                  U     lsrr  z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  1      4     0.50                  U     lsrr  z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        lsrr  z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        lsrr  z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        lsrr  z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        lsrr  z0.s, p0/m, z0.s, z0.s
 # CHECK-NEXT:  1      9     0.50                        mad z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  1      9     0.50                        mad z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      9     0.50                        mad z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      9     0.50                        mad z0.s, p7/m, z1.s, z31.s
 # CHECK-NEXT:  1      9     0.50                        mla z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  1      9     0.50                        mla z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      9     0.50                        mla z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      9     0.50                        mla z0.s, p7/m, z1.s, z31.s
 # CHECK-NEXT:  1      9     0.50                        mls z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  1      9     0.50                        mls z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      9     0.50                        mls z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      9     0.50                        mls z0.s, p7/m, z1.s, z31.s
 # CHECK-NEXT:  1      3     1.00                        mov p0.b, p0.b
 # CHECK-NEXT:  1      3     1.00                        mov p0.b, p0/m, p0.b
 # CHECK-NEXT:  1      3     1.00                        mov p0.b, p0/z, p0.b
 # CHECK-NEXT:  1      3     1.00                        mov p15.b, p15.b
 # CHECK-NEXT:  1      3     1.00                        mov p15.b, p15/m, p15.b
 # CHECK-NEXT:  1      3     1.00                        mov p15.b, p15/z, p15.b
 # CHECK-NEXT:  1      4     1.00                        mov z0.b, #127
 # CHECK-NEXT:  1      4     1.00                        mov z0.b, b0
-# CHECK-NEXT:  1      6     1.00                  U     mov z0.b, p0/m, b0
+# CHECK-NEXT:  1      6     1.00                        mov z0.b, p0/m, b0
 # CHECK-NEXT:  1      8     1.00                        mov z0.b, p0/m, w0
 # CHECK-NEXT:  1      4     1.00                        mov z0.b, p0/z, #127
 # CHECK-NEXT:  1      6     1.00                        mov z0.b, w0
 # CHECK-NEXT:  1      4     1.00                        mov z0.d, #0
 # CHECK-NEXT:  1      4     1.00                        mov z0.d, #0xe0000000000003ff
 # CHECK-NEXT:  1      4     1.00                        mov z0.d, #0xffffffffffff7fff
 # CHECK-NEXT:  1      4     1.00                        mov z0.d, #32768
 # CHECK-NEXT:  1      4     1.00                        mov z0.d, d0

 # CHECK-NEXT:  1      4     1.00                        mov z21.s, #32512
 # CHECK-NEXT:  1      4     1.00                        mov z21.s, p0/z, #-128
 # CHECK-NEXT:  1      4     1.00                        mov z21.s, p0/z, #-32768
 # CHECK-NEXT:  1      4     1.00                        mov z21.s, p0/z, #127
 # CHECK-NEXT:  1      4     1.00                        mov z21.s, p0/z, #32512
 # CHECK-NEXT:  1      4     1.00                        mov z21.s, p15/m, #-128
 # CHECK-NEXT:  1      4     1.00                        mov z21.s, p15/m, #-32768
 # CHECK-NEXT:  1      4     0.50                        mov z31.b, p15/m, z31.b
-# CHECK-NEXT:  1      6     1.00                  U     mov z31.b, p7/m, b31
+# CHECK-NEXT:  1      6     1.00                        mov z31.b, p7/m, b31
 # CHECK-NEXT:  1      1     0.17                        movprfx z31, z6
 # CHECK-NEXT:  1      8     1.00                        mov z31.b, p7/m, wsp
 # CHECK-NEXT:  1      6     1.00                        mov z31.b, wsp
 # CHECK-NEXT:  1      4     1.00                        mov z31.b, z31.b[63]
 # CHECK-NEXT:  1      4     0.50                        mov z31.d, p15/m, z31.d
 # CHECK-NEXT:  1      6     1.00                        mov z31.d, p7/m, d31
-# CHECK-NEXT:  1      1     0.17                  U     movprfx z31.d, p7/z, z6.d
+# CHECK-NEXT:  1      1     0.17                        movprfx z31.d, p7/z, z6.d
 # CHECK-NEXT:  1      8     1.00                        mov z31.d, p7/m, sp
 # CHECK-NEXT:  1      6     1.00                        mov z31.d, sp
 # CHECK-NEXT:  1      4     0.50                        mov z31.d, z0.d
 # CHECK-NEXT:  1      4     1.00                        mov z31.d, z31.d[7]
 # CHECK-NEXT:  1      4     0.50                        mov z31.h, p15/m, z31.h
 # CHECK-NEXT:  1      6     1.00                        mov z31.h, p7/m, h31
 # CHECK-NEXT:  1      8     1.00                        mov z31.h, p7/m, wsp
 # CHECK-NEXT:  1      6     1.00                        mov z31.h, wsp

 # CHECK-NEXT:  1      4     1.00                        mov z5.b, p0/z, #-1
 # CHECK-NEXT:  1      4     1.00                        mov z5.b, p0/z, #-128
 # CHECK-NEXT:  1      4     1.00                        mov z5.b, p0/z, #127
 # CHECK-NEXT:  1      4     1.00                        mov z5.b, p15/m, #-128
 # CHECK-NEXT:  1      4     1.00                        mov z5.d, #-6
 # CHECK-NEXT:  1      4     1.00                        mov z5.h, #-6
 # CHECK-NEXT:  1      4     1.00                        mov z5.q, z17.q[3]
 # CHECK-NEXT:  1      4     1.00                        mov z5.s, #-6
-# CHECK-NEXT:  1      3     1.00                  U     movs  p0.b, p0.b
-# CHECK-NEXT:  1      3     1.00                  U     movs  p0.b, p0/z, p0.b
-# CHECK-NEXT:  1      3     1.00                  U     movs  p15.b, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     movs  p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     1.00                        movs  p0.b, p0.b
+# CHECK-NEXT:  1      3     1.00                        movs  p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      3     1.00                        movs  p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        movs  p15.b, p15/z, p15.b
 # CHECK-NEXT:  1      1     0.17                  U     mrs x3, ID_AA64ZFR0_EL1
 # CHECK-NEXT:  1      1     0.17                  U     mrs x3, ZCR_EL1
 # CHECK-NEXT:  1      1     0.17                  U     mrs x3, ZCR_EL12
 # CHECK-NEXT:  1      1     0.17                  U     mrs x3, ZCR_EL2
 # CHECK-NEXT:  1      1     0.17                  U     mrs x3, ZCR_EL3
 # CHECK-NEXT:  1      9     0.50                        msb z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  1      9     0.50                        msb z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      9     0.50                        msb z0.h, p7/m, z1.h, z31.h

 # CHECK-NEXT:  1      9     1.00                        mul z31.d, z31.d, #-128
 # CHECK-NEXT:  1      9     1.00                        mul z31.d, z31.d, #127
 # CHECK-NEXT:  1      9     1.00                        mul z31.h, z31.h, #-128
 # CHECK-NEXT:  1      9     1.00                        mul z31.h, z31.h, #127
 # CHECK-NEXT:  1      9     1.00                        mul z31.s, z31.s, #-128
 # CHECK-NEXT:  1      9     1.00                        mul z31.s, z31.s, #127
 # CHECK-NEXT:  1      3     1.00                        nand  p0.b, p0/z, p0.b, p0.b
 # CHECK-NEXT:  1      3     1.00                        nand  p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     nands p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  1      3     1.00                  U     nands p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        nands p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      3     1.00                        nands p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      4     1.00                        neg z0.b, p0/m, z0.b
 # CHECK-NEXT:  1      4     1.00                        neg z0.d, p0/m, z0.d
 # CHECK-NEXT:  1      4     1.00                        neg z0.h, p0/m, z0.h
 # CHECK-NEXT:  1      4     1.00                        neg z0.s, p0/m, z0.s
 # CHECK-NEXT:  1      4     1.00                        neg z31.b, p7/m, z31.b
 # CHECK-NEXT:  1      4     1.00                        neg z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      4     1.00                        neg z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      4     1.00                        neg z31.s, p7/m, z31.s
 # CHECK-NEXT:  1      3     1.00                        nor p0.b, p0/z, p0.b, p0.b
 # CHECK-NEXT:  1      3     1.00                        nor p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     nors  p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  1      3     1.00                  U     nors  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        nors  p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      3     1.00                        nors  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      3     1.00                        not p0.b, p0/z, p0.b
 # CHECK-NEXT:  1      3     1.00                        not p15.b, p15/z, p15.b
 # CHECK-NEXT:  1      4     0.50                        not z31.b, p7/m, z31.b
 # CHECK-NEXT:  1      4     0.50                        not z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      4     0.50                        not z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      4     0.50                        not z31.s, p7/m, z31.s
-# CHECK-NEXT:  1      3     1.00                  U     nots  p0.b, p0/z, p0.b
-# CHECK-NEXT:  1      3     1.00                  U     nots  p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     1.00                        nots  p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      3     1.00                        nots  p15.b, p15/z, p15.b
 # CHECK-NEXT:  1      3     1.00                        orn p0.b, p0/z, p0.b, p0.b
 # CHECK-NEXT:  1      3     1.00                        orn p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  1      3     1.00                  U     orns  p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  1      3     1.00                  U     orns  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        orns  p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      3     1.00                        orns  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      3     1.00                        orr p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  1      4     1.00                        orr z0.d, z0.d, #0x6
 # CHECK-NEXT:  1      4     1.00                        orr z0.d, z0.d, #0xfffffffffffffff9
 # CHECK-NEXT:  1      4     1.00                        orr z0.s, z0.s, #0x6
 # CHECK-NEXT:  1      4     1.00                        orr z0.s, z0.s, #0xfffffff9
 # CHECK-NEXT:  1      4     0.50                        orr z23.d, z13.d, z8.d
 # CHECK-NEXT:  1      4     1.00                        orr z23.h, z23.h, #0x6
 # CHECK-NEXT:  1      4     1.00                        orr z23.h, z23.h, #0xfff9
 # CHECK-NEXT:  1      4     0.50                        orr z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  1      4     0.50                        orr z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  1      4     0.50                        orr z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        orr z31.s, p7/m, z31.s, z31.s
 # CHECK-NEXT:  1      4     1.00                        orr z5.b, z5.b, #0x6
 # CHECK-NEXT:  1      4     1.00                        orr z5.b, z5.b, #0xf9
-# CHECK-NEXT:  1      3     1.00                  U     orrs  p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     1.00                        orrs  p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  10     46    5.00                        orv b0, p7, z31.b
 # CHECK-NEXT:  7      34    3.50                        orv d0, p7, z31.d
 # CHECK-NEXT:  9      42    4.50                        orv h0, p7, z31.h
 # CHECK-NEXT:  8      38    4.00                        orv s0, p7, z31.s
 # CHECK-NEXT:  1      3     1.00                        pfalse  p15.b
 # CHECK-NEXT:  1      3     1.00                        pfirst  p0.b, p15, p0.b
 # CHECK-NEXT:  1      3     1.00                        pfirst  p15.b, p15, p15.b
 # CHECK-NEXT:  1      3     1.00                        pnext p0.b, p15, p0.b
 # CHECK-NEXT:  1      3     1.00                        pnext p0.d, p15, p0.d
 # CHECK-NEXT:  1      3     1.00                        pnext p0.h, p15, p0.h
 # CHECK-NEXT:  1      3     1.00                        pnext p0.s, p15, p0.s
 # CHECK-NEXT:  1      3     1.00                        pnext p15.b, p15, p15.b
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  #14, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  #15, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  #6, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  #7, p0, [x0]
-# CHECK-NEXT:  1      1     2.00    *      *            prfb  #7, p3, [z13.s, #31]
-# CHECK-NEXT:  1      1     2.00    *      *            prfb  #7, p3, [z13.s]
+# CHECK-NEXT:  1      1     2.00    *      *      U     prfb  #7, p3, [z13.s, #31]
+# CHECK-NEXT:  1      1     2.00    *      *      U     prfb  #7, p3, [z13.s]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfb  pldl1keep, p0, [x0, z0.d, uxtw]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfb  pldl1keep, p0, [x0, z0.d]
 # CHECK-NEXT:  1      1     2.00    *      *      U     prfb  pldl1keep, p0, [x0, z0.s, uxtw]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pldl1keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pldl1strm, p0, [x0, #-32, mul vl]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pldl1strm, p0, [x0, #31, mul vl]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pldl1strm, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pldl2keep, p0, [x0]

 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pstl2keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pstl2strm, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pstl3keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfb  pstl3strm, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  #14, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  #15, p0, [x0]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfd  #15, p7, [z31.d, #248]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfd  #15, p7, [z31.d]
-# CHECK-NEXT:  1      1     2.00    *      *            prfd  #15, p7, [z31.s, #248]
-# CHECK-NEXT:  1      1     2.00    *      *            prfd  #15, p7, [z31.s]
+# CHECK-NEXT:  1      1     2.00    *      *      U     prfd  #15, p7, [z31.s, #248]
+# CHECK-NEXT:  1      1     2.00    *      *      U     prfd  #15, p7, [z31.s]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  #6, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  #7, p0, [x0]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfd  pldl1keep, p0, [x0, z0.d, lsl #3]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfd  pldl1keep, p0, [x0, z0.d, sxtw #3]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfd  pldl1keep, p0, [x0, z0.d, uxtw #3]
 # CHECK-NEXT:  1      1     2.00    *      *      U     prfd  pldl1keep, p0, [x0, z0.s, sxtw #3]
 # CHECK-NEXT:  1      1     2.00    *      *      U     prfd  pldl1keep, p0, [x0, z0.s, uxtw #3]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  pldl1keep, p0, [x0]

 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  pstl2keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  pstl2strm, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  pstl3keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfd  pstl3strm, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  #14, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  #15, p0, [x0]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfh  #15, p7, [z31.d, #62]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfh  #15, p7, [z31.d]
-# CHECK-NEXT:  1      1     2.00    *      *            prfh  #15, p7, [z31.s, #62]
-# CHECK-NEXT:  1      1     2.00    *      *            prfh  #15, p7, [z31.s]
+# CHECK-NEXT:  1      1     2.00    *      *      U     prfh  #15, p7, [z31.s, #62]
+# CHECK-NEXT:  1      1     2.00    *      *      U     prfh  #15, p7, [z31.s]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  #6, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  #7, p0, [x0]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfh  pldl1keep, p0, [x0, z0.d, lsl #1]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pldl1keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pldl1strm, p0, [x0, #-32, mul vl]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pldl1strm, p0, [x0, #31, mul vl]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pldl1strm, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pldl2keep, p0, [x0]

 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pstl2keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pstl2strm, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pstl3keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfh  pstl3strm, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfw  #14, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfw  #15, p0, [x0]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfw  #15, p7, [z31.d, #124]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfw  #15, p7, [z31.d]
-# CHECK-NEXT:  1      1     2.00    *      *            prfw  #15, p7, [z31.s, #124]
-# CHECK-NEXT:  1      1     2.00    *      *            prfw  #15, p7, [z31.s]
+# CHECK-NEXT:  1      1     2.00    *      *      U     prfw  #15, p7, [z31.s, #124]
+# CHECK-NEXT:  1      1     2.00    *      *      U     prfw  #15, p7, [z31.s]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfw  #6, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfw  #7, p0, [x0]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfw  #7, p3, [x13, z8.d, uxtw #2]
 # CHECK-NEXT:  1      1     1.00    *      *      U     prfw  pldl1keep, p0, [x0, z0.d, sxtw #2]
 # CHECK-NEXT:  1      1     2.00    *      *      U     prfw  pldl1keep, p0, [x0, z0.s, uxtw #2]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfw  pldl1keep, p0, [x0]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfw  pldl1strm, p0, [x0, #-32, mul vl]
 # CHECK-NEXT:  1      1     0.50    *      *      U     prfw  pldl1strm, p0, [x0, #31, mul vl]

 # CHECK-NEXT:  1      3     1.00                        ptrue p7.s, vl3
 # CHECK-NEXT:  1      3     1.00                        ptrue p7.s, vl32
 # CHECK-NEXT:  1      3     1.00                        ptrue p7.s, vl4
 # CHECK-NEXT:  1      3     1.00                        ptrue p7.s, vl5
 # CHECK-NEXT:  1      3     1.00                        ptrue p7.s, vl6
 # CHECK-NEXT:  1      3     1.00                        ptrue p7.s, vl64
 # CHECK-NEXT:  1      3     1.00                        ptrue p7.s, vl7
 # CHECK-NEXT:  1      3     1.00                        ptrue p7.s, vl8
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p0.b, pow2
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p0.d, pow2
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p0.h, pow2
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p0.s, pow2
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p15.b
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p15.d
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p15.h
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p15.s
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #14
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #15
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #16
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #17
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #18
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #19
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #20
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #21
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #22
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #23
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #24
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #25
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #26
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #27
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, #28
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, mul3
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, mul4
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl1
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl128
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl16
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl2
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl256
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl3
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl32
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl4
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl5
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl6
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl64
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl7
-# CHECK-NEXT:  1      3     1.00                  U     ptrues  p7.s, vl8
+# CHECK-NEXT:  1      3     1.00                        ptrues  p0.b, pow2
+# CHECK-NEXT:  1      3     1.00                        ptrues  p0.d, pow2
+# CHECK-NEXT:  1      3     1.00                        ptrues  p0.h, pow2
+# CHECK-NEXT:  1      3     1.00                        ptrues  p0.s, pow2
+# CHECK-NEXT:  1      3     1.00                        ptrues  p15.b
+# CHECK-NEXT:  1      3     1.00                        ptrues  p15.d
+# CHECK-NEXT:  1      3     1.00                        ptrues  p15.h
+# CHECK-NEXT:  1      3     1.00                        ptrues  p15.s
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #14
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #15
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #16
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #17
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #18
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #19
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #20
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #21
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #22
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #23
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #24
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #25
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #26
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #27
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, #28
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, mul3
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, mul4
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl1
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl128
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl16
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl2
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl256
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl3
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl32
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl4
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl5
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl6
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl64
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl7
+# CHECK-NEXT:  1      3     1.00                        ptrues  p7.s, vl8
 # CHECK-NEXT:  1      3     1.00                        punpkhi p0.h, p0.b
 # CHECK-NEXT:  1      3     1.00                        punpkhi p15.h, p15.b
 # CHECK-NEXT:  1      3     1.00                        punpklo p0.h, p0.b
 # CHECK-NEXT:  1      3     1.00                        punpklo p15.h, p15.b
 # CHECK-NEXT:  1      4     0.50                        rbit  z0.b, p7/m, z31.b
 # CHECK-NEXT:  1      4     0.50                        rbit  z0.d, p7/m, z31.d
 # CHECK-NEXT:  1      4     0.50                        rbit  z0.h, p7/m, z31.h
 # CHECK-NEXT:  1      4     0.50                        rbit  z0.s, p7/m, z31.s

 # CHECK-NEXT:  1      4     0.50                        sqadd z31.d, z31.d, #65280
 # CHECK-NEXT:  1      4     0.50                        sqadd z31.h, z31.h, #65280
 # CHECK-NEXT:  1      4     0.50                        sqadd z31.s, z31.s, #65280
 # CHECK-NEXT:  1      2     1.00                        sqdecb  x0
 # CHECK-NEXT:  1      2     1.00                        sqdecb  x0, #14
 # CHECK-NEXT:  1      2     1.00                        sqdecb  x0, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdecb  x0, pow2
 # CHECK-NEXT:  1      2     1.00                        sqdecb  x0, vl1
-# CHECK-NEXT:  1      2     1.00                  U     sqdecb  x0, w0
-# CHECK-NEXT:  1      2     1.00                  U     sqdecb  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     1.00                  U     sqdecb  x0, w0, pow2
-# CHECK-NEXT:  1      2     1.00                  U     sqdecb  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqdecb  x0, w0
+# CHECK-NEXT:  1      2     1.00                        sqdecb  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqdecb  x0, w0, pow2
+# CHECK-NEXT:  1      2     1.00                        sqdecb  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdecd  x0
 # CHECK-NEXT:  1      2     1.00                        sqdecd  x0, #14
 # CHECK-NEXT:  1      2     1.00                        sqdecd  x0, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdecd  x0, pow2
 # CHECK-NEXT:  1      2     1.00                        sqdecd  x0, vl1
-# CHECK-NEXT:  1      2     1.00                  U     sqdecd  x0, w0
-# CHECK-NEXT:  1      2     1.00                  U     sqdecd  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     1.00                  U     sqdecd  x0, w0, pow2
-# CHECK-NEXT:  1      2     1.00                  U     sqdecd  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqdecd  x0, w0
+# CHECK-NEXT:  1      2     1.00                        sqdecd  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqdecd  x0, w0, pow2
+# CHECK-NEXT:  1      2     1.00                        sqdecd  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqdecd  z0.d
 # CHECK-NEXT:  1      4     0.50                        sqdecd  z0.d, all, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqdecd  z0.d, pow2
 # CHECK-NEXT:  1      4     0.50                        sqdecd  z0.d, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdech  x0
 # CHECK-NEXT:  1      2     1.00                        sqdech  x0, #14
 # CHECK-NEXT:  1      2     1.00                        sqdech  x0, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdech  x0, pow2
 # CHECK-NEXT:  1      2     1.00                        sqdech  x0, vl1
-# CHECK-NEXT:  1      2     1.00                  U     sqdech  x0, w0
-# CHECK-NEXT:  1      2     1.00                  U     sqdech  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     1.00                  U     sqdech  x0, w0, pow2
-# CHECK-NEXT:  1      2     1.00                  U     sqdech  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqdech  x0, w0
+# CHECK-NEXT:  1      2     1.00                        sqdech  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqdech  x0, w0, pow2
+# CHECK-NEXT:  1      2     1.00                        sqdech  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqdech  z0.h
 # CHECK-NEXT:  1      4     0.50                        sqdech  z0.h, all, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqdech  z0.h, pow2
 # CHECK-NEXT:  1      4     0.50                        sqdech  z0.h, pow2, mul #16
 # CHECK-NEXT:  2      6     1.50                        sqdecp  x0, p0.b
 # CHECK-NEXT:  2      6     1.50                        sqdecp  x0, p0.d
 # CHECK-NEXT:  2      6     1.50                        sqdecp  x0, p0.h
 # CHECK-NEXT:  2      6     1.50                        sqdecp  x0, p0.s
-# CHECK-NEXT:  2      6     1.50                  U     sqdecp  xzr, p15.b, wzr
-# CHECK-NEXT:  2      6     1.50                  U     sqdecp  xzr, p15.d, wzr
-# CHECK-NEXT:  2      6     1.50                  U     sqdecp  xzr, p15.h, wzr
-# CHECK-NEXT:  2      6     1.50                  U     sqdecp  xzr, p15.s, wzr
+# CHECK-NEXT:  2      6     1.50                        sqdecp  xzr, p15.b, wzr
+# CHECK-NEXT:  2      6     1.50                        sqdecp  xzr, p15.d, wzr
+# CHECK-NEXT:  2      6     1.50                        sqdecp  xzr, p15.h, wzr
+# CHECK-NEXT:  2      6     1.50                        sqdecp  xzr, p15.s, wzr
 # CHECK-NEXT:  1      12    1.00                        sqdecp  z0.d, p0.d
 # CHECK-NEXT:  1      12    1.00                        sqdecp  z0.h, p0.h
 # CHECK-NEXT:  1      12    1.00                        sqdecp  z0.s, p0.s
 # CHECK-NEXT:  1      2     1.00                        sqdecw  x0
 # CHECK-NEXT:  1      2     1.00                        sqdecw  x0, #14
 # CHECK-NEXT:  1      2     1.00                        sqdecw  x0, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdecw  x0, pow2
 # CHECK-NEXT:  1      2     1.00                        sqdecw  x0, vl1
-# CHECK-NEXT:  1      2     1.00                  U     sqdecw  x0, w0
-# CHECK-NEXT:  1      2     1.00                  U     sqdecw  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     1.00                  U     sqdecw  x0, w0, pow2
-# CHECK-NEXT:  1      2     1.00                  U     sqdecw  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqdecw  x0, w0
+# CHECK-NEXT:  1      2     1.00                        sqdecw  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqdecw  x0, w0, pow2
+# CHECK-NEXT:  1      2     1.00                        sqdecw  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqdecw  z0.s
 # CHECK-NEXT:  1      4     0.50                        sqdecw  z0.s, all, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqdecw  z0.s, pow2
 # CHECK-NEXT:  1      4     0.50                        sqdecw  z0.s, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincb  x0
 # CHECK-NEXT:  1      2     1.00                        sqincb  x0, #14
 # CHECK-NEXT:  1      2     1.00                        sqincb  x0, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincb  x0, pow2
 # CHECK-NEXT:  1      2     1.00                        sqincb  x0, vl1
-# CHECK-NEXT:  1      2     1.00                  U     sqincb  x0, w0
-# CHECK-NEXT:  1      2     1.00                  U     sqincb  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     1.00                  U     sqincb  x0, w0, pow2
-# CHECK-NEXT:  1      2     1.00                  U     sqincb  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqincb  x0, w0
+# CHECK-NEXT:  1      2     1.00                        sqincb  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqincb  x0, w0, pow2
+# CHECK-NEXT:  1      2     1.00                        sqincb  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincd  x0
 # CHECK-NEXT:  1      2     1.00                        sqincd  x0, #14
 # CHECK-NEXT:  1      2     1.00                        sqincd  x0, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincd  x0, pow2
 # CHECK-NEXT:  1      2     1.00                        sqincd  x0, vl1
-# CHECK-NEXT:  1      2     1.00                  U     sqincd  x0, w0
-# CHECK-NEXT:  1      2     1.00                  U     sqincd  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     1.00                  U     sqincd  x0, w0, pow2
-# CHECK-NEXT:  1      2     1.00                  U     sqincd  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqincd  x0, w0
+# CHECK-NEXT:  1      2     1.00                        sqincd  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqincd  x0, w0, pow2
+# CHECK-NEXT:  1      2     1.00                        sqincd  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqincd  z0.d
 # CHECK-NEXT:  1      4     0.50                        sqincd  z0.d, all, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqincd  z0.d, pow2
 # CHECK-NEXT:  1      4     0.50                        sqincd  z0.d, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqinch  x0
 # CHECK-NEXT:  1      2     1.00                        sqinch  x0, #14
 # CHECK-NEXT:  1      2     1.00                        sqinch  x0, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqinch  x0, pow2
 # CHECK-NEXT:  1      2     1.00                        sqinch  x0, vl1
-# CHECK-NEXT:  1      2     1.00                  U     sqinch  x0, w0
-# CHECK-NEXT:  1      2     1.00                  U     sqinch  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     1.00                  U     sqinch  x0, w0, pow2
-# CHECK-NEXT:  1      2     1.00                  U     sqinch  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqinch  x0, w0
+# CHECK-NEXT:  1      2     1.00                        sqinch  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqinch  x0, w0, pow2
+# CHECK-NEXT:  1      2     1.00                        sqinch  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqinch  z0.h
 # CHECK-NEXT:  1      4     0.50                        sqinch  z0.h, all, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqinch  z0.h, pow2
 # CHECK-NEXT:  1      4     0.50                        sqinch  z0.h, pow2, mul #16
 # CHECK-NEXT:  2      6     1.50                        sqincp  x0, p0.b
 # CHECK-NEXT:  2      6     1.50                        sqincp  x0, p0.d
 # CHECK-NEXT:  2      6     1.50                        sqincp  x0, p0.h
 # CHECK-NEXT:  2      6     1.50                        sqincp  x0, p0.s
-# CHECK-NEXT:  2      6     1.50                  U     sqincp  xzr, p15.b, wzr
-# CHECK-NEXT:  2      6     1.50                  U     sqincp  xzr, p15.d, wzr
-# CHECK-NEXT:  2      6     1.50                  U     sqincp  xzr, p15.h, wzr
-# CHECK-NEXT:  2      6     1.50                  U     sqincp  xzr, p15.s, wzr
+# CHECK-NEXT:  2      6     1.50                        sqincp  xzr, p15.b, wzr
+# CHECK-NEXT:  2      6     1.50                        sqincp  xzr, p15.d, wzr
+# CHECK-NEXT:  2      6     1.50                        sqincp  xzr, p15.h, wzr
+# CHECK-NEXT:  2      6     1.50                        sqincp  xzr, p15.s, wzr
 # CHECK-NEXT:  1      12    1.00                        sqincp  z0.d, p0.d
 # CHECK-NEXT:  1      12    1.00                        sqincp  z0.h, p0.h
 # CHECK-NEXT:  1      12    1.00                        sqincp  z0.s, p0.s
 # CHECK-NEXT:  1      2     1.00                        sqincw  x0
 # CHECK-NEXT:  1      2     1.00                        sqincw  x0, #14
 # CHECK-NEXT:  1      2     1.00                        sqincw  x0, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincw  x0, pow2
 # CHECK-NEXT:  1      2     1.00                        sqincw  x0, vl1
-# CHECK-NEXT:  1      2     1.00                  U     sqincw  x0, w0
-# CHECK-NEXT:  1      2     1.00                  U     sqincw  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     1.00                  U     sqincw  x0, w0, pow2
-# CHECK-NEXT:  1      2     1.00                  U     sqincw  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqincw  x0, w0
+# CHECK-NEXT:  1      2     1.00                        sqincw  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     1.00                        sqincw  x0, w0, pow2
+# CHECK-NEXT:  1      2     1.00                        sqincw  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqincw  z0.s
 # CHECK-NEXT:  1      4     0.50                        sqincw  z0.s, all, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqincw  z0.s, pow2
 # CHECK-NEXT:  1      4     0.50                        sqincw  z0.s, pow2, mul #16
 # CHECK-NEXT:  1      4     0.50                        sqsub z0.b, z0.b, #0
 # CHECK-NEXT:  1      4     0.50                        sqsub z0.b, z0.b, z0.b
 # CHECK-NEXT:  1      4     0.50                        sqsub z0.d, z0.d, #0
 # CHECK-NEXT:  1      4     0.50                        sqsub z0.d, z0.d, #0, lsl #8

 # CHECK-NEXT:  1      11    1.00           *            st1w  { z0.s }, p0, [x0]
 # CHECK-NEXT:  8      16    12.00          *            st1w  { z0.s }, p7, [z0.s]
 # CHECK-NEXT:  1      11    1.00           *            st1w  { z21.d }, p5, [x10, #5, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            st1w  { z21.s }, p5, [x10, #5, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            st1w  { z31.d }, p7, [sp, #-1, mul vl]
 # CHECK-NEXT:  4      16    4.00           *            st1w  { z31.d }, p7, [z31.d, #124]
 # CHECK-NEXT:  1      11    1.00           *            st1w  { z31.s }, p7, [sp, #-1, mul vl]
 # CHECK-NEXT:  8      16    12.00          *            st1w  { z31.s }, p7, [z31.s, #124]
-# CHECK-NEXT:  3      12    8.00           *      U     st2b  { z0.b, z1.b }, p0, [x0, x0]
-# CHECK-NEXT:  3      12    8.00           *      U     st2b  { z0.b, z1.b }, p0, [x0]
-# CHECK-NEXT:  3      12    8.00           *      U     st2b  { z21.b, z22.b }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  3      12    8.00           *      U     st2b  { z23.b, z24.b }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  3      12    8.00           *      U     st2b  { z5.b, z6.b }, p3, [x17, x16]
-# CHECK-NEXT:  2      11    2.00           *      U     st2d  { z0.d, z1.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  3      12    2.00           *      U     st2d  { z0.d, z1.d }, p0, [x0]
-# CHECK-NEXT:  3      12    2.00           *      U     st2d  { z21.d, z22.d }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  3      12    2.00           *      U     st2d  { z23.d, z24.d }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      11    2.00           *      U     st2d  { z5.d, z6.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  3      12    8.00           *      U     st2h  { z0.h, z1.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  3      12    8.00           *      U     st2h  { z0.h, z1.h }, p0, [x0]
-# CHECK-NEXT:  3      12    8.00           *      U     st2h  { z21.h, z22.h }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  3      12    8.00           *      U     st2h  { z23.h, z24.h }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  3      12    8.00           *      U     st2h  { z5.h, z6.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  2      11    2.00           *      U     st2w  { z0.s, z1.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  3      12    2.00           *      U     st2w  { z0.s, z1.s }, p0, [x0]
-# CHECK-NEXT:  3      12    2.00           *      U     st2w  { z21.s, z22.s }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  3      12    2.00           *      U     st2w  { z23.s, z24.s }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      11    2.00           *      U     st2w  { z5.s, z6.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  4      15    12.00          *      U     st3b  { z0.b - z2.b }, p0, [x0, x0]
-# CHECK-NEXT:  4      15    12.00          *      U     st3b  { z0.b - z2.b }, p0, [x0]
-# CHECK-NEXT:  4      15    12.00          *      U     st3b  { z21.b - z23.b }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  4      15    12.00          *      U     st3b  { z23.b - z25.b }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  4      15    12.00          *      U     st3b  { z5.b - z7.b }, p3, [x17, x16]
-# CHECK-NEXT:  3      11    3.00           *      U     st3d  { z0.d - z2.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  4      12    3.00           *      U     st3d  { z0.d - z2.d }, p0, [x0]
-# CHECK-NEXT:  4      12    3.00           *      U     st3d  { z21.d - z23.d }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  4      12    3.00           *      U     st3d  { z23.d - z25.d }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  3      11    3.00           *      U     st3d  { z5.d - z7.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  4      15    12.00          *      U     st3h  { z0.h - z2.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  4      15    12.00          *      U     st3h  { z0.h - z2.h }, p0, [x0]
-# CHECK-NEXT:  4      15    12.00          *      U     st3h  { z21.h - z23.h }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  4      15    12.00          *      U     st3h  { z23.h - z25.h }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  4      15    12.00          *      U     st3h  { z5.h - z7.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  3      11    3.00           *      U     st3w  { z0.s - z2.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  4      12    3.00           *      U     st3w  { z0.s - z2.s }, p0, [x0]
-# CHECK-NEXT:  4      12    3.00           *      U     st3w  { z21.s - z23.s }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  4      12    3.00           *      U     st3w  { z23.s - z25.s }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  3      11    3.00           *      U     st3w  { z5.s - z7.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  5      15    16.00          *      U     st4b  { z0.b - z3.b }, p0, [x0, x0]
-# CHECK-NEXT:  5      15    16.00          *      U     st4b  { z0.b - z3.b }, p0, [x0]
-# CHECK-NEXT:  5      15    16.00          *      U     st4b  { z21.b - z24.b }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  5      15    16.00          *      U     st4b  { z23.b - z26.b }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  5      15    16.00          *      U     st4b  { z5.b - z8.b }, p3, [x17, x16]
-# CHECK-NEXT:  4      11    4.00           *      U     st4d  { z0.d - z3.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  5      12    4.00           *      U     st4d  { z0.d - z3.d }, p0, [x0]
-# CHECK-NEXT:  5      12    4.00           *      U     st4d  { z21.d - z24.d }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  5      12    4.00           *      U     st4d  { z23.d - z26.d }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  4      11    4.00           *      U     st4d  { z5.d - z8.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  5      15    16.00          *      U     st4h  { z0.h - z3.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  5      15    16.00          *      U     st4h  { z0.h - z3.h }, p0, [x0]
-# CHECK-NEXT:  5      15    16.00          *      U     st4h  { z21.h - z24.h }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  5      15    16.00          *      U     st4h  { z23.h - z26.h }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  5      15    16.00          *      U     st4h  { z5.h - z8.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  4      11    4.00           *      U     st4w  { z0.s - z3.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  5      12    4.00           *      U     st4w  { z0.s - z3.s }, p0, [x0]
-# CHECK-NEXT:  5      12    4.00           *      U     st4w  { z21.s - z24.s }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  5      12    4.00           *      U     st4w  { z23.s - z26.s }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  4      11    4.00           *      U     st4w  { z5.s - z8.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  3      12    8.00           *            st2b  { z0.b, z1.b }, p0, [x0, x0]
+# CHECK-NEXT:  3      12    8.00           *            st2b  { z0.b, z1.b }, p0, [x0]
+# CHECK-NEXT:  3      12    8.00           *            st2b  { z21.b, z22.b }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  3      12    8.00           *            st2b  { z23.b, z24.b }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  3      12    8.00           *            st2b  { z5.b, z6.b }, p3, [x17, x16]
+# CHECK-NEXT:  2      11    2.00           *            st2d  { z0.d, z1.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  3      12    2.00           *            st2d  { z0.d, z1.d }, p0, [x0]
+# CHECK-NEXT:  3      12    2.00           *            st2d  { z21.d, z22.d }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  3      12    2.00           *            st2d  { z23.d, z24.d }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      11    2.00           *            st2d  { z5.d, z6.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  3      12    8.00           *            st2h  { z0.h, z1.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  3      12    8.00           *            st2h  { z0.h, z1.h }, p0, [x0]
+# CHECK-NEXT:  3      12    8.00           *            st2h  { z21.h, z22.h }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  3      12    8.00           *            st2h  { z23.h, z24.h }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  3      12    8.00           *            st2h  { z5.h, z6.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  2      11    2.00           *            st2w  { z0.s, z1.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  3      12    2.00           *            st2w  { z0.s, z1.s }, p0, [x0]
+# CHECK-NEXT:  3      12    2.00           *            st2w  { z21.s, z22.s }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  3      12    2.00           *            st2w  { z23.s, z24.s }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      11    2.00           *            st2w  { z5.s, z6.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  4      15    12.00          *            st3b  { z0.b - z2.b }, p0, [x0, x0]
+# CHECK-NEXT:  4      15    12.00          *            st3b  { z0.b - z2.b }, p0, [x0]
+# CHECK-NEXT:  4      15    12.00          *            st3b  { z21.b - z23.b }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  4      15    12.00          *            st3b  { z23.b - z25.b }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  4      15    12.00          *            st3b  { z5.b - z7.b }, p3, [x17, x16]
+# CHECK-NEXT:  3      11    3.00           *            st3d  { z0.d - z2.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  4      12    3.00           *            st3d  { z0.d - z2.d }, p0, [x0]
+# CHECK-NEXT:  4      12    3.00           *            st3d  { z21.d - z23.d }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  4      12    3.00           *            st3d  { z23.d - z25.d }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  3      11    3.00           *            st3d  { z5.d - z7.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  4      15    12.00          *            st3h  { z0.h - z2.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  4      15    12.00          *            st3h  { z0.h - z2.h }, p0, [x0]
+# CHECK-NEXT:  4      15    12.00          *            st3h  { z21.h - z23.h }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  4      15    12.00          *            st3h  { z23.h - z25.h }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  4      15    12.00          *            st3h  { z5.h - z7.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  3      11    3.00           *            st3w  { z0.s - z2.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  4      12    3.00           *            st3w  { z0.s - z2.s }, p0, [x0]
+# CHECK-NEXT:  4      12    3.00           *            st3w  { z21.s - z23.s }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  4      12    3.00           *            st3w  { z23.s - z25.s }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  3      11    3.00           *            st3w  { z5.s - z7.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  5      15    16.00          *            st4b  { z0.b - z3.b }, p0, [x0, x0]
+# CHECK-NEXT:  5      15    16.00          *            st4b  { z0.b - z3.b }, p0, [x0]
+# CHECK-NEXT:  5      15    16.00          *            st4b  { z21.b - z24.b }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  5      15    16.00          *            st4b  { z23.b - z26.b }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  5      15    16.00          *            st4b  { z5.b - z8.b }, p3, [x17, x16]
+# CHECK-NEXT:  4      11    4.00           *            st4d  { z0.d - z3.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  5      12    4.00           *            st4d  { z0.d - z3.d }, p0, [x0]
+# CHECK-NEXT:  5      12    4.00           *            st4d  { z21.d - z24.d }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  5      12    4.00           *            st4d  { z23.d - z26.d }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  4      11    4.00           *            st4d  { z5.d - z8.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  5      15    16.00          *            st4h  { z0.h - z3.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  5      15    16.00          *            st4h  { z0.h - z3.h }, p0, [x0]
+# CHECK-NEXT:  5      15    16.00          *            st4h  { z21.h - z24.h }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  5      15    16.00          *            st4h  { z23.h - z26.h }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  5      15    16.00          *            st4h  { z5.h - z8.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  4      11    4.00           *            st4w  { z0.s - z3.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  5      12    4.00           *            st4w  { z0.s - z3.s }, p0, [x0]
+# CHECK-NEXT:  5      12    4.00           *            st4w  { z21.s - z24.s }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  5      12    4.00           *            st4w  { z23.s - z26.s }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  4      11    4.00           *            st4w  { z5.s - z8.s }, p3, [x17, x16, lsl #2]
 # CHECK-NEXT:  1      11    1.00           *            stnt1b  { z0.b }, p0, [x0, x0]
 # CHECK-NEXT:  1      11    1.00           *            stnt1b  { z0.b }, p0, [x0]
 # CHECK-NEXT:  1      11    1.00           *            stnt1b  { z21.b }, p5, [x10, #7, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            stnt1b  { z23.b }, p3, [x13, #-8, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            stnt1d  { z0.d }, p0, [x0, x0, lsl #3]
 # CHECK-NEXT:  1      11    1.00           *            stnt1d  { z0.d }, p0, [x0]
 # CHECK-NEXT:  1      11    1.00           *            stnt1d  { z21.d }, p5, [x10, #7, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            stnt1d  { z23.d }, p3, [x13, #-8, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            stnt1h  { z0.h }, p0, [x0, x0, lsl #1]
 # CHECK-NEXT:  1      11    1.00           *            stnt1h  { z0.h }, p0, [x0]
 # CHECK-NEXT:  1      11    1.00           *            stnt1h  { z21.h }, p5, [x10, #7, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            stnt1h  { z23.h }, p3, [x13, #-8, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            stnt1w  { z0.s }, p0, [x0, x0, lsl #2]
 # CHECK-NEXT:  1      11    1.00           *            stnt1w  { z0.s }, p0, [x0]
 # CHECK-NEXT:  1      11    1.00           *            stnt1w  { z21.s }, p5, [x10, #7, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            stnt1w  { z23.s }, p3, [x13, #-8, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            str p0, [x0]
 # CHECK-NEXT:  1      11    1.00           *            str p15, [sp, #-256, mul vl]
 # CHECK-NEXT:  1      11    1.00           *            str p5, [x10, #255, mul vl]
-# CHECK-NEXT:  1      11    1.00           *      U     str z0, [x0]
-# CHECK-NEXT:  1      11    1.00           *      U     str z21, [x10, #-256, mul vl]
-# CHECK-NEXT:  1      11    1.00           *      U     str z31, [sp, #255, mul vl]
+# CHECK-NEXT:  1      11    1.00           *            str z0, [x0]
+# CHECK-NEXT:  1      11    1.00           *            str z21, [x10, #-256, mul vl]
+# CHECK-NEXT:  1      11    1.00           *            str z31, [sp, #255, mul vl]
 # CHECK-NEXT:  1      4     0.50                        sub z0.b, p0/m, z0.b, z0.b
 # CHECK-NEXT:  1      4     1.00                        sub z0.b, z0.b, #0
 # CHECK-NEXT:  1      4     0.50                        sub z0.b, z0.b, z0.b
 # CHECK-NEXT:  1      4     0.50                        sub z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  1      4     1.00                        sub z0.d, z0.d, #0
 # CHECK-NEXT:  1      4     1.00                        sub z0.d, z0.d, #0, lsl #8
 # CHECK-NEXT:  1      4     0.50                        sub z0.d, z0.d, z0.d
 # CHECK-NEXT:  1      4     0.50                        sub z0.h, p0/m, z0.h, z0.h

llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s

 # CHECK-NEXT:  1      2     0.50                        addhnb  z0.s, z1.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        addhnt  z0.b, z1.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        addhnt  z0.h, z1.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        addhnt  z0.s, z1.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        addp  z0.b, p0/m, z0.b, z1.b
 # CHECK-NEXT:  1      2     0.50                        addp  z0.h, p0/m, z0.h, z1.h
 # CHECK-NEXT:  1      2     0.50                        addp  z29.s, p7/m, z29.s, z30.s
 # CHECK-NEXT:  1      2     0.50                        addp  z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  1      2     0.50                  U     addpl sp, sp, #31
-# CHECK-NEXT:  1      2     0.50                  U     addpl x0, x0, #-32
-# CHECK-NEXT:  1      2     0.50                  U     addpl x21, x21, #0
-# CHECK-NEXT:  1      2     0.50                  U     addpl x23, x8, #-1
+# CHECK-NEXT:  1      2     0.50                        addpl sp, sp, #31
+# CHECK-NEXT:  1      2     0.50                        addpl x0, x0, #-32
+# CHECK-NEXT:  1      2     0.50                        addpl x21, x21, #0
+# CHECK-NEXT:  1      2     0.50                        addpl x23, x8, #-1
 # CHECK-NEXT:  1      2     0.50                        addvl sp, sp, #31
 # CHECK-NEXT:  1      2     0.50                        addvl x0, x0, #-32
 # CHECK-NEXT:  1      2     0.50                        addvl x21, x21, #0
 # CHECK-NEXT:  1      2     0.50                        addvl x23, x8, #-1
 # CHECK-NEXT:  1      2     0.50                        adr z0.d, [z0.d, z0.d, lsl #1]
 # CHECK-NEXT:  1      2     0.50                        adr z0.d, [z0.d, z0.d, lsl #2]
 # CHECK-NEXT:  1      2     0.50                        adr z0.d, [z0.d, z0.d, lsl #3]
 # CHECK-NEXT:  1      2     0.50                        adr z0.d, [z0.d, z0.d, sxtw #1]

 # CHECK-NEXT:  1      2     0.50                        and z23.h, z23.h, #0x6
 # CHECK-NEXT:  1      2     0.50                        and z23.h, z23.h, #0xfff9
 # CHECK-NEXT:  1      2     0.50                        and z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  1      2     0.50                        and z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        and z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        and z31.s, p7/m, z31.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        and z5.b, z5.b, #0x6
 # CHECK-NEXT:  1      2     0.50                        and z5.b, z5.b, #0xf9
-# CHECK-NEXT:  2      2     1.00                  U     ands  p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  2      2     1.00                        ands  p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  2      6     1.00                        andv  b0, p7, z31.b
 # CHECK-NEXT:  2      6     1.00                        andv  d0, p7, z31.d
 # CHECK-NEXT:  2      6     1.00                        andv  h0, p7, z31.h
 # CHECK-NEXT:  2      6     1.00                        andv  s0, p7, z31.s
 # CHECK-NEXT:  1      2     1.00                        asr z0.b, p0/m, z0.b, #1
 # CHECK-NEXT:  1      2     1.00                        asr z0.b, p0/m, z0.b, z0.b
 # CHECK-NEXT:  1      2     1.00                        asr z0.b, p0/m, z0.b, z1.d
 # CHECK-NEXT:  1      2     1.00                        asr z0.b, z0.b, #1

 # CHECK-NEXT:  1      4     1.00                        asrd  z0.b, p0/m, z0.b, #1
 # CHECK-NEXT:  1      4     1.00                        asrd  z0.d, p0/m, z0.d, #1
 # CHECK-NEXT:  1      4     1.00                        asrd  z0.h, p0/m, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        asrd  z0.s, p0/m, z0.s, #1
 # CHECK-NEXT:  1      4     1.00                        asrd  z31.b, p0/m, z31.b, #8
 # CHECK-NEXT:  1      4     1.00                        asrd  z31.d, p0/m, z31.d, #64
 # CHECK-NEXT:  1      4     1.00                        asrd  z31.h, p0/m, z31.h, #16
 # CHECK-NEXT:  1      4     1.00                        asrd  z31.s, p0/m, z31.s, #32
-# CHECK-NEXT:  1      2     1.00                  U     asrr  z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  1      2     1.00                  U     asrr  z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  1      2     1.00                  U     asrr  z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  1      2     1.00                  U     asrr  z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      2     1.00                        asrr  z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      2     1.00                        asrr  z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      2     1.00                        asrr  z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      2     1.00                        asrr  z0.s, p0/m, z0.s, z0.s
 # CHECK-NEXT:  1      2     1.00                        bcax  z29.d, z29.d, z30.d, z31.d
 # CHECK-NEXT:  2      6     2.00                        bdep  z0.b, z1.b, z31.b
 # CHECK-NEXT:  2      6     2.00                        bdep  z0.d, z1.d, z31.d
 # CHECK-NEXT:  2      6     2.00                        bdep  z0.h, z1.h, z31.h
 # CHECK-NEXT:  2      6     2.00                        bdep  z0.s, z1.s, z31.s
 # CHECK-NEXT:  2      6     2.00                        bext  z0.b, z1.b, z31.b
 # CHECK-NEXT:  2      6     2.00                        bext  z0.d, z1.d, z31.d
 # CHECK-NEXT:  2      6     2.00                        bext  z0.h, z1.h, z31.h
 # CHECK-NEXT:  2      6     2.00                        bext  z0.s, z1.s, z31.s
-# CHECK-NEXT:  1      3     1.00                  U     bfcvt z0.h, p0/m, z1.s
-# CHECK-NEXT:  1      3     1.00                  U     bfcvtnt z0.h, p0/m, z1.s
+# CHECK-NEXT:  1      3     1.00                        bfcvt z0.h, p0/m, z1.s
+# CHECK-NEXT:  1      3     1.00                        bfcvtnt z0.h, p0/m, z1.s
 # CHECK-NEXT:  1      4     0.50                        bfdot z0.s, z1.h, z2.h
 # CHECK-NEXT:  1      4     0.50                        bfdot z0.s, z1.h, z2.h[0]
 # CHECK-NEXT:  1      4     0.50                        bfdot z0.s, z1.h, z2.h[3]
 # CHECK-NEXT:  1      4     0.50                        bfmlalb z0.s, z1.h, z2.h
 # CHECK-NEXT:  1      4     0.50                        bfmlalb z0.s, z1.h, z2.h[0]
 # CHECK-NEXT:  1      4     0.50                        bfmlalb z0.s, z1.h, z2.h[7]
 # CHECK-NEXT:  1      4     0.50                        bfmlalb z10.s, z21.h, z14.h
 # CHECK-NEXT:  1      4     0.50                        bfmlalb z21.s, z14.h, z3.h[2]

 # CHECK-NEXT:  1      1     1.00                        bic p0.b, p0/z, p0.b, p0.b
 # CHECK-NEXT:  1      1     1.00                        bic p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      2     0.50                        bic z0.d, z0.d, z0.d
 # CHECK-NEXT:  1      2     0.50                        bic z23.d, z13.d, z8.d
 # CHECK-NEXT:  1      2     0.50                        bic z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  1      2     0.50                        bic z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        bic z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        bic z31.s, p7/m, z31.s, z31.s
-# CHECK-NEXT:  2      2     1.00                  U     bics  p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  2      2     1.00                  U     bics  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  2      2     1.00                        bics  p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  2      2     1.00                        bics  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      2     0.50                        brka  p0.b, p15/m, p15.b
 # CHECK-NEXT:  1      2     0.50                        brka  p0.b, p15/z, p15.b
-# CHECK-NEXT:  1      3     0.50                  U     brkas p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     0.50                        brkas p0.b, p15/z, p15.b
 # CHECK-NEXT:  1      2     0.50                        brkb  p0.b, p15/m, p15.b
 # CHECK-NEXT:  1      2     0.50                        brkb  p0.b, p15/z, p15.b
-# CHECK-NEXT:  1      3     0.50                  U     brkbs p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     0.50                        brkbs p0.b, p15/z, p15.b
 # CHECK-NEXT:  1      2     2.00                        brkn  p0.b, p15/z, p1.b, p0.b
 # CHECK-NEXT:  1      2     2.00                        brkn  p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  2      3     1.00                  U     brkns p0.b, p15/z, p1.b, p0.b
-# CHECK-NEXT:  2      3     1.00                  U     brkns p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  2      3     1.00                        brkns p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  2      3     1.00                        brkns p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      2     2.00                        brkpa p0.b, p15/z, p1.b, p2.b
 # CHECK-NEXT:  1      2     2.00                        brkpa p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  2      3     1.00                  U     brkpas  p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  2      3     1.00                  U     brkpas  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  2      3     1.00                        brkpas  p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  2      3     1.00                        brkpas  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      2     2.00                        brkpb p0.b, p15/z, p1.b, p2.b
 # CHECK-NEXT:  1      2     2.00                        brkpb p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  2      3     1.00                  U     brkpbs  p0.b, p15/z, p1.b, p2.b
-# CHECK-NEXT:  2      3     1.00                  U     brkpbs  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  2      3     1.00                        brkpbs  p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  2      3     1.00                        brkpbs  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      2     0.50                        bsl z0.d, z0.d, z1.d, z2.d
 # CHECK-NEXT:  1      2     0.50                        bsl1n z0.d, z0.d, z1.d, z2.d
 # CHECK-NEXT:  1      2     0.50                        bsl2n z0.d, z0.d, z1.d, z2.d
 # CHECK-NEXT:  1      2     0.50                        cadd  z0.b, z0.b, z0.b, #90
 # CHECK-NEXT:  1      2     0.50                        cadd  z0.d, z0.d, z0.d, #90
 # CHECK-NEXT:  1      2     0.50                        cadd  z0.h, z0.h, z0.h, #90
 # CHECK-NEXT:  1      2     0.50                        cadd  z0.s, z0.s, z0.s, #90
 # CHECK-NEXT:  1      2     0.50                        cadd  z31.b, z31.b, z31.b, #270
 # CHECK-NEXT:  1      2     0.50                        cadd  z31.d, z31.d, z31.d, #270
 # CHECK-NEXT:  1      2     0.50                        cadd  z31.h, z31.h, z31.h, #270
 # CHECK-NEXT:  1      2     0.50                        cadd  z31.s, z31.s, z31.s, #270
 # CHECK-NEXT:  1      4     1.00                        cdot  z0.d, z1.h, z15.h[1], #0
 # CHECK-NEXT:  1      4     1.00                        cdot  z0.d, z1.h, z31.h, #0
 # CHECK-NEXT:  1      4     1.00                        cdot  z0.d, z1.h, z31.h, #180
 # CHECK-NEXT:  1      4     1.00                        cdot  z0.d, z1.h, z31.h, #270
 # CHECK-NEXT:  1      4     1.00                        cdot  z0.d, z1.h, z31.h, #90
 # CHECK-NEXT:  1      3     0.50                        cdot  z0.s, z1.b, z31.b, #0
 # CHECK-NEXT:  1      3     0.50                        cdot  z0.s, z1.b, z7.b[3], #0
 # CHECK-NEXT:  1      4     1.00                        cdot  z29.d, z30.h, z0.h[0], #180
 # CHECK-NEXT:  1      4     1.00                        cdot  z31.d, z30.h, z7.h[1], #270
 # CHECK-NEXT:  1      4     1.00                        cdot  z5.d, z6.h, z3.h[0], #90
-# CHECK-NEXT:  1      3     1.00                  U     clasta  b0, p7, b0, z31.b
+# CHECK-NEXT:  1      3     1.00                        clasta  b0, p7, b0, z31.b
 # CHECK-NEXT:  1      3     1.00                        clasta  d0, p7, d0, z31.d
 # CHECK-NEXT:  1      3     1.00                        clasta  h0, p7, h0, z31.h
 # CHECK-NEXT:  1      3     1.00                        clasta  s0, p7, s0, z31.s
 # CHECK-NEXT:  3      8     1.00                        clasta  w0, p7, w0, z31.b
 # CHECK-NEXT:  3      8     1.00                        clasta  w0, p7, w0, z31.h
 # CHECK-NEXT:  3      8     1.00                        clasta  w0, p7, w0, z31.s
 # CHECK-NEXT:  3      8     1.00                        clasta  x0, p7, x0, z31.d
 # CHECK-NEXT:  1      3     1.00                        clasta  z0.b, p7, z0.b, z31.b
 # CHECK-NEXT:  1      3     1.00                        clasta  z0.d, p7, z0.d, z31.d
 # CHECK-NEXT:  1      3     1.00                        clasta  z0.h, p7, z0.h, z31.h
 # CHECK-NEXT:  1      3     1.00                        clasta  z0.s, p7, z0.s, z31.s
-# CHECK-NEXT:  1      3     1.00                  U     clastb  b0, p7, b0, z31.b
+# CHECK-NEXT:  1      3     1.00                        clastb  b0, p7, b0, z31.b
 # CHECK-NEXT:  1      3     1.00                        clastb  d0, p7, d0, z31.d
 # CHECK-NEXT:  1      3     1.00                        clastb  h0, p7, h0, z31.h
 # CHECK-NEXT:  1      3     1.00                        clastb  s0, p7, s0, z31.s
 # CHECK-NEXT:  3      8     1.00                        clastb  w0, p7, w0, z31.b
 # CHECK-NEXT:  3      8     1.00                        clastb  w0, p7, w0, z31.h
 # CHECK-NEXT:  3      8     1.00                        clastb  w0, p7, w0, z31.s
 # CHECK-NEXT:  3      8     1.00                        clastb  x0, p7, x0, z31.d
 # CHECK-NEXT:  1      3     1.00                        clastb  z0.b, p7, z0.b, z31.b

 # CHECK-NEXT:  1      2     0.50                        cntp  x0, p15, p0.h
 # CHECK-NEXT:  1      2     0.50                        cntp  x0, p15, p0.s
 # CHECK-NEXT:  1      2     0.50                        cntw  x0
 # CHECK-NEXT:  1      2     0.50                        cntw  x0, #28
 # CHECK-NEXT:  1      2     0.50                        cntw  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        cntw  x0, pow2
 # CHECK-NEXT:  1      3     1.00                        compact z31.d, p7, z31.d
 # CHECK-NEXT:  1      3     1.00                        compact z31.s, p7, z31.s
-# CHECK-NEXT:  1      1     0.50                  U     ctermeq w30, wzr
-# CHECK-NEXT:  1      1     0.50                  U     ctermeq wzr, w30
-# CHECK-NEXT:  1      1     0.50                  U     ctermeq x30, xzr
-# CHECK-NEXT:  1      1     0.50                  U     ctermeq xzr, x30
-# CHECK-NEXT:  1      1     0.50                  U     ctermne w30, wzr
-# CHECK-NEXT:  1      1     0.50                  U     ctermne wzr, w30
-# CHECK-NEXT:  1      1     0.50                  U     ctermne x30, xzr
-# CHECK-NEXT:  1      1     0.50                  U     ctermne xzr, x30
+# CHECK-NEXT:  1      1     0.50                        ctermeq w30, wzr
+# CHECK-NEXT:  1      1     0.50                        ctermeq wzr, w30
+# CHECK-NEXT:  1      1     0.50                        ctermeq x30, xzr
+# CHECK-NEXT:  1      1     0.50                        ctermeq xzr, x30
+# CHECK-NEXT:  1      1     0.50                        ctermne w30, wzr
+# CHECK-NEXT:  1      1     0.50                        ctermne wzr, w30
+# CHECK-NEXT:  1      1     0.50                        ctermne x30, xzr
+# CHECK-NEXT:  1      1     0.50                        ctermne xzr, x30
 # CHECK-NEXT:  1      2     0.50                        decb  x0
 # CHECK-NEXT:  1      2     0.50                        decb  x0, #14
 # CHECK-NEXT:  1      2     0.50                        decb  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        decb  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        decb  x0, vl1
 # CHECK-NEXT:  1      2     0.50                        decd  x0
 # CHECK-NEXT:  1      2     0.50                        decd  x0, #14
 # CHECK-NEXT:  1      2     0.50                        decd  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        decd  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        decd  x0, vl1
 # CHECK-NEXT:  1      2     0.50                        dech  x0
 # CHECK-NEXT:  1      2     0.50                        dech  x0, #14
 # CHECK-NEXT:  1      2     0.50                        dech  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        dech  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        dech  x0, vl1
 # CHECK-NEXT:  1      2     0.50                        decp  x0, p0.b
 # CHECK-NEXT:  1      2     0.50                        decp  x0, p0.d
 # CHECK-NEXT:  1      2     0.50                        decp  x0, p0.h
 # CHECK-NEXT:  1      2     0.50                        decp  x0, p0.s
 # CHECK-NEXT:  1      2     0.50                        decp  xzr, p15.b
 # CHECK-NEXT:  1      2     0.50                        decp  xzr, p15.d
 # CHECK-NEXT:  1      2     0.50                        decp  xzr, p15.h
 # CHECK-NEXT:  1      2     0.50                        decp  xzr, p15.s
-# CHECK-NEXT:  3      7     1.00                  U     decp  z31.d, p15.d
-# CHECK-NEXT:  3      7     1.00                  U     decp  z31.h, p15.h
-# CHECK-NEXT:  3      7     1.00                  U     decp  z31.s, p15.s
+# CHECK-NEXT:  3      7     1.00                        decp  z31.d, p15.d
+# CHECK-NEXT:  3      7     1.00                        decp  z31.h, p15.h
+# CHECK-NEXT:  3      7     1.00                        decp  z31.s, p15.s
 # CHECK-NEXT:  1      2     0.50                        decw  x0
 # CHECK-NEXT:  1      2     0.50                        decw  x0, #14
 # CHECK-NEXT:  1      2     0.50                        decw  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        decw  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        decw  x0, vl1
 # CHECK-NEXT:  1      2     0.50                        dupm  z0.d, #0xfffffffffffffff9
 # CHECK-NEXT:  1      2     0.50                        dupm  z0.s, #0xfffffff9
 # CHECK-NEXT:  1      2     0.50                        dupm  z23.h, #0xfff9

 # CHECK-NEXT:  1      2     0.50                        eor z31.s, p7/m, z31.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        eor z5.b, z5.b, #0x6
 # CHECK-NEXT:  1      2     0.50                        eor z5.b, z5.b, #0xf9
 # CHECK-NEXT:  1      2     1.00                        eor3  z29.d, z29.d, z30.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        eorbt z0.b, z1.b, z31.b
 # CHECK-NEXT:  1      2     0.50                        eorbt z0.d, z1.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        eorbt z0.h, z1.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        eorbt z0.s, z1.s, z31.s
-# CHECK-NEXT:  2      2     1.00                  U     eors  p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  2      2     1.00                        eors  p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  1      2     0.50                        eortb z0.b, z1.b, z31.b
 # CHECK-NEXT:  1      2     0.50                        eortb z0.d, z1.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        eortb z0.h, z1.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        eortb z0.s, z1.s, z31.s
 # CHECK-NEXT:  2      6     1.00                        eorv  b0, p7, z31.b
 # CHECK-NEXT:  2      6     1.00                        eorv  d0, p7, z31.d
 # CHECK-NEXT:  2      6     1.00                        eorv  h0, p7, z31.h
 # CHECK-NEXT:  2      6     1.00                        eorv  s0, p7, z31.s
-# CHECK-NEXT:  1      2     0.50                  U     ext z0.b, { z1.b, z2.b }, #0
+# CHECK-NEXT:  1      2     0.50                        ext z0.b, { z1.b, z2.b }, #0
 # CHECK-NEXT:  1      2     0.50                        ext z31.b, z31.b, z0.b, #0
 # CHECK-NEXT:  1      2     0.50                        ext z31.b, z31.b, z0.b, #255
-# CHECK-NEXT:  1      2     0.50                  U     ext z31.b, { z30.b, z31.b }, #255
+# CHECK-NEXT:  1      2     0.50                        ext z31.b, { z30.b, z31.b }, #255
 # CHECK-NEXT:  1      2     0.50                        fabd  z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        fabd  z0.h, p7/m, z0.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        fabd  z0.s, p7/m, z0.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        fabs  z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      2     0.50                        fabs  z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      2     0.50                        fabs  z31.s, p7/m, z31.s
 # CHECK-NEXT:  1      2     1.00                        facge p0.d, p0/z, z0.d, z1.d
 # CHECK-NEXT:  1      2     1.00                        facge p0.d, p0/z, z1.d, z0.d

 # CHECK-NEXT:  1      4     0.50                        fmlslb  z0.s, z1.h, z7.h[0]
 # CHECK-NEXT:  1      4     0.50                        fmlslb  z29.s, z30.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        fmlslb  z30.s, z31.h, z7.h[7]
 # CHECK-NEXT:  1      4     0.50                        fmlslt  z0.s, z1.h, z7.h[0]
 # CHECK-NEXT:  1      4     0.50                        fmlslt  z29.s, z30.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        fmlslt  z30.s, z31.h, z7.h[7]
 # CHECK-NEXT:  1      2     0.50                        fmov  z0.d, #-10.00000000
 # CHECK-NEXT:  1      2     0.50                        fmov  z0.d, #0.12500000
-# CHECK-NEXT:  1      2     0.50                  U     fmov  z0.d, p0/m, #-10.00000000
-# CHECK-NEXT:  1      2     0.50                  U     fmov  z0.d, p0/m, #0.12500000
+# CHECK-NEXT:  1      2     0.50                        fmov  z0.d, p0/m, #-10.00000000
+# CHECK-NEXT:  1      2     0.50                        fmov  z0.d, p0/m, #0.12500000
 # CHECK-NEXT:  1      2     0.50                        fmov  z0.h, #-0.12500000
-# CHECK-NEXT:  1      2     0.50                  U     fmov  z0.h, p0/m, #-0.12500000
+# CHECK-NEXT:  1      2     0.50                        fmov  z0.h, p0/m, #-0.12500000
 # CHECK-NEXT:  1      2     0.50                        fmov  z0.s, #-0.12500000
-# CHECK-NEXT:  1      2     0.50                  U     fmov  z0.s, p0/m, #-0.12500000
+# CHECK-NEXT:  1      2     0.50                        fmov  z0.s, p0/m, #-0.12500000
 # CHECK-NEXT:  1      4     0.50                        fmsb  z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      4     0.50                        fmsb  z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      4     0.50                        fmsb  z0.s, p7/m, z1.s, z31.s
 # CHECK-NEXT:  1      3     0.50                        fmul  z0.d, p0/m, z0.d, #0.5
 # CHECK-NEXT:  1      3     0.50                        fmul  z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  1      3     0.50                        fmul  z0.d, z0.d, z0.d[0]
 # CHECK-NEXT:  1      3     0.50                        fmul  z0.d, z1.d, z31.d
 # CHECK-NEXT:  1      3     0.50                        fmul  z0.h, p0/m, z0.h, #0.5

 # CHECK-NEXT:  1      2     0.50                        incp  x0, p0.b
 # CHECK-NEXT:  1      2     0.50                        incp  x0, p0.d
 # CHECK-NEXT:  1      2     0.50                        incp  x0, p0.h
 # CHECK-NEXT:  1      2     0.50                        incp  x0, p0.s
 # CHECK-NEXT:  1      2     0.50                        incp  xzr, p15.b
 # CHECK-NEXT:  1      2     0.50                        incp  xzr, p15.d
 # CHECK-NEXT:  1      2     0.50                        incp  xzr, p15.h
 # CHECK-NEXT:  1      2     0.50                        incp  xzr, p15.s
-# CHECK-NEXT:  3      7     1.00                  U     incp  z31.d, p15.d
-# CHECK-NEXT:  3      7     1.00                  U     incp  z31.h, p15.h
-# CHECK-NEXT:  3      7     1.00                  U     incp  z31.s, p15.s
+# CHECK-NEXT:  3      7     1.00                        incp  z31.d, p15.d
+# CHECK-NEXT:  3      7     1.00                        incp  z31.h, p15.h
+# CHECK-NEXT:  3      7     1.00                        incp  z31.s, p15.s
 # CHECK-NEXT:  1      2     0.50                        incw  x0
 # CHECK-NEXT:  1      2     0.50                        incw  x0, #14
 # CHECK-NEXT:  1      2     0.50                        incw  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        incw  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        incw  x0, vl1
 # CHECK-NEXT:  1      2     1.00                        incw  z0.s
 # CHECK-NEXT:  1      2     1.00                        incw  z0.s, all, mul #16
 # CHECK-NEXT:  1      4     1.00                        index z0.b, #0, #0

 # CHECK-NEXT:  1      3     1.00                        insr  z31.b, b31
 # CHECK-NEXT:  2      5     1.00                        insr  z31.b, wzr
 # CHECK-NEXT:  1      3     1.00                        insr  z31.d, d31
 # CHECK-NEXT:  2      5     1.00                        insr  z31.d, xzr
 # CHECK-NEXT:  1      3     1.00                        insr  z31.h, h31
 # CHECK-NEXT:  2      5     1.00                        insr  z31.h, wzr
 # CHECK-NEXT:  1      3     1.00                        insr  z31.s, s31
 # CHECK-NEXT:  2      5     1.00                        insr  z31.s, wzr
-# CHECK-NEXT:  1      3     1.00                  U     lasta b0, p7, z31.b
+# CHECK-NEXT:  1      3     1.00                        lasta b0, p7, z31.b
 # CHECK-NEXT:  1      3     1.00                        lasta d0, p7, z31.d
 # CHECK-NEXT:  1      3     1.00                        lasta h0, p7, z31.h
 # CHECK-NEXT:  1      3     1.00                        lasta s0, p7, z31.s
 # CHECK-NEXT:  2      5     1.00                        lasta w0, p7, z31.b
 # CHECK-NEXT:  2      5     1.00                        lasta w0, p7, z31.h
 # CHECK-NEXT:  2      5     1.00                        lasta w0, p7, z31.s
 # CHECK-NEXT:  2      5     1.00                        lasta x0, p7, z31.d
-# CHECK-NEXT:  1      3     1.00                  U     lastb b0, p7, z31.b
+# CHECK-NEXT:  1      3     1.00                        lastb b0, p7, z31.b
 # CHECK-NEXT:  1      3     1.00                        lastb d0, p7, z31.d
 # CHECK-NEXT:  1      3     1.00                        lastb h0, p7, z31.h
 # CHECK-NEXT:  1      3     1.00                        lastb s0, p7, z31.s
 # CHECK-NEXT:  2      5     1.00                        lastb w0, p7, z31.b
 # CHECK-NEXT:  2      5     1.00                        lastb w0, p7, z31.h
 # CHECK-NEXT:  2      5     1.00                        lastb w0, p7, z31.s
 # CHECK-NEXT:  2      5     1.00                        lastb x0, p7, z31.d
 # CHECK-NEXT:  1      6     0.50    *                   ld1b  { z0.b }, p0/z, [sp, x0]
 # CHECK-NEXT:  1      6     0.50    *                   ld1b  { z0.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z0.b }, p0/z, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1b  { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1b  { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1b  { z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1b  { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z21.b }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1b  { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1b  { z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld1b  { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld1b  { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  2      9     0.50    *                   ld1b  { z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld1b  { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1b  { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1b  { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      6     0.50    *                   ld1b  { z21.s }, p5/z, [x10, x21]
 # CHECK-NEXT:  1      6     0.50    *                   ld1b  { z23.d }, p3/z, [x13, x8]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z31.b }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1b  { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1b  { z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1b  { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1b  { z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1b  { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  4      9     1.00    *                   ld1b  { z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1b  { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld1b  { z31.s }, p7/z, [z31.s, #31]
 # CHECK-NEXT:  1      6     0.50    *                   ld1b  { z5.h }, p3/z, [x17, x16]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1d  { z0.d }, p0/z, [x0, z0.d, sxtw #3]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1d  { z0.d }, p0/z, [x0, z0.d, uxtw #3]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1d  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1d  { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1d  { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1d  { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1d  { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1d  { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  4      9     1.00    *                   ld1d  { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      6     0.33    *                   ld1d  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld1d  { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      6     0.33    *                   ld1d  { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1d  { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1d  { z21.d }, p5/z, [x10, z21.d, uxtw]
 # CHECK-NEXT:  1      6     0.50    *                   ld1d  { z23.d }, p3/z, [sp, x8, lsl #3]
 # CHECK-NEXT:  1      6     0.50    *                   ld1d  { z23.d }, p3/z, [x13, x8, lsl #3]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1d  { z23.d }, p3/z, [x13, z8.d, lsl #3]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1d  { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1d  { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1d  { z31.d }, p7/z, [z31.d, #248]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1h  { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1h  { z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1h  { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1d  { z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  1      6     0.33    *                   ld1d  { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1d  { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  4      9     1.00    *                   ld1d  { z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  4      9     1.00    *                   ld1h  { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  4      9     1.00    *                   ld1h  { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld1h  { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld1h  { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  2      9     0.50    *                   ld1h  { z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld1h  { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1h  { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1h  { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      6     0.50    *                   ld1h  { z21.s }, p5/z, [x10, x21, lsl #1]
 # CHECK-NEXT:  1      6     0.50    *                   ld1h  { z23.d }, p3/z, [x13, x8, lsl #1]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1h  { z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1h  { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      10    1.00    *             U     ld1h  { z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  4      10    1.00    *             U     ld1h  { z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1h  { z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  4      9     1.00    *                   ld1h  { z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1h  { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  4      9     1.00    *                   ld1h  { z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1h  { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      10    1.00    *                   ld1h  { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  4      10    1.00    *                   ld1h  { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  2      9     0.50    *                   ld1h  { z31.s }, p7/z, [z31.s, #62]
 # CHECK-NEXT:  1      6     0.50    *                   ld1h  { z5.h }, p3/z, [sp, x16, lsl #1]
 # CHECK-NEXT:  1      6     0.50    *                   ld1h  { z5.h }, p3/z, [x17, x16, lsl #1]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rb { z0.b }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rb { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rb { z0.h }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rb { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rb { z31.b }, p7/z, [sp, #63]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rb { z31.d }, p7/z, [sp, #63]

 # CHECK-NEXT:  1      6     0.33    *                   ld1rsh  { z31.d }, p7/z, [sp, #126]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rsh  { z31.s }, p7/z, [sp, #126]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rsw  { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rsw  { z31.d }, p7/z, [sp, #252]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rw { z0.d }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rw { z0.s }, p0/z, [x0]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rw { z31.d }, p7/z, [sp, #252]
 # CHECK-NEXT:  1      6     0.33    *                   ld1rw { z31.s }, p7/z, [sp, #252]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sb { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sb { z0.d }, p0/z, [z0.d]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb { z0.h }, p0/z, [sp, x0]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb { z0.h }, p0/z, [x0, x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z0.h }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1sb { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sb { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sb { z21.d }, p5/z, [x10, z21.d, uxtw]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z21.h }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z0.h }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld1sb { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sb { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sb { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb { z21.s }, p5/z, [x10, x21]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb { z23.d }, p3/z, [x13, x8]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sb { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sb { z31.d }, p7/z, [z31.d, #31]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z31.h }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sb { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1sb { z31.s }, p7/z, [z31.s, #31]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sh { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sh { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sh { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1sh { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sh { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sh { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sh { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sb { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sb { z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sb { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld1sb { z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sh { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sh { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  2      9     0.50    *                   ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  2      9     0.50    *                   ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sh { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld1sh { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sh { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sh { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sh { z21.d }, p5/z, [x10, z21.d, uxtw]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sh { z21.s }, p5/z, [sp, x21, lsl #1]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sh { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sh { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sh { z21.s }, p5/z, [x10, x21, lsl #1]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sh { z23.d }, p3/z, [x13, x8, lsl #1]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sh { z23.d }, p3/z, [x13, z8.d, lsl #1]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sh { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sh { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sh { z31.d }, p7/z, [z31.d, #62]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sh { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      10    1.00    *             U     ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1]
-# CHECK-NEXT:  4      10    1.00    *             U     ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1sh { z31.s }, p7/z, [z31.s, #62]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sw { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sw { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sw { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sw { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sw { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sh { z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sh { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sh { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sh { z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sh { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      10    1.00    *                   ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  4      10    1.00    *                   ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  2      9     0.50    *                   ld1sh { z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sw { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sw { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sw { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sw { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sw { z21.d }, p5/z, [x10, z21.d, uxtw]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sw { z23.d }, p3/z, [sp, x8, lsl #2]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sw { z23.d }, p3/z, [x13, x8, lsl #2]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sw { z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1sw { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sw { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1sw { z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1w  { z0.d }, p0/z, [x0, z0.d, sxtw #2]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1w  { z0.d }, p0/z, [x0, z0.d, uxtw #2]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1w  { z0.d }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1w  { z0.d }, p0/z, [z0.d]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1w  { z0.s }, p0/z, [x0, z0.s, sxtw]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1w  { z0.s }, p0/z, [x0, z0.s, uxtw]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1w  { z0.s }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1w  { z0.s }, p0/z, [z0.s]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1w  { z21.d }, p5/z, [x10, #5, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1w  { z21.d }, p5/z, [x10, z21.d, sxtw]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1w  { z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sw { z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      6     0.33    *                   ld1sw { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sw { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  4      9     1.00    *                   ld1sw { z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  4      9     1.00    *                   ld1w  { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  4      9     1.00    *                   ld1w  { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      6     0.33    *                   ld1w  { z0.d }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld1w  { z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  2      9     0.50    *                   ld1w  { z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  2      9     0.50    *                   ld1w  { z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ld1w  { z0.s }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld1w  { z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      6     0.33    *                   ld1w  { z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1w  { z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  4      9     1.00    *                   ld1w  { z21.d }, p5/z, [x10, z21.d, uxtw]
 # CHECK-NEXT:  1      6     0.50    *                   ld1w  { z21.s }, p5/z, [sp, x21, lsl #2]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1w  { z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ld1w  { z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      6     0.50    *                   ld1w  { z21.s }, p5/z, [x10, x21, lsl #2]
 # CHECK-NEXT:  1      6     0.50    *                   ld1w  { z23.d }, p3/z, [x13, x8, lsl #2]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1w  { z23.d }, p3/z, [x13, z8.d, lsl #2]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1w  { z31.d }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1w  { z31.d }, p7/z, [sp, z31.d]
-# CHECK-NEXT:  4      9     1.00    *             U     ld1w  { z31.d }, p7/z, [z31.d, #124]
-# CHECK-NEXT:  1      6     0.33    *             U     ld1w  { z31.s }, p7/z, [sp, #-1, mul vl]
-# CHECK-NEXT:  4      10    1.00    *             U     ld1w  { z31.s }, p7/z, [sp, z31.s, sxtw #2]
-# CHECK-NEXT:  4      10    1.00    *             U     ld1w  { z31.s }, p7/z, [sp, z31.s, uxtw #2]
-# CHECK-NEXT:  2      9     0.50    *             U     ld1w  { z31.s }, p7/z, [z31.s, #124]
-# CHECK-NEXT:  2      9     0.50    *             U     ld2b  { z0.b, z1.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2b  { z0.b, z1.b }, p0/z, [x0]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2b  { z21.b, z22.b }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2b  { z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld2b  { z5.b, z6.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  2      9     0.50    *             U     ld2d  { z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2d  { z0.d, z1.d }, p0/z, [x0]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2d  { z21.d, z22.d }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2d  { z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld2d  { z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  2      9     0.50    *             U     ld2h  { z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2h  { z0.h, z1.h }, p0/z, [x0]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2h  { z21.h, z22.h }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2h  { z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld2h  { z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  2      9     0.50    *             U     ld2w  { z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2w  { z0.s, z1.s }, p0/z, [x0]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2w  { z21.s, z22.s }, p5/z, [x10, #10, mul vl]
-# CHECK-NEXT:  2      8     0.50    *             U     ld2w  { z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld2w  { z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  3      10    0.67    *             U     ld3b  { z0.b - z2.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3b  { z0.b - z2.b }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3b  { z21.b - z23.b }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3b  { z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  3      10    0.67    *             U     ld3b  { z5.b - z7.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  3      10    0.67    *             U     ld3d  { z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3d  { z0.d - z2.d }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3d  { z21.d - z23.d }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3d  { z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  3      10    0.67    *             U     ld3d  { z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  3      10    0.67    *             U     ld3h  { z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3h  { z0.h - z2.h }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3h  { z21.h - z23.h }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3h  { z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  3      10    0.67    *             U     ld3h  { z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  3      10    0.67    *             U     ld3w  { z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3w  { z0.s - z2.s }, p0/z, [x0]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3w  { z21.s - z23.s }, p5/z, [x10, #15, mul vl]
-# CHECK-NEXT:  2      9     0.50    *             U     ld3w  { z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
-# CHECK-NEXT:  3      10    0.67    *             U     ld3w  { z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
-# CHECK-NEXT:  6      10    1.00    *             U     ld4b  { z0.b - z3.b }, p0/z, [x0, x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4b  { z0.b - z3.b }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4b  { z21.b - z24.b }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4b  { z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  6      10    1.00    *             U     ld4b  { z5.b - z8.b }, p3/z, [x17, x16]
-# CHECK-NEXT:  6      10    1.00    *             U     ld4d  { z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4d  { z0.d - z3.d }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4d  { z21.d - z24.d }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4d  { z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  6      10    1.00    *             U     ld4d  { z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
-# CHECK-NEXT:  6      10    1.00    *             U     ld4h  { z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4h  { z0.h - z3.h }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4h  { z21.h - z24.h }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4h  { z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  6      10    1.00    *             U     ld4h  { z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
-# CHECK-NEXT:  6      10    1.00    *             U     ld4w  { z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4w  { z0.s - z3.s }, p0/z, [x0]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4w  { z21.s - z24.s }, p5/z, [x10, #20, mul vl]
-# CHECK-NEXT:  4      9     1.00    *             U     ld4w  { z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
-# CHECK-NEXT:  6      10    1.00    *             U     ld4w  { z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  4      9     1.00    *                   ld1w  { z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      6     0.33    *                   ld1w  { z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld1w  { z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  4      9     1.00    *                   ld1w  { z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      6     0.33    *                   ld1w  { z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  4      10    1.00    *                   ld1w  { z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  4      10    1.00    *                   ld1w  { z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  2      9     0.50    *                   ld1w  { z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  2      9     0.50    *                   ld2b  { z0.b, z1.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  2      8     0.50    *                   ld2b  { z0.b, z1.b }, p0/z, [x0]
+# CHECK-NEXT:  2      8     0.50    *                   ld2b  { z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  2      8     0.50    *                   ld2b  { z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld2b  { z5.b, z6.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  2      9     0.50    *                   ld2d  { z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  2      8     0.50    *                   ld2d  { z0.d, z1.d }, p0/z, [x0]
+# CHECK-NEXT:  2      8     0.50    *                   ld2d  { z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  2      8     0.50    *                   ld2d  { z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld2d  { z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  2      9     0.50    *                   ld2h  { z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  2      8     0.50    *                   ld2h  { z0.h, z1.h }, p0/z, [x0]
+# CHECK-NEXT:  2      8     0.50    *                   ld2h  { z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  2      8     0.50    *                   ld2h  { z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld2h  { z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  2      9     0.50    *                   ld2w  { z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  2      8     0.50    *                   ld2w  { z0.s, z1.s }, p0/z, [x0]
+# CHECK-NEXT:  2      8     0.50    *                   ld2w  { z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  2      8     0.50    *                   ld2w  { z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld2w  { z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  3      10    0.67    *                   ld3b  { z0.b - z2.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld3b  { z0.b - z2.b }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld3b  { z21.b - z23.b }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld3b  { z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  3      10    0.67    *                   ld3b  { z5.b - z7.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  3      10    0.67    *                   ld3d  { z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  2      9     0.50    *                   ld3d  { z0.d - z2.d }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld3d  { z21.d - z23.d }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld3d  { z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  3      10    0.67    *                   ld3d  { z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  3      10    0.67    *                   ld3h  { z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  2      9     0.50    *                   ld3h  { z0.h - z2.h }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld3h  { z21.h - z23.h }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld3h  { z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  3      10    0.67    *                   ld3h  { z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  3      10    0.67    *                   ld3w  { z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  2      9     0.50    *                   ld3w  { z0.s - z2.s }, p0/z, [x0]
+# CHECK-NEXT:  2      9     0.50    *                   ld3w  { z21.s - z23.s }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  2      9     0.50    *                   ld3w  { z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  3      10    0.67    *                   ld3w  { z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  6      10    1.00    *                   ld4b  { z0.b - z3.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld4b  { z0.b - z3.b }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld4b  { z21.b - z24.b }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld4b  { z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  6      10    1.00    *                   ld4b  { z5.b - z8.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  6      10    1.00    *                   ld4d  { z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  4      9     1.00    *                   ld4d  { z0.d - z3.d }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld4d  { z21.d - z24.d }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld4d  { z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  6      10    1.00    *                   ld4d  { z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  6      10    1.00    *                   ld4h  { z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  4      9     1.00    *                   ld4h  { z0.h - z3.h }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld4h  { z21.h - z24.h }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld4h  { z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  6      10    1.00    *                   ld4h  { z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  6      10    1.00    *                   ld4w  { z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  4      9     1.00    *                   ld4w  { z0.s - z3.s }, p0/z, [x0]
+# CHECK-NEXT:  4      9     1.00    *                   ld4w  { z21.s - z24.s }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  4      9     1.00    *                   ld4w  { z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  6      10    1.00    *                   ld4w  { z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
 # CHECK-NEXT:  2      6     0.50    *             U     ldff1b  { z0.d }, p0/z, [x0, x0]
 # CHECK-NEXT:  4      9     1.00    *             U     ldff1b  { z0.d }, p0/z, [z0.d]
 # CHECK-NEXT:  2      6     0.50    *             U     ldff1b  { z0.h }, p0/z, [x0, x0]
 # CHECK-NEXT:  2      6     0.50    *             U     ldff1b  { z0.s }, p0/z, [x0, x0]
 # CHECK-NEXT:  2      9     0.50    *             U     ldff1b  { z0.s }, p0/z, [x0, z0.s, sxtw]
 # CHECK-NEXT:  2      9     0.50    *             U     ldff1b  { z0.s }, p0/z, [x0, z0.s, uxtw]
 # CHECK-NEXT:  2      9     0.50    *             U     ldff1b  { z0.s }, p0/z, [z0.s]
 # CHECK-NEXT:  4      9     1.00    *             U     ldff1b  { z21.d }, p5/z, [x10, z21.d, sxtw]

 # CHECK-NEXT:  1      6     0.33    *                   ldnt1w  { z23.s }, p3/z, [x13, #-8, mul vl]
 # CHECK-NEXT:  4      10    2.00    *                   ldnt1w  { z31.d }, p7/z, [z31.d, x0]
 # CHECK-NEXT:  4      10    2.00    *                   ldnt1w  { z31.d }, p7/z, [z31.d]
 # CHECK-NEXT:  2      9     0.50    *                   ldnt1w  { z31.s }, p7/z, [z31.s, x0]
 # CHECK-NEXT:  2      9     0.50    *                   ldnt1w  { z31.s }, p7/z, [z31.s]
 # CHECK-NEXT:  2      6     0.50    *                   ldr p0, [x0]
 # CHECK-NEXT:  2      6     0.50    *                   ldr p5, [x10, #255, mul vl]
 # CHECK-NEXT:  2      6     0.50    *                   ldr p7, [x13, #-256, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ldr z0, [x0]
-# CHECK-NEXT:  1      6     0.33    *             U     ldr z23, [x13, #255, mul vl]
-# CHECK-NEXT:  1      6     0.33    *             U     ldr z31, [sp, #-256, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ldr z0, [x0]
+# CHECK-NEXT:  1      6     0.33    *                   ldr z23, [x13, #255, mul vl]
+# CHECK-NEXT:  1      6     0.33    *                   ldr z31, [sp, #-256, mul vl]
 # CHECK-NEXT:  1      2     1.00                        lsl z0.b, p0/m, z0.b, #0
 # CHECK-NEXT:  1      2     1.00                        lsl z0.b, p0/m, z0.b, z0.b
 # CHECK-NEXT:  1      2     1.00                        lsl z0.b, p0/m, z0.b, z1.d
 # CHECK-NEXT:  1      2     1.00                        lsl z0.b, z0.b, #0
 # CHECK-NEXT:  1      2     1.00                        lsl z0.b, z1.b, z2.d
 # CHECK-NEXT:  1      2     1.00                        lsl z0.d, p0/m, z0.d, #0
 # CHECK-NEXT:  1      2     1.00                        lsl z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  1      2     1.00                        lsl z0.d, z0.d, #0

 # CHECK-NEXT:  1      2     1.00                        lsl z31.b, p0/m, z31.b, #7
 # CHECK-NEXT:  1      2     1.00                        lsl z31.b, z31.b, #7
 # CHECK-NEXT:  1      2     1.00                        lsl z31.d, p0/m, z31.d, #63
 # CHECK-NEXT:  1      2     1.00                        lsl z31.d, z31.d, #63
 # CHECK-NEXT:  1      2     1.00                        lsl z31.h, p0/m, z31.h, #15
 # CHECK-NEXT:  1      2     1.00                        lsl z31.h, z31.h, #15
 # CHECK-NEXT:  1      2     1.00                        lsl z31.s, p0/m, z31.s, #31
 # CHECK-NEXT:  1      2     1.00                        lsl z31.s, z31.s, #31
-# CHECK-NEXT:  1      2     1.00                  U     lslr  z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  1      2     1.00                  U     lslr  z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  1      2     1.00                  U     lslr  z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  1      2     1.00                  U     lslr  z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      2     1.00                        lslr  z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      2     1.00                        lslr  z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      2     1.00                        lslr  z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      2     1.00                        lslr  z0.s, p0/m, z0.s, z0.s
 # CHECK-NEXT:  1      2     1.00                        lsr z0.b, p0/m, z0.b, #1
 # CHECK-NEXT:  1      2     1.00                        lsr z0.b, p0/m, z0.b, z0.b
 # CHECK-NEXT:  1      2     1.00                        lsr z0.b, p0/m, z0.b, z1.d
 # CHECK-NEXT:  1      2     1.00                        lsr z0.b, z0.b, #1
 # CHECK-NEXT:  1      2     1.00                        lsr z0.b, z1.b, z2.d
 # CHECK-NEXT:  1      2     1.00                        lsr z0.d, p0/m, z0.d, #1
 # CHECK-NEXT:  1      2     1.00                        lsr z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  1      2     1.00                        lsr z0.d, z0.d, #1

 # CHECK-NEXT:  1      2     1.00                        lsr z31.b, p0/m, z31.b, #8
 # CHECK-NEXT:  1      2     1.00                        lsr z31.b, z31.b, #8
 # CHECK-NEXT:  1      2     1.00                        lsr z31.d, p0/m, z31.d, #64
 # CHECK-NEXT:  1      2     1.00                        lsr z31.d, z31.d, #64
 # CHECK-NEXT:  1      2     1.00                        lsr z31.h, p0/m, z31.h, #16
 # CHECK-NEXT:  1      2     1.00                        lsr z31.h, z31.h, #16
 # CHECK-NEXT:  1      2     1.00                        lsr z31.s, p0/m, z31.s, #32
 # CHECK-NEXT:  1      2     1.00                        lsr z31.s, z31.s, #32
-# CHECK-NEXT:  1      2     1.00                  U     lsrr  z0.b, p0/m, z0.b, z0.b
-# CHECK-NEXT:  1      2     1.00                  U     lsrr  z0.d, p0/m, z0.d, z0.d
-# CHECK-NEXT:  1      2     1.00                  U     lsrr  z0.h, p0/m, z0.h, z0.h
-# CHECK-NEXT:  1      2     1.00                  U     lsrr  z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      2     1.00                        lsrr  z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      2     1.00                        lsrr  z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      2     1.00                        lsrr  z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      2     1.00                        lsrr  z0.s, p0/m, z0.s, z0.s
 # CHECK-NEXT:  1      4     1.00                        mad z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  2      5     2.00                        mad z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      4     1.00                        mad z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      4     1.00                        mad z0.s, p7/m, z1.s, z31.s
 # CHECK-NEXT:  2      2     1.00                        match p0.b, p0/z, z0.b, z0.b
 # CHECK-NEXT:  2      2     1.00                        match p0.h, p0/z, z0.h, z0.h
 # CHECK-NEXT:  2      2     1.00                        match p15.b, p7/z, z30.b, z31.b
 # CHECK-NEXT:  2      2     1.00                        match p15.h, p7/z, z30.h, z31.h

 # CHECK-NEXT:  1      1     1.00                        mov p0.b, p0.b
 # CHECK-NEXT:  1      1     1.00                        mov p0.b, p0/m, p0.b
 # CHECK-NEXT:  1      1     1.00                        mov p0.b, p0/z, p0.b
 # CHECK-NEXT:  1      1     1.00                        mov p15.b, p15.b
 # CHECK-NEXT:  1      1     1.00                        mov p15.b, p15/m, p15.b
 # CHECK-NEXT:  1      1     1.00                        mov p15.b, p15/z, p15.b
 # CHECK-NEXT:  1      2     0.50                        mov z0.b, #127
 # CHECK-NEXT:  1      2     0.50                        mov z0.b, b0
-# CHECK-NEXT:  1      2     0.50                  U     mov z0.b, p0/m, b0
+# CHECK-NEXT:  1      2     0.50                        mov z0.b, p0/m, b0
 # CHECK-NEXT:  2      5     1.00                        mov z0.b, p0/m, w0
 # CHECK-NEXT:  1      2     0.50                        mov z0.b, p0/z, #127
 # CHECK-NEXT:  1      3     3.00                        mov z0.b, w0
 # CHECK-NEXT:  1      2     0.50                        mov z0.d, #0
 # CHECK-NEXT:  1      2     0.50                        mov z0.d, #0xe0000000000003ff
 # CHECK-NEXT:  1      2     0.50                        mov z0.d, #0xffffffffffff7fff
 # CHECK-NEXT:  1      2     0.50                        mov z0.d, #32768
 # CHECK-NEXT:  1      2     0.50                        mov z0.d, d0

 # CHECK-NEXT:  1      2     0.50                        mov z21.s, #32512
 # CHECK-NEXT:  1      2     0.50                        mov z21.s, p0/z, #-128
 # CHECK-NEXT:  1      2     0.50                        mov z21.s, p0/z, #-32768
 # CHECK-NEXT:  1      2     0.50                        mov z21.s, p0/z, #127
 # CHECK-NEXT:  1      2     0.50                        mov z21.s, p0/z, #32512
 # CHECK-NEXT:  1      2     0.50                        mov z21.s, p15/m, #-128
 # CHECK-NEXT:  1      2     0.50                        mov z21.s, p15/m, #-32768
 # CHECK-NEXT:  1      2     0.50                        mov z31.b, p15/m, z31.b
-# CHECK-NEXT:  1      2     0.50                  U     mov z31.b, p7/m, b31
+# CHECK-NEXT:  1      2     0.50                        mov z31.b, p7/m, b31
 # CHECK-NEXT:  1      2     0.50                        movprfx z31, z6
 # CHECK-NEXT:  2      5     1.00                        mov z31.b, p7/m, wsp
 # CHECK-NEXT:  1      3     3.00                        mov z31.b, wsp
 # CHECK-NEXT:  1      2     0.50                        mov z31.b, z31.b[63]
 # CHECK-NEXT:  1      2     0.50                        mov z31.d, p15/m, z31.d
 # CHECK-NEXT:  1      2     0.50                        mov z31.d, p7/m, d31
-# CHECK-NEXT:  1      2     0.50                  U     movprfx z31.d, p7/z, z6.d
+# CHECK-NEXT:  1      2     0.50                        movprfx z31.d, p7/z, z6.d
 # CHECK-NEXT:  2      5     1.00                        mov z31.d, p7/m, sp
 # CHECK-NEXT:  1      3     3.00                        mov z31.d, sp
 # CHECK-NEXT:  1      2     0.50                        mov z31.d, z0.d
 # CHECK-NEXT:  1      2     0.50                        mov z31.d, z31.d[7]
 # CHECK-NEXT:  1      2     0.50                        mov z31.h, p15/m, z31.h
 # CHECK-NEXT:  1      2     0.50                        mov z31.h, p7/m, h31
 # CHECK-NEXT:  2      5     1.00                        mov z31.h, p7/m, wsp
 # CHECK-NEXT:  1      3     3.00                        mov z31.h, wsp

 # CHECK-NEXT:  1      2     0.50                        mov z5.b, p0/z, #-1
 # CHECK-NEXT:  1      2     0.50                        mov z5.b, p0/z, #-128
 # CHECK-NEXT:  1      2     0.50                        mov z5.b, p0/z, #127
 # CHECK-NEXT:  1      2     0.50                        mov z5.b, p15/m, #-128
 # CHECK-NEXT:  1      2     0.50                        mov z5.d, #-6
 # CHECK-NEXT:  1      2     0.50                        mov z5.h, #-6
 # CHECK-NEXT:  1      2     0.50                        mov z5.q, z17.q[3]
 # CHECK-NEXT:  1      2     0.50                        mov z5.s, #-6
-# CHECK-NEXT:  2      2     1.00                  U     movs  p0.b, p0.b
-# CHECK-NEXT:  2      2     1.00                  U     movs  p0.b, p0/z, p0.b
-# CHECK-NEXT:  2      2     1.00                  U     movs  p15.b, p15.b
-# CHECK-NEXT:  2      2     1.00                  U     movs  p15.b, p15/z, p15.b
+# CHECK-NEXT:  2      2     1.00                        movs  p0.b, p0.b
+# CHECK-NEXT:  2      2     1.00                        movs  p0.b, p0/z, p0.b
+# CHECK-NEXT:  2      2     1.00                        movs  p15.b, p15.b
+# CHECK-NEXT:  2      2     1.00                        movs  p15.b, p15/z, p15.b
 # CHECK-NEXT:  1      1     0.10                  U     mrs x3, ID_AA64ZFR0_EL1
 # CHECK-NEXT:  1      1     0.10                  U     mrs x3, ZCR_EL1
 # CHECK-NEXT:  1      1     0.10                  U     mrs x3, ZCR_EL12
 # CHECK-NEXT:  1      1     0.10                  U     mrs x3, ZCR_EL2
 # CHECK-NEXT:  1      1     0.10                  U     mrs x3, ZCR_EL3
 # CHECK-NEXT:  1      4     1.00                        msb z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  2      5     2.00                        msb z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      4     1.00                        msb z0.h, p7/m, z1.h, z31.h

 # CHECK-NEXT:  2      5     2.00                        mul z31.d, z31.d, #127
 # CHECK-NEXT:  2      5     2.00                        mul z31.d, z31.d, z31.d
 # CHECK-NEXT:  1      4     1.00                        mul z31.h, z31.h, #-128
 # CHECK-NEXT:  1      4     1.00                        mul z31.h, z31.h, #127
 # CHECK-NEXT:  1      4     1.00                        mul z31.s, z31.s, #-128
 # CHECK-NEXT:  1      4     1.00                        mul z31.s, z31.s, #127
 # CHECK-NEXT:  1      1     1.00                        nand  p0.b, p0/z, p0.b, p0.b
 # CHECK-NEXT:  1      1     1.00                        nand  p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  2      2     1.00                  U     nands p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  2      2     1.00                  U     nands p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  2      2     1.00                        nands p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  2      2     1.00                        nands p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      2     0.50                        nbsl  z0.d, z0.d, z1.d, z2.d
 # CHECK-NEXT:  1      2     0.50                        neg z0.b, p0/m, z0.b
 # CHECK-NEXT:  1      2     0.50                        neg z0.d, p0/m, z0.d
 # CHECK-NEXT:  1      2     0.50                        neg z0.h, p0/m, z0.h
 # CHECK-NEXT:  1      2     0.50                        neg z0.s, p0/m, z0.s
 # CHECK-NEXT:  1      2     0.50                        neg z31.b, p7/m, z31.b
 # CHECK-NEXT:  1      2     0.50                        neg z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      2     0.50                        neg z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      2     0.50                        neg z31.s, p7/m, z31.s
 # CHECK-NEXT:  2      2     1.00                        nmatch  p0.b, p0/z, z0.b, z0.b
 # CHECK-NEXT:  2      2     1.00                        nmatch  p0.h, p0/z, z0.h, z0.h
 # CHECK-NEXT:  2      2     1.00                        nmatch  p15.b, p7/z, z30.b, z31.b
 # CHECK-NEXT:  2      2     1.00                        nmatch  p15.h, p7/z, z30.h, z31.h
 # CHECK-NEXT:  1      1     1.00                        nor p0.b, p0/z, p0.b, p0.b
 # CHECK-NEXT:  1      1     1.00                        nor p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  2      2     1.00                  U     nors  p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  2      2     1.00                  U     nors  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  2      2     1.00                        nors  p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  2      2     1.00                        nors  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      1     1.00                        not p0.b, p0/z, p0.b
 # CHECK-NEXT:  1      1     1.00                        not p15.b, p15/z, p15.b
 # CHECK-NEXT:  1      2     0.50                        not z31.b, p7/m, z31.b
 # CHECK-NEXT:  1      2     0.50                        not z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      2     0.50                        not z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      2     0.50                        not z31.s, p7/m, z31.s
-# CHECK-NEXT:  2      2     1.00                  U     nots  p0.b, p0/z, p0.b
-# CHECK-NEXT:  2      2     1.00                  U     nots  p15.b, p15/z, p15.b
+# CHECK-NEXT:  2      2     1.00                        nots  p0.b, p0/z, p0.b
+# CHECK-NEXT:  2      2     1.00                        nots  p15.b, p15/z, p15.b
 # CHECK-NEXT:  1      1     1.00                        orn p0.b, p0/z, p0.b, p0.b
 # CHECK-NEXT:  1      1     1.00                        orn p15.b, p15/z, p15.b, p15.b
-# CHECK-NEXT:  2      2     1.00                  U     orns  p0.b, p0/z, p0.b, p0.b
-# CHECK-NEXT:  2      2     1.00                  U     orns  p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  2      2     1.00                        orns  p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  2      2     1.00                        orns  p15.b, p15/z, p15.b, p15.b
 # CHECK-NEXT:  1      1     1.00                        orr p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  1      2     0.50                        orr z0.d, z0.d, #0x6
 # CHECK-NEXT:  1      2     0.50                        orr z0.d, z0.d, #0xfffffffffffffff9
 # CHECK-NEXT:  1      2     0.50                        orr z0.s, z0.s, #0x6
 # CHECK-NEXT:  1      2     0.50                        orr z0.s, z0.s, #0xfffffff9
 # CHECK-NEXT:  1      2     0.50                        orr z23.d, z13.d, z8.d
 # CHECK-NEXT:  1      2     0.50                        orr z23.h, z23.h, #0x6
 # CHECK-NEXT:  1      2     0.50                        orr z23.h, z23.h, #0xfff9
 # CHECK-NEXT:  1      2     0.50                        orr z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  1      2     0.50                        orr z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        orr z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        orr z31.s, p7/m, z31.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        orr z5.b, z5.b, #0x6
 # CHECK-NEXT:  1      2     0.50                        orr z5.b, z5.b, #0xf9
-# CHECK-NEXT:  2      2     1.00                  U     orrs  p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  2      2     1.00                        orrs  p0.b, p0/z, p0.b, p1.b
 # CHECK-NEXT:  2      6     1.00                        orv b0, p7, z31.b
 # CHECK-NEXT:  2      6     1.00                        orv d0, p7, z31.d
 # CHECK-NEXT:  2      6     1.00                        orv h0, p7, z31.h
 # CHECK-NEXT:  2      6     1.00                        orv s0, p7, z31.s
 # CHECK-NEXT:  1      2     0.50                        pfalse  p15.b
 # CHECK-NEXT:  1      3     0.50                        pfirst  p0.b, p15, p0.b
 # CHECK-NEXT:  1      3     0.50                        pfirst  p15.b, p15, p15.b
 # CHECK-NEXT:  1      2     1.00                        pmul  z0.b, z1.b, z2.b
 # CHECK-NEXT:  1      2     1.00                        pmul  z29.b, z30.b, z31.b
 # CHECK-NEXT:  1      2     1.00                        pmullb  z0.h, z1.b, z2.b
 # CHECK-NEXT:  1      2     1.00                        pmullb  z29.q, z30.d, z31.d
 # CHECK-NEXT:  1      2     1.00                        pmullb  z31.d, z31.s, z31.s
 # CHECK-NEXT:  1      2     1.00                        pmullt  z0.h, z1.b, z2.b
 # CHECK-NEXT:  1      2     1.00                        pmullt  z29.q, z30.d, z31.d
 # CHECK-NEXT:  1      2     1.00                        pmullt  z31.d, z31.s, z31.s
 # CHECK-NEXT:  1      3     0.50                        pnext p0.b, p15, p0.b
 # CHECK-NEXT:  1      3     0.50                        pnext p0.d, p15, p0.d
 # CHECK-NEXT:  1      3     0.50                        pnext p0.h, p15, p0.h
 # CHECK-NEXT:  1      3     0.50                        pnext p0.s, p15, p0.s
 # CHECK-NEXT:  1      3     0.50                        pnext p15.b, p15, p15.b
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  #14, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  #15, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  #6, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  #7, p0, [x0]
-# CHECK-NEXT:  1      4     0.33    *      *            prfb  #7, p3, [z13.s, #31]
-# CHECK-NEXT:  1      4     0.33    *      *            prfb  #7, p3, [z13.s]
+# CHECK-NEXT:  1      4     0.33    *      *      U     prfb  #7, p3, [z13.s, #31]
+# CHECK-NEXT:  1      4     0.33    *      *      U     prfb  #7, p3, [z13.s]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pldl1keep, p0, [x0, z0.d, uxtw]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pldl1keep, p0, [x0, z0.d]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pldl1keep, p0, [x0, z0.s, uxtw]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pldl1keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pldl1strm, p0, [x0, #-32, mul vl]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pldl1strm, p0, [x0, #31, mul vl]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pldl1strm, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pldl2keep, p0, [x0]

 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pstl2keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pstl2strm, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pstl3keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfb  pstl3strm, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  #14, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  #15, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  #15, p7, [z31.d, #248]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  #15, p7, [z31.d]
-# CHECK-NEXT:  1      4     0.33    *      *            prfd  #15, p7, [z31.s, #248]
-# CHECK-NEXT:  1      4     0.33    *      *            prfd  #15, p7, [z31.s]
+# CHECK-NEXT:  1      4     0.33    *      *      U     prfd  #15, p7, [z31.s, #248]
+# CHECK-NEXT:  1      4     0.33    *      *      U     prfd  #15, p7, [z31.s]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  #6, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  #7, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pldl1keep, p0, [x0, z0.d, lsl #3]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pldl1keep, p0, [x0, z0.d, sxtw #3]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pldl1keep, p0, [x0, z0.d, uxtw #3]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pldl1keep, p0, [x0, z0.s, sxtw #3]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pldl1keep, p0, [x0, z0.s, uxtw #3]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pldl1keep, p0, [x0]

 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pstl2keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pstl2strm, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pstl3keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfd  pstl3strm, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  #14, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  #15, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  #15, p7, [z31.d, #62]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  #15, p7, [z31.d]
-# CHECK-NEXT:  1      4     0.33    *      *            prfh  #15, p7, [z31.s, #62]
-# CHECK-NEXT:  1      4     0.33    *      *            prfh  #15, p7, [z31.s]
+# CHECK-NEXT:  1      4     0.33    *      *      U     prfh  #15, p7, [z31.s, #62]
+# CHECK-NEXT:  1      4     0.33    *      *      U     prfh  #15, p7, [z31.s]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  #6, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  #7, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pldl1keep, p0, [x0, z0.d, lsl #1]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pldl1keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pldl1strm, p0, [x0, #-32, mul vl]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pldl1strm, p0, [x0, #31, mul vl]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pldl1strm, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pldl2keep, p0, [x0]

 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pstl2keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pstl2strm, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pstl3keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfh  pstl3strm, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #14, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #15, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #15, p7, [z31.d, #124]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #15, p7, [z31.d]
-# CHECK-NEXT:  1      4     0.33    *      *            prfw  #15, p7, [z31.s, #124]
-# CHECK-NEXT:  1      4     0.33    *      *            prfw  #15, p7, [z31.s]
+# CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #15, p7, [z31.s, #124]
+# CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #15, p7, [z31.s]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #6, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #7, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  #7, p3, [x13, z8.d, uxtw #2]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  pldl1keep, p0, [x0, z0.d, sxtw #2]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  pldl1keep, p0, [x0, z0.s, uxtw #2]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  pldl1keep, p0, [x0]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  pldl1strm, p0, [x0, #-32, mul vl]
 # CHECK-NEXT:  1      4     0.33    *      *      U     prfw  pldl1strm, p0, [x0, #31, mul vl]

 # CHECK-NEXT:  1      2     0.50                        ptrue p7.s, vl3
 # CHECK-NEXT:  1      2     0.50                        ptrue p7.s, vl32
 # CHECK-NEXT:  1      2     0.50                        ptrue p7.s, vl4
 # CHECK-NEXT:  1      2     0.50                        ptrue p7.s, vl5
 # CHECK-NEXT:  1      2     0.50                        ptrue p7.s, vl6
 # CHECK-NEXT:  1      2     0.50                        ptrue p7.s, vl64
 # CHECK-NEXT:  1      2     0.50                        ptrue p7.s, vl7
 # CHECK-NEXT:  1      2     0.50                        ptrue p7.s, vl8
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p0.b, pow2
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p0.d, pow2
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p0.h, pow2
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p0.s, pow2
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p15.b
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p15.d
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p15.h
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p15.s
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #14
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #15
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #16
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #17
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #18
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #19
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #20
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #21
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #22
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #23
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #24
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #25
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #26
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #27
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, #28
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, mul3
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, mul4
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl1
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl128
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl16
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl2
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl256
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl3
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl32
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl4
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl5
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl6
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl64
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl7
-# CHECK-NEXT:  1      3     0.50                  U     ptrues  p7.s, vl8
+# CHECK-NEXT:  1      3     0.50                        ptrues  p0.b, pow2
+# CHECK-NEXT:  1      3     0.50                        ptrues  p0.d, pow2
+# CHECK-NEXT:  1      3     0.50                        ptrues  p0.h, pow2
+# CHECK-NEXT:  1      3     0.50                        ptrues  p0.s, pow2
+# CHECK-NEXT:  1      3     0.50                        ptrues  p15.b
+# CHECK-NEXT:  1      3     0.50                        ptrues  p15.d
+# CHECK-NEXT:  1      3     0.50                        ptrues  p15.h
+# CHECK-NEXT:  1      3     0.50                        ptrues  p15.s
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #14
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #15
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #16
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #17
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #18
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #19
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #20
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #21
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #22
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #23
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #24
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #25
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #26
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #27
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, #28
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, mul3
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, mul4
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl1
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl128
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl16
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl2
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl256
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl3
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl32
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl4
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl5
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl6
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl64
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl7
+# CHECK-NEXT:  1      3     0.50                        ptrues  p7.s, vl8
 # CHECK-NEXT:  1      2     0.50                        punpkhi p0.h, p0.b
 # CHECK-NEXT:  1      2     0.50                        punpkhi p15.h, p15.b
 # CHECK-NEXT:  1      2     0.50                        punpklo p0.h, p0.b
 # CHECK-NEXT:  1      2     0.50                        punpklo p15.h, p15.b
 # CHECK-NEXT:  1      2     0.50                        raddhnb z0.b, z1.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        raddhnb z0.h, z1.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        raddhnb z0.s, z1.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        raddhnt z0.b, z1.h, z31.h

 # CHECK-NEXT:  1      4     1.00                        smullb  z0.s, z1.h, z7.h[7]
 # CHECK-NEXT:  1      4     1.00                        smullb  z29.s, z30.h, z31.h
 # CHECK-NEXT:  1      4     1.00                        smullb  z31.d, z31.s, z31.s
 # CHECK-NEXT:  1      4     1.00                        smullt  z0.d, z1.s, z15.s[1]
 # CHECK-NEXT:  1      4     1.00                        smullt  z0.h, z1.b, z2.b
 # CHECK-NEXT:  1      4     1.00                        smullt  z0.s, z1.h, z7.h[7]
 # CHECK-NEXT:  1      4     1.00                        smullt  z29.s, z30.h, z31.h
 # CHECK-NEXT:  1      4     1.00                        smullt  z31.d, z31.s, z31.s
-# CHECK-NEXT:  1      3     1.00                  U     splice  z29.b, p7, { z30.b, z31.b }
-# CHECK-NEXT:  1      3     1.00                  U     splice  z29.d, p7, { z30.d, z31.d }
-# CHECK-NEXT:  1      3     1.00                  U     splice  z29.h, p7, { z30.h, z31.h }
-# CHECK-NEXT:  1      3     1.00                  U     splice  z29.s, p7, { z30.s, z31.s }
+# CHECK-NEXT:  1      3     1.00                        splice  z29.b, p7, { z30.b, z31.b }
+# CHECK-NEXT:  1      3     1.00                        splice  z29.d, p7, { z30.d, z31.d }
+# CHECK-NEXT:  1      3     1.00                        splice  z29.h, p7, { z30.h, z31.h }
+# CHECK-NEXT:  1      3     1.00                        splice  z29.s, p7, { z30.s, z31.s }
 # CHECK-NEXT:  1      3     1.00                        splice  z31.b, p7, z31.b, z31.b
 # CHECK-NEXT:  1      3     1.00                        splice  z31.d, p7, z31.d, z31.d
 # CHECK-NEXT:  1      3     1.00                        splice  z31.h, p7, z31.h, z31.h
 # CHECK-NEXT:  1      3     1.00                        splice  z31.s, p7, z31.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        sqabs z31.b, p7/m, z31.b
 # CHECK-NEXT:  1      2     0.50                        sqabs z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      2     0.50                        sqabs z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      2     0.50                        sqabs z31.s, p7/m, z31.s

 # CHECK-NEXT:  1      2     0.50                        sqcadd  z31.d, z31.d, z31.d, #270
 # CHECK-NEXT:  1      2     0.50                        sqcadd  z31.h, z31.h, z31.h, #270
 # CHECK-NEXT:  1      2     0.50                        sqcadd  z31.s, z31.s, z31.s, #270
 # CHECK-NEXT:  1      2     0.50                        sqdecb  x0
 # CHECK-NEXT:  1      2     0.50                        sqdecb  x0, #14
 # CHECK-NEXT:  1      2     0.50                        sqdecb  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqdecb  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        sqdecb  x0, vl1
-# CHECK-NEXT:  1      2     0.50                  U     sqdecb  x0, w0
-# CHECK-NEXT:  1      2     0.50                  U     sqdecb  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     0.50                  U     sqdecb  x0, w0, pow2
-# CHECK-NEXT:  1      2     0.50                  U     sqdecb  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqdecb  x0, w0
+# CHECK-NEXT:  1      2     0.50                        sqdecb  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqdecb  x0, w0, pow2
+# CHECK-NEXT:  1      2     0.50                        sqdecb  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqdecd  x0
 # CHECK-NEXT:  1      2     0.50                        sqdecd  x0, #14
 # CHECK-NEXT:  1      2     0.50                        sqdecd  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqdecd  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        sqdecd  x0, vl1
-# CHECK-NEXT:  1      2     0.50                  U     sqdecd  x0, w0
-# CHECK-NEXT:  1      2     0.50                  U     sqdecd  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     0.50                  U     sqdecd  x0, w0, pow2
-# CHECK-NEXT:  1      2     0.50                  U     sqdecd  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqdecd  x0, w0
+# CHECK-NEXT:  1      2     0.50                        sqdecd  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqdecd  x0, w0, pow2
+# CHECK-NEXT:  1      2     0.50                        sqdecd  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdecd  z0.d
 # CHECK-NEXT:  1      2     1.00                        sqdecd  z0.d, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdecd  z0.d, pow2
 # CHECK-NEXT:  1      2     1.00                        sqdecd  z0.d, pow2, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqdech  x0
 # CHECK-NEXT:  1      2     0.50                        sqdech  x0, #14
 # CHECK-NEXT:  1      2     0.50                        sqdech  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqdech  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        sqdech  x0, vl1
-# CHECK-NEXT:  1      2     0.50                  U     sqdech  x0, w0
-# CHECK-NEXT:  1      2     0.50                  U     sqdech  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     0.50                  U     sqdech  x0, w0, pow2
-# CHECK-NEXT:  1      2     0.50                  U     sqdech  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqdech  x0, w0
+# CHECK-NEXT:  1      2     0.50                        sqdech  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqdech  x0, w0, pow2
+# CHECK-NEXT:  1      2     0.50                        sqdech  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdech  z0.h
 # CHECK-NEXT:  1      2     1.00                        sqdech  z0.h, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdech  z0.h, pow2
 # CHECK-NEXT:  1      2     1.00                        sqdech  z0.h, pow2, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqdecp  x0, p0.b
 # CHECK-NEXT:  1      2     0.50                        sqdecp  x0, p0.d
 # CHECK-NEXT:  1      2     0.50                        sqdecp  x0, p0.h
 # CHECK-NEXT:  1      2     0.50                        sqdecp  x0, p0.s
-# CHECK-NEXT:  1      2     0.50                  U     sqdecp  xzr, p15.b, wzr
-# CHECK-NEXT:  1      2     0.50                  U     sqdecp  xzr, p15.d, wzr
-# CHECK-NEXT:  1      2     0.50                  U     sqdecp  xzr, p15.h, wzr
-# CHECK-NEXT:  1      2     0.50                  U     sqdecp  xzr, p15.s, wzr
+# CHECK-NEXT:  1      2     0.50                        sqdecp  xzr, p15.b, wzr
+# CHECK-NEXT:  1      2     0.50                        sqdecp  xzr, p15.d, wzr
+# CHECK-NEXT:  1      2     0.50                        sqdecp  xzr, p15.h, wzr
+# CHECK-NEXT:  1      2     0.50                        sqdecp  xzr, p15.s, wzr
 # CHECK-NEXT:  3      7     1.00                        sqdecp  z0.d, p0.d
 # CHECK-NEXT:  3      7     1.00                        sqdecp  z0.h, p0.h
 # CHECK-NEXT:  3      7     1.00                        sqdecp  z0.s, p0.s
 # CHECK-NEXT:  1      2     0.50                        sqdecw  x0
 # CHECK-NEXT:  1      2     0.50                        sqdecw  x0, #14
 # CHECK-NEXT:  1      2     0.50                        sqdecw  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqdecw  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        sqdecw  x0, vl1
-# CHECK-NEXT:  1      2     0.50                  U     sqdecw  x0, w0
-# CHECK-NEXT:  1      2     0.50                  U     sqdecw  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     0.50                  U     sqdecw  x0, w0, pow2
-# CHECK-NEXT:  1      2     0.50                  U     sqdecw  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqdecw  x0, w0
+# CHECK-NEXT:  1      2     0.50                        sqdecw  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqdecw  x0, w0, pow2
+# CHECK-NEXT:  1      2     0.50                        sqdecw  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdecw  z0.s
 # CHECK-NEXT:  1      2     1.00                        sqdecw  z0.s, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqdecw  z0.s, pow2
 # CHECK-NEXT:  1      2     1.00                        sqdecw  z0.s, pow2, mul #16
 # CHECK-NEXT:  1      4     1.00                        sqdmlalb  z0.d, z1.s, z15.s[3]
 # CHECK-NEXT:  1      4     1.00                        sqdmlalb  z0.d, z1.s, z31.s
 # CHECK-NEXT:  1      4     1.00                        sqdmlalb  z0.h, z1.b, z31.b
 # CHECK-NEXT:  1      4     1.00                        sqdmlalb  z0.s, z1.h, z31.h

 # CHECK-NEXT:  1      4     1.00                        sqdmullt  z0.s, z1.h, z7.h[7]
 # CHECK-NEXT:  1      4     1.00                        sqdmullt  z29.s, z30.h, z31.h
 # CHECK-NEXT:  1      4     1.00                        sqdmullt  z31.d, z31.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        sqincb  x0
 # CHECK-NEXT:  1      2     0.50                        sqincb  x0, #14
 # CHECK-NEXT:  1      2     0.50                        sqincb  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqincb  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        sqincb  x0, vl1
-# CHECK-NEXT:  1      2     0.50                  U     sqincb  x0, w0
-# CHECK-NEXT:  1      2     0.50                  U     sqincb  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     0.50                  U     sqincb  x0, w0, pow2
-# CHECK-NEXT:  1      2     0.50                  U     sqincb  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqincb  x0, w0
+# CHECK-NEXT:  1      2     0.50                        sqincb  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqincb  x0, w0, pow2
+# CHECK-NEXT:  1      2     0.50                        sqincb  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqincd  x0
 # CHECK-NEXT:  1      2     0.50                        sqincd  x0, #14
 # CHECK-NEXT:  1      2     0.50                        sqincd  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqincd  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        sqincd  x0, vl1
-# CHECK-NEXT:  1      2     0.50                  U     sqincd  x0, w0
-# CHECK-NEXT:  1      2     0.50                  U     sqincd  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     0.50                  U     sqincd  x0, w0, pow2
-# CHECK-NEXT:  1      2     0.50                  U     sqincd  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqincd  x0, w0
+# CHECK-NEXT:  1      2     0.50                        sqincd  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqincd  x0, w0, pow2
+# CHECK-NEXT:  1      2     0.50                        sqincd  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincd  z0.d
 # CHECK-NEXT:  1      2     1.00                        sqincd  z0.d, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincd  z0.d, pow2
 # CHECK-NEXT:  1      2     1.00                        sqincd  z0.d, pow2, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqinch  x0
 # CHECK-NEXT:  1      2     0.50                        sqinch  x0, #14
 # CHECK-NEXT:  1      2     0.50                        sqinch  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqinch  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        sqinch  x0, vl1
-# CHECK-NEXT:  1      2     0.50                  U     sqinch  x0, w0
-# CHECK-NEXT:  1      2     0.50                  U     sqinch  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     0.50                  U     sqinch  x0, w0, pow2
-# CHECK-NEXT:  1      2     0.50                  U     sqinch  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqinch  x0, w0
+# CHECK-NEXT:  1      2     0.50                        sqinch  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqinch  x0, w0, pow2
+# CHECK-NEXT:  1      2     0.50                        sqinch  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqinch  z0.h
 # CHECK-NEXT:  1      2     1.00                        sqinch  z0.h, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqinch  z0.h, pow2
 # CHECK-NEXT:  1      2     1.00                        sqinch  z0.h, pow2, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqincp  x0, p0.b
 # CHECK-NEXT:  1      2     0.50                        sqincp  x0, p0.d
 # CHECK-NEXT:  1      2     0.50                        sqincp  x0, p0.h
 # CHECK-NEXT:  1      2     0.50                        sqincp  x0, p0.s
-# CHECK-NEXT:  1      2     0.50                  U     sqincp  xzr, p15.b, wzr
-# CHECK-NEXT:  1      2     0.50                  U     sqincp  xzr, p15.d, wzr
-# CHECK-NEXT:  1      2     0.50                  U     sqincp  xzr, p15.h, wzr
-# CHECK-NEXT:  1      2     0.50                  U     sqincp  xzr, p15.s, wzr
+# CHECK-NEXT:  1      2     0.50                        sqincp  xzr, p15.b, wzr
+# CHECK-NEXT:  1      2     0.50                        sqincp  xzr, p15.d, wzr
+# CHECK-NEXT:  1      2     0.50                        sqincp  xzr, p15.h, wzr
+# CHECK-NEXT:  1      2     0.50                        sqincp  xzr, p15.s, wzr
 # CHECK-NEXT:  3      7     1.00                        sqincp  z0.d, p0.d
 # CHECK-NEXT:  3      7     1.00                        sqincp  z0.h, p0.h
 # CHECK-NEXT:  3      7     1.00                        sqincp  z0.s, p0.s
 # CHECK-NEXT:  1      2     0.50                        sqincw  x0
 # CHECK-NEXT:  1      2     0.50                        sqincw  x0, #14
 # CHECK-NEXT:  1      2     0.50                        sqincw  x0, all, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqincw  x0, pow2
 # CHECK-NEXT:  1      2     0.50                        sqincw  x0, vl1
-# CHECK-NEXT:  1      2     0.50                  U     sqincw  x0, w0
-# CHECK-NEXT:  1      2     0.50                  U     sqincw  x0, w0, all, mul #16
-# CHECK-NEXT:  1      2     0.50                  U     sqincw  x0, w0, pow2
-# CHECK-NEXT:  1      2     0.50                  U     sqincw  x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqincw  x0, w0
+# CHECK-NEXT:  1      2     0.50                        sqincw  x0, w0, all, mul #16
+# CHECK-NEXT:  1      2     0.50                        sqincw  x0, w0, pow2
+# CHECK-NEXT:  1      2     0.50                        sqincw  x0, w0, pow2, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincw  z0.s
 # CHECK-NEXT:  1      2     1.00                        sqincw  z0.s, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        sqincw  z0.s, pow2
 # CHECK-NEXT:  1      2     1.00                        sqincw  z0.s, pow2, mul #16
 # CHECK-NEXT:  1      2     0.50                        sqneg z31.b, p7/m, z31.b
 # CHECK-NEXT:  1      2     0.50                        sqneg z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      2     0.50                        sqneg z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      2     0.50                        sqneg z31.s, p7/m, z31.s

 # CHECK-NEXT:  1      4     1.00                        sqrdmulh  z0.h, z1.h, z7.h[7]
 # CHECK-NEXT:  1      4     1.00                        sqrdmulh  z0.s, z1.s, z7.s[3]
 # CHECK-NEXT:  1      4     1.00                        sqrdmulh  z29.s, z30.s, z31.s
 # CHECK-NEXT:  2      5     2.00                        sqrdmulh  z31.d, z31.d, z31.d
 # CHECK-NEXT:  1      4     1.00                        sqrshl  z0.b, p0/m, z0.b, z1.b
 # CHECK-NEXT:  1      4     1.00                        sqrshl  z0.h, p0/m, z0.h, z1.h
 # CHECK-NEXT:  1      4     1.00                        sqrshl  z29.s, p7/m, z29.s, z30.s
 # CHECK-NEXT:  1      4     1.00                        sqrshl  z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  1      4     1.00                  U     sqrshlr z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  1      4     1.00                  U     sqrshlr z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  1      4     1.00                  U     sqrshlr z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  1      4     1.00                  U     sqrshlr z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqrshlr z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqrshlr z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqrshlr z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqrshlr z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        sqrshrnb  z0.b, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        sqrshrnb  z0.h, z0.s, #1
 # CHECK-NEXT:  1      4     1.00                        sqrshrnb  z0.s, z0.d, #1
 # CHECK-NEXT:  1      4     1.00                        sqrshrnb  z31.b, z31.h, #8
 # CHECK-NEXT:  1      4     1.00                        sqrshrnb  z31.h, z31.s, #16
 # CHECK-NEXT:  1      4     1.00                        sqrshrnb  z31.s, z31.d, #32
 # CHECK-NEXT:  1      4     1.00                        sqrshrnt  z0.b, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        sqrshrnt  z0.h, z0.s, #1

 # CHECK-NEXT:  1      4     1.00                        sqshl z0.h, p0/m, z0.h, z1.h
 # CHECK-NEXT:  1      4     1.00                        sqshl z0.s, p0/m, z0.s, #0
 # CHECK-NEXT:  1      4     1.00                        sqshl z29.s, p7/m, z29.s, z30.s
 # CHECK-NEXT:  1      4     1.00                        sqshl z31.b, p0/m, z31.b, #7
 # CHECK-NEXT:  1      4     1.00                        sqshl z31.d, p0/m, z31.d, #63
 # CHECK-NEXT:  1      4     1.00                        sqshl z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        sqshl z31.h, p0/m, z31.h, #15
 # CHECK-NEXT:  1      4     1.00                        sqshl z31.s, p0/m, z31.s, #31
-# CHECK-NEXT:  1      4     1.00                  U     sqshlr  z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  1      4     1.00                  U     sqshlr  z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  1      4     1.00                  U     sqshlr  z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  1      4     1.00                  U     sqshlr  z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqshlr  z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqshlr  z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqshlr  z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqshlr  z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        sqshlu  z0.b, p0/m, z0.b, #0
 # CHECK-NEXT:  1      4     1.00                        sqshlu  z0.d, p0/m, z0.d, #0
 # CHECK-NEXT:  1      4     1.00                        sqshlu  z0.h, p0/m, z0.h, #0
 # CHECK-NEXT:  1      4     1.00                        sqshlu  z0.s, p0/m, z0.s, #0
 # CHECK-NEXT:  1      4     1.00                        sqshlu  z31.b, p0/m, z31.b, #7
 # CHECK-NEXT:  1      4     1.00                        sqshlu  z31.d, p0/m, z31.d, #63
 # CHECK-NEXT:  1      4     1.00                        sqshlu  z31.h, p0/m, z31.h, #15
 # CHECK-NEXT:  1      4     1.00                        sqshlu  z31.s, p0/m, z31.s, #31

 # CHECK-NEXT:  1      2     1.00                        sri z31.b, z31.b, #8
 # CHECK-NEXT:  1      2     1.00                        sri z31.d, z31.d, #64
 # CHECK-NEXT:  1      2     1.00                        sri z31.h, z31.h, #16
 # CHECK-NEXT:  1      2     1.00                        sri z31.s, z31.s, #32
 # CHECK-NEXT:  1      4     1.00                        srshl z0.b, p0/m, z0.b, z1.b
 # CHECK-NEXT:  1      4     1.00                        srshl z0.h, p0/m, z0.h, z1.h
 # CHECK-NEXT:  1      4     1.00                        srshl z29.s, p7/m, z29.s, z30.s
 # CHECK-NEXT:  1      4     1.00                        srshl z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  1      4     1.00                  U     srshlr  z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  1      4     1.00                  U     srshlr  z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  1      4     1.00                  U     srshlr  z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  1      4     1.00                  U     srshlr  z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        srshlr  z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        srshlr  z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        srshlr  z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        srshlr  z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        srshr z0.b, p0/m, z0.b, #1
 # CHECK-NEXT:  1      4     1.00                        srshr z0.d, p0/m, z0.d, #1
 # CHECK-NEXT:  1      4     1.00                        srshr z0.h, p0/m, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        srshr z0.s, p0/m, z0.s, #1
 # CHECK-NEXT:  1      4     1.00                        srshr z31.b, p0/m, z31.b, #8
 # CHECK-NEXT:  1      4     1.00                        srshr z31.d, p0/m, z31.d, #64
 # CHECK-NEXT:  1      4     1.00                        srshr z31.h, p0/m, z31.h, #16
 # CHECK-NEXT:  1      4     1.00                        srshr z31.s, p0/m, z31.s, #32

 # CHECK-NEXT:  2      2     0.50           *            st1w  { z0.s }, p0, [x0]
 # CHECK-NEXT:  4      4     1.00           *            st1w  { z0.s }, p7, [z0.s]
 # CHECK-NEXT:  2      2     0.50           *            st1w  { z21.d }, p5, [x10, #5, mul vl]
 # CHECK-NEXT:  2      2     0.50           *            st1w  { z21.s }, p5, [x10, #5, mul vl]
 # CHECK-NEXT:  2      2     0.50           *            st1w  { z31.d }, p7, [sp, #-1, mul vl]
 # CHECK-NEXT:  2      2     0.50           *            st1w  { z31.d }, p7, [z31.d, #124]
 # CHECK-NEXT:  2      2     0.50           *            st1w  { z31.s }, p7, [sp, #-1, mul vl]
 # CHECK-NEXT:  4      4     1.00           *            st1w  { z31.s }, p7, [z31.s, #124]
-# CHECK-NEXT:  2      4     0.50           *      U     st2b  { z0.b, z1.b }, p0, [x0, x0]
-# CHECK-NEXT:  2      4     0.50           *      U     st2b  { z0.b, z1.b }, p0, [x0]
-# CHECK-NEXT:  2      4     0.50           *      U     st2b  { z21.b, z22.b }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  2      4     0.50           *      U     st2b  { z23.b, z24.b }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      4     0.50           *      U     st2b  { z5.b, z6.b }, p3, [x17, x16]
-# CHECK-NEXT:  2      4     0.50           *      U     st2d  { z0.d, z1.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  2      4     0.50           *      U     st2d  { z0.d, z1.d }, p0, [x0]
-# CHECK-NEXT:  2      4     0.50           *      U     st2d  { z21.d, z22.d }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  2      4     0.50           *      U     st2d  { z23.d, z24.d }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      4     0.50           *      U     st2d  { z5.d, z6.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  3      4     0.50           *      U     st2h  { z0.h, z1.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  2      4     0.50           *      U     st2h  { z0.h, z1.h }, p0, [x0]
-# CHECK-NEXT:  2      4     0.50           *      U     st2h  { z21.h, z22.h }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  2      4     0.50           *      U     st2h  { z23.h, z24.h }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  3      4     0.50           *      U     st2h  { z5.h, z6.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  2      4     0.50           *      U     st2w  { z0.s, z1.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  2      4     0.50           *      U     st2w  { z0.s, z1.s }, p0, [x0]
-# CHECK-NEXT:  2      4     0.50           *      U     st2w  { z21.s, z22.s }, p5, [x10, #10, mul vl]
-# CHECK-NEXT:  2      4     0.50           *      U     st2w  { z23.s, z24.s }, p3, [x13, #-16, mul vl]
-# CHECK-NEXT:  2      4     0.50           *      U     st2w  { z5.s, z6.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  15     7     2.50           *      U     st3b  { z0.b - z2.b }, p0, [x0, x0]
-# CHECK-NEXT:  10     7     2.50           *      U     st3b  { z0.b - z2.b }, p0, [x0]
-# CHECK-NEXT:  10     7     2.50           *      U     st3b  { z21.b - z23.b }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  10     7     2.50           *      U     st3b  { z23.b - z25.b }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  15     7     2.50           *      U     st3b  { z5.b - z7.b }, p3, [x17, x16]
-# CHECK-NEXT:  15     7     2.50           *      U     st3d  { z0.d - z2.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  10     7     2.50           *      U     st3d  { z0.d - z2.d }, p0, [x0]
-# CHECK-NEXT:  10     7     2.50           *      U     st3d  { z21.d - z23.d }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  10     7     2.50           *      U     st3d  { z23.d - z25.d }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  15     7     2.50           *      U     st3d  { z5.d - z7.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  15     7     2.50           *      U     st3h  { z0.h - z2.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  10     7     2.50           *      U     st3h  { z0.h - z2.h }, p0, [x0]
-# CHECK-NEXT:  10     7     2.50           *      U     st3h  { z21.h - z23.h }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  10     7     2.50           *      U     st3h  { z23.h - z25.h }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  15     7     2.50           *      U     st3h  { z5.h - z7.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  15     7     2.50           *      U     st3w  { z0.s - z2.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  10     7     2.50           *      U     st3w  { z0.s - z2.s }, p0, [x0]
-# CHECK-NEXT:  10     7     2.50           *      U     st3w  { z21.s - z23.s }, p5, [x10, #15, mul vl]
-# CHECK-NEXT:  10     7     2.50           *      U     st3w  { z23.s - z25.s }, p3, [x13, #-24, mul vl]
-# CHECK-NEXT:  15     7     2.50           *      U     st3w  { z5.s - z7.s }, p3, [x17, x16, lsl #2]
-# CHECK-NEXT:  27     11    4.50           *      U     st4b  { z0.b - z3.b }, p0, [x0, x0]
-# CHECK-NEXT:  18     11    4.50           *      U     st4b  { z0.b - z3.b }, p0, [x0]
-# CHECK-NEXT:  18     11    4.50           *      U     st4b  { z21.b - z24.b }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  18     11    4.50           *      U     st4b  { z23.b - z26.b }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  27     11    4.50           *      U     st4b  { z5.b - z8.b }, p3, [x17, x16]
-# CHECK-NEXT:  27     11    4.50           *      U     st4d  { z0.d - z3.d }, p0, [x0, x0, lsl #3]
-# CHECK-NEXT:  18     11    4.50           *      U     st4d  { z0.d - z3.d }, p0, [x0]
-# CHECK-NEXT:  18     11    4.50           *      U     st4d  { z21.d - z24.d }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  18     11    4.50           *      U     st4d  { z23.d - z26.d }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  27     11    4.50           *      U     st4d  { z5.d - z8.d }, p3, [x17, x16, lsl #3]
-# CHECK-NEXT:  27     11    4.50           *      U     st4h  { z0.h - z3.h }, p0, [x0, x0, lsl #1]
-# CHECK-NEXT:  18     11    4.50           *      U     st4h  { z0.h - z3.h }, p0, [x0]
-# CHECK-NEXT:  18     11    4.50           *      U     st4h  { z21.h - z24.h }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  18     11    4.50           *      U     st4h  { z23.h - z26.h }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  27     11    4.50           *      U     st4h  { z5.h - z8.h }, p3, [x17, x16, lsl #1]
-# CHECK-NEXT:  27     11    4.50           *      U     st4w  { z0.s - z3.s }, p0, [x0, x0, lsl #2]
-# CHECK-NEXT:  18     11    4.50           *      U     st4w  { z0.s - z3.s }, p0, [x0]
-# CHECK-NEXT:  18     11    4.50           *      U     st4w  { z21.s - z24.s }, p5, [x10, #20, mul vl]
-# CHECK-NEXT:  18     11    4.50           *      U     st4w  { z23.s - z26.s }, p3, [x13, #-32, mul vl]
-# CHECK-NEXT:  27     11    4.50           *      U     st4w  { z5.s - z8.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  2      4     0.50           *            st2b  { z0.b, z1.b }, p0, [x0, x0]
+# CHECK-NEXT:  2      4     0.50           *            st2b  { z0.b, z1.b }, p0, [x0]
+# CHECK-NEXT:  2      4     0.50           *            st2b  { z21.b, z22.b }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  2      4     0.50           *            st2b  { z23.b, z24.b }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      4     0.50           *            st2b  { z5.b, z6.b }, p3, [x17, x16]
+# CHECK-NEXT:  2      4     0.50           *            st2d  { z0.d, z1.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  2      4     0.50           *            st2d  { z0.d, z1.d }, p0, [x0]
+# CHECK-NEXT:  2      4     0.50           *            st2d  { z21.d, z22.d }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  2      4     0.50           *            st2d  { z23.d, z24.d }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      4     0.50           *            st2d  { z5.d, z6.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  3      4     0.50           *            st2h  { z0.h, z1.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  2      4     0.50           *            st2h  { z0.h, z1.h }, p0, [x0]
+# CHECK-NEXT:  2      4     0.50           *            st2h  { z21.h, z22.h }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  2      4     0.50           *            st2h  { z23.h, z24.h }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  3      4     0.50           *            st2h  { z5.h, z6.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  2      4     0.50           *            st2w  { z0.s, z1.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  2      4     0.50           *            st2w  { z0.s, z1.s }, p0, [x0]
+# CHECK-NEXT:  2      4     0.50           *            st2w  { z21.s, z22.s }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  2      4     0.50           *            st2w  { z23.s, z24.s }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  2      4     0.50           *            st2w  { z5.s, z6.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  15     7     2.50           *            st3b  { z0.b - z2.b }, p0, [x0, x0]
+# CHECK-NEXT:  10     7     2.50           *            st3b  { z0.b - z2.b }, p0, [x0]
+# CHECK-NEXT:  10     7     2.50           *            st3b  { z21.b - z23.b }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  10     7     2.50           *            st3b  { z23.b - z25.b }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  15     7     2.50           *            st3b  { z5.b - z7.b }, p3, [x17, x16]
+# CHECK-NEXT:  15     7     2.50           *            st3d  { z0.d - z2.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  10     7     2.50           *            st3d  { z0.d - z2.d }, p0, [x0]
+# CHECK-NEXT:  10     7     2.50           *            st3d  { z21.d - z23.d }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  10     7     2.50           *            st3d  { z23.d - z25.d }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  15     7     2.50           *            st3d  { z5.d - z7.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  15     7     2.50           *            st3h  { z0.h - z2.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  10     7     2.50           *            st3h  { z0.h - z2.h }, p0, [x0]
+# CHECK-NEXT:  10     7     2.50           *            st3h  { z21.h - z23.h }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  10     7     2.50           *            st3h  { z23.h - z25.h }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  15     7     2.50           *            st3h  { z5.h - z7.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  15     7     2.50           *            st3w  { z0.s - z2.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  10     7     2.50           *            st3w  { z0.s - z2.s }, p0, [x0]
+# CHECK-NEXT:  10     7     2.50           *            st3w  { z21.s - z23.s }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  10     7     2.50           *            st3w  { z23.s - z25.s }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  15     7     2.50           *            st3w  { z5.s - z7.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  27     11    4.50           *            st4b  { z0.b - z3.b }, p0, [x0, x0]
+# CHECK-NEXT:  18     11    4.50           *            st4b  { z0.b - z3.b }, p0, [x0]
+# CHECK-NEXT:  18     11    4.50           *            st4b  { z21.b - z24.b }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  18     11    4.50           *            st4b  { z23.b - z26.b }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  27     11    4.50           *            st4b  { z5.b - z8.b }, p3, [x17, x16]
+# CHECK-NEXT:  27     11    4.50           *            st4d  { z0.d - z3.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  18     11    4.50           *            st4d  { z0.d - z3.d }, p0, [x0]
+# CHECK-NEXT:  18     11    4.50           *            st4d  { z21.d - z24.d }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  18     11    4.50           *            st4d  { z23.d - z26.d }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  27     11    4.50           *            st4d  { z5.d - z8.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  27     11    4.50           *            st4h  { z0.h - z3.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  18     11    4.50           *            st4h  { z0.h - z3.h }, p0, [x0]
+# CHECK-NEXT:  18     11    4.50           *            st4h  { z21.h - z24.h }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  18     11    4.50           *            st4h  { z23.h - z26.h }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  27     11    4.50           *            st4h  { z5.h - z8.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  27     11    4.50           *            st4w  { z0.s - z3.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  18     11    4.50           *            st4w  { z0.s - z3.s }, p0, [x0]
+# CHECK-NEXT:  18     11    4.50           *            st4w  { z21.s - z24.s }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  18     11    4.50           *            st4w  { z23.s - z26.s }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  27     11    4.50           *            st4w  { z5.s - z8.s }, p3, [x17, x16, lsl #2]
 # CHECK-NEXT:  2      2     0.50           *            stnt1b  { z0.b }, p0, [x0, x0]
 # CHECK-NEXT:  2      2     0.50           *            stnt1b  { z0.b }, p0, [x0]
 # CHECK-NEXT:  2      2     0.50           *            stnt1b  { z0.d }, p0, [z1.d]
 # CHECK-NEXT:  4      4     1.00           *            stnt1b  { z0.s }, p0, [z1.s]
 # CHECK-NEXT:  2      2     0.50           *            stnt1b  { z21.b }, p5, [x10, #7, mul vl]
 # CHECK-NEXT:  2      2     0.50           *            stnt1b  { z23.b }, p3, [x13, #-8, mul vl]
 # CHECK-NEXT:  2      2     0.50           *            stnt1b  { z31.d }, p7, [z31.d, x0]
 # CHECK-NEXT:  2      2     0.50           *            stnt1b  { z31.d }, p7, [z31.d]

 # CHECK-NEXT:  2      2     0.50           *            stnt1w  { z23.s }, p3, [x13, #-8, mul vl]
 # CHECK-NEXT:  2      2     0.50           *            stnt1w  { z31.d }, p7, [z31.d, x0]
 # CHECK-NEXT:  2      2     0.50           *            stnt1w  { z31.d }, p7, [z31.d]
 # CHECK-NEXT:  4      4     1.00           *            stnt1w  { z31.s }, p7, [z31.s, x0]
 # CHECK-NEXT:  4      4     1.00           *            stnt1w  { z31.s }, p7, [z31.s]
 # CHECK-NEXT:  1      1     0.50           *            str p0, [x0]
 # CHECK-NEXT:  1      1     0.50           *            str p15, [sp, #-256, mul vl]
 # CHECK-NEXT:  1      1     0.50           *            str p5, [x10, #255, mul vl]
-# CHECK-NEXT:  2      2     0.50           *      U     str z0, [x0]
-# CHECK-NEXT:  2      2     0.50           *      U     str z21, [x10, #-256, mul vl]
-# CHECK-NEXT:  2      2     0.50           *      U     str z31, [sp, #255, mul vl]
+# CHECK-NEXT:  2      2     0.50           *            str z0, [x0]
+# CHECK-NEXT:  2      2     0.50           *            str z21, [x10, #-256, mul vl]
+# CHECK-NEXT:  2      2     0.50           *            str z31, [sp, #255, mul vl]
 # CHECK-NEXT:  1      2     0.50                        sub z0.b, p0/m, z0.b, z0.b
 # CHECK-NEXT:  1      2     0.50                        sub z0.b, z0.b, #0
 # CHECK-NEXT:  1      2     0.50                        sub z0.b, z0.b, z0.b
 # CHECK-NEXT:  1      2     0.50                        sub z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  1      2     0.50                        sub z0.d, z0.d, #0
 # CHECK-NEXT:  1      2     0.50                        sub z0.d, z0.d, #0, lsl #8
 # CHECK-NEXT:  1      2     0.50                        sub z0.d, z0.d, z0.d
 # CHECK-NEXT:  1      2     0.50                        sub z0.h, p0/m, z0.h, z0.h

 # CHECK-NEXT:  1      2     1.00                        sxtb  z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      2     1.00                        sxtb  z31.s, p7/m, z31.s
 # CHECK-NEXT:  1      2     1.00                        sxth  z0.d, p0/m, z0.d
 # CHECK-NEXT:  1      2     1.00                        sxth  z0.s, p0/m, z0.s
 # CHECK-NEXT:  1      2     1.00                        sxth  z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      2     1.00                        sxth  z31.s, p7/m, z31.s
 # CHECK-NEXT:  1      2     1.00                        sxtw  z0.d, p0/m, z0.d
 # CHECK-NEXT:  1      2     1.00                        sxtw  z31.d, p7/m, z31.d
-# CHECK-NEXT:  1      2     0.50                  U     tbl z28.b, { z29.b, z30.b }, z31.b
-# CHECK-NEXT:  1      2     0.50                  U     tbl z28.d, { z29.d, z30.d }, z31.d
-# CHECK-NEXT:  1      2     0.50                  U     tbl z28.h, { z29.h, z30.h }, z31.h
-# CHECK-NEXT:  1      2     0.50                  U     tbl z28.s, { z29.s, z30.s }, z31.s
+# CHECK-NEXT:  1      2     0.50                        tbl z28.b, { z29.b, z30.b }, z31.b
+# CHECK-NEXT:  1      2     0.50                        tbl z28.d, { z29.d, z30.d }, z31.d
+# CHECK-NEXT:  1      2     0.50                        tbl z28.h, { z29.h, z30.h }, z31.h
+# CHECK-NEXT:  1      2     0.50                        tbl z28.s, { z29.s, z30.s }, z31.s
 # CHECK-NEXT:  1      2     0.50                        tbl z31.b, { z31.b }, z31.b
 # CHECK-NEXT:  1      2     0.50                        tbl z31.d, { z31.d }, z31.d
 # CHECK-NEXT:  1      2     0.50                        tbl z31.h, { z31.h }, z31.h
 # CHECK-NEXT:  1      2     0.50                        tbl z31.s, { z31.s }, z31.s
 # CHECK-NEXT:  1      2     0.50                        tbx z31.b, z31.b, z31.b
 # CHECK-NEXT:  1      2     0.50                        tbx z31.d, z31.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        tbx z31.h, z31.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        tbx z31.s, z31.s, z31.s

 # CHECK-NEXT:  1      2     1.00                        uqincw  z0.s
 # CHECK-NEXT:  1      2     1.00                        uqincw  z0.s, all, mul #16
 # CHECK-NEXT:  1      2     1.00                        uqincw  z0.s, pow2
 # CHECK-NEXT:  1      2     1.00                        uqincw  z0.s, pow2, mul #16
 # CHECK-NEXT:  1      4     1.00                        uqrshl  z0.b, p0/m, z0.b, z1.b
 # CHECK-NEXT:  1      4     1.00                        uqrshl  z0.h, p0/m, z0.h, z1.h
 # CHECK-NEXT:  1      4     1.00                        uqrshl  z29.s, p7/m, z29.s, z30.s
 # CHECK-NEXT:  1      4     1.00                        uqrshl  z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  1      4     1.00                  U     uqrshlr z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  1      4     1.00                  U     uqrshlr z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  1      4     1.00                  U     uqrshlr z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  1      4     1.00                  U     uqrshlr z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqrshlr z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqrshlr z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqrshlr z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqrshlr z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        uqrshrnb  z0.b, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        uqrshrnb  z0.h, z0.s, #1
 # CHECK-NEXT:  1      4     1.00                        uqrshrnb  z0.s, z0.d, #1
 # CHECK-NEXT:  1      4     1.00                        uqrshrnb  z31.b, z31.h, #8
 # CHECK-NEXT:  1      4     1.00                        uqrshrnb  z31.h, z31.s, #16
 # CHECK-NEXT:  1      4     1.00                        uqrshrnb  z31.s, z31.d, #32
 # CHECK-NEXT:  1      4     1.00                        uqrshrnt  z0.b, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        uqrshrnt  z0.h, z0.s, #1
 # CHECK-NEXT:  1      4     1.00                        uqrshrnt  z0.s, z0.d, #1
 # CHECK-NEXT:  1      4     1.00                        uqrshrnt  z31.b, z31.h, #8
 # CHECK-NEXT:  1      4     1.00                        uqrshrnt  z31.h, z31.s, #16
 # CHECK-NEXT:  1      4     1.00                        uqrshrnt  z31.s, z31.d, #32
 # CHECK-NEXT:  1      4     1.00                        uqshl z0.b, p0/m, z0.b, #0
 # CHECK-NEXT:  1      4     1.00                        uqshl z0.b, p0/m, z0.b, z1.b
 # CHECK-NEXT:  1      4     1.00                        uqshl z0.d, p0/m, z0.d, #0
 # CHECK-NEXT:  1      4     1.00                        uqshl z0.h, p0/m, z0.h, #0
 # CHECK-NEXT:  1      4     1.00                        uqshl z0.h, p0/m, z0.h, z1.h
 # CHECK-NEXT:  1      4     1.00                        uqshl z0.s, p0/m, z0.s, #0
 # CHECK-NEXT:  1      4     1.00                        uqshl z29.s, p7/m, z29.s, z30.s
 # CHECK-NEXT:  1      4     1.00                        uqshl z31.b, p0/m, z31.b, #7
 # CHECK-NEXT:  1      4     1.00                        uqshl z31.d, p0/m, z31.d, #63
 # CHECK-NEXT:  1      4     1.00                        uqshl z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        uqshl z31.h, p0/m, z31.h, #15
 # CHECK-NEXT:  1      4     1.00                        uqshl z31.s, p0/m, z31.s, #31
-# CHECK-NEXT:  1      4     1.00                  U     uqshlr  z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  1      4     1.00                  U     uqshlr  z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  1      4     1.00                  U     uqshlr  z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  1      4     1.00                  U     uqshlr  z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqshlr  z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqshlr  z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqshlr  z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqshlr  z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        uqshrnb z0.b, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        uqshrnb z0.h, z0.s, #1
 # CHECK-NEXT:  1      4     1.00                        uqshrnb z0.s, z0.d, #1
 # CHECK-NEXT:  1      4     1.00                        uqshrnb z31.b, z31.h, #8
 # CHECK-NEXT:  1      4     1.00                        uqshrnb z31.h, z31.s, #16
 # CHECK-NEXT:  1      4     1.00                        uqshrnb z31.s, z31.d, #32
 # CHECK-NEXT:  1      4     1.00                        uqshrnt z0.b, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        uqshrnt z0.h, z0.s, #1

 # CHECK-NEXT:  1      2     0.50                        urhadd  z0.b, p0/m, z0.b, z1.b
 # CHECK-NEXT:  1      2     0.50                        urhadd  z0.h, p0/m, z0.h, z1.h
 # CHECK-NEXT:  1      2     0.50                        urhadd  z29.s, p7/m, z29.s, z30.s
 # CHECK-NEXT:  1      2     0.50                        urhadd  z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        urshl z0.b, p0/m, z0.b, z1.b
 # CHECK-NEXT:  1      4     1.00                        urshl z0.h, p0/m, z0.h, z1.h
 # CHECK-NEXT:  1      4     1.00                        urshl z29.s, p7/m, z29.s, z30.s
 # CHECK-NEXT:  1      4     1.00                        urshl z31.d, p7/m, z31.d, z30.d
-# CHECK-NEXT:  1      4     1.00                  U     urshlr  z0.b, p0/m, z0.b, z1.b
-# CHECK-NEXT:  1      4     1.00                  U     urshlr  z0.h, p0/m, z0.h, z1.h
-# CHECK-NEXT:  1      4     1.00                  U     urshlr  z29.s, p7/m, z29.s, z30.s
-# CHECK-NEXT:  1      4     1.00                  U     urshlr  z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        urshlr  z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        urshlr  z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        urshlr  z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        urshlr  z31.d, p7/m, z31.d, z30.d
 # CHECK-NEXT:  1      4     1.00                        urshr z0.b, p0/m, z0.b, #1
 # CHECK-NEXT:  1      4     1.00                        urshr z0.d, p0/m, z0.d, #1
 # CHECK-NEXT:  1      4     1.00                        urshr z0.h, p0/m, z0.h, #1
 # CHECK-NEXT:  1      4     1.00                        urshr z0.s, p0/m, z0.s, #1
 # CHECK-NEXT:  1      4     1.00                        urshr z31.b, p0/m, z31.b, #8
 # CHECK-NEXT:  1      4     1.00                        urshr z31.d, p0/m, z31.d, #64
 # CHECK-NEXT:  1      4     1.00                        urshr z31.h, p0/m, z31.h, #16
 # CHECK-NEXT:  1      4     1.00                        urshr z31.s, p0/m, z31.s, #32

 # CHECK-NEXT:  1      3     0.50                        whilege p15.b, x0, xzr
 # CHECK-NEXT:  1      3     0.50                        whilege p15.b, xzr, x0
 # CHECK-NEXT:  1      3     0.50                        whilege p15.d, w0, wzr
 # CHECK-NEXT:  1      3     0.50                        whilege p15.d, x0, xzr
 # CHECK-NEXT:  1      3     0.50                        whilege p15.h, w0, wzr
 # CHECK-NEXT:  1      3     0.50                        whilege p15.h, x0, xzr
 # CHECK-NEXT:  1      3     0.50                        whilege p15.s, w0, wzr
 # CHECK-NEXT:  1      3     0.50                        whilege p15.s, x0, xzr
-# CHECK-NEXT:  1      3     0.50    *      *      U     whilerw p15.b, x30, x30
-# CHECK-NEXT:  1      3     0.50    *      *      U     whilerw p15.d, x30, x30
-# CHECK-NEXT:  1      3     0.50    *      *      U     whilerw p15.h, x30, x30
-# CHECK-NEXT:  1      3     0.50    *      *      U     whilerw p15.s, x30, x30
-# CHECK-NEXT:  1      3     0.50    *      *      U     whilewr p15.b, x30, x30
-# CHECK-NEXT:  1      3     0.50    *      *      U     whilewr p15.d, x30, x30
-# CHECK-NEXT:  1      3     0.50    *      *      U     whilewr p15.h, x30, x30
-# CHECK-NEXT:  1      3     0.50    *      *      U     whilewr p15.s, x30, x30
+# CHECK-NEXT:  1      3     0.50                        whilerw p15.b, x30, x30
+# CHECK-NEXT:  1      3     0.50                        whilerw p15.d, x30, x30
+# CHECK-NEXT:  1      3     0.50                        whilerw p15.h, x30, x30
+# CHECK-NEXT:  1      3     0.50                        whilerw p15.s, x30, x30
+# CHECK-NEXT:  1      3     0.50                        whilewr p15.b, x30, x30
+# CHECK-NEXT:  1      3     0.50                        whilewr p15.d, x30, x30
+# CHECK-NEXT:  1      3     0.50                        whilewr p15.h, x30, x30
+# CHECK-NEXT:  1      3     0.50                        whilewr p15.s, x30, x30
 # CHECK-NEXT:  1      2     2.00           *      U     wrffr p0.b
 # CHECK-NEXT:  1      2     2.00           *      U     wrffr p15.b
 # CHECK-NEXT:  1      2     1.00                        xar z0.b, z0.b, z1.b, #1
 # CHECK-NEXT:  1      2     1.00                        xar z0.d, z0.d, z1.d, #1
 # CHECK-NEXT:  1      2     1.00                        xar z0.h, z0.h, z1.h, #1
 # CHECK-NEXT:  1      2     1.00                        xar z0.s, z0.s, z1.s, #1
 # CHECK-NEXT:  1      2     1.00                        xar z31.b, z31.b, z30.b, #8
 # CHECK-NEXT:  1      2     1.00                        xar z31.d, z31.d, z30.d, #64