[ARM][MVE] Tail predicate VMAXV(unsigned) and VMAXAV

[samparker]

Summary

With predicated false lanes now tracked to guarantee zeros in the false bytes, we can allow vmaxv.u* and vmaxav instructions to be tail predicated if the argument requirements are met.

[SjoerdMeijer]

The unit test failure looks genuine, does that needs fixing?

A minor question inlined.

llvm/lib/Target/ARM/ARMInstrMVE.td

850

same for these 3 (why not suitable for TP?)

897

I am being lazy here (haven't checked the ARMARM), but why is this vmin not suitable for TP?

[samparker]

The unit test failure looks genuine, does that needs fixing?

Ah, I haven't updated the unit test in D76708.

For min/max, we can't support an implicit vmin because the results may not be the same after the conversion. So, say we only have three 32-bit elements left to process (and the fourth element is the LHS 0x00):

opcode	input	original result	tail predicated result
VMAXV.u32	0x00010203	0x03	0x03
VMINV.u32	0x00010203	0x00	0x01

The tail predicated instruction will ignore the predicated lanes/bytes, whereas the original doesn't.
We're also only supporting unsigned values because we know that the 'FalseLaneZeros' can't interfere with the result, because the zero will only be the answer if the rest of the elements are also zero. This is not true for signed values though, where the false zero may be the largest value.

[SjoerdMeijer]

Thanks for that example! I asked this question because I expected the vmax and vmin to behave roughly the same. In your example, if you change the input and example from 0x00010203 to 0x04010203, then the VMAX will also give a different result after tail-predication, or am I still missing something?

[samparker]

if you change the input and example from 0x00010203 to 0x04010203, then the VMAX will also give a different result after tail-predication

Indeed! Which is why we track for our zero'd false lanes. With vmax being a horizontal operations, we check that it operates upon on registers that we know have zero'd false lanes.

[SjoerdMeijer]

In D76716#1940911, @samparker wrote:

if you change the input and example from 0x00010203 to 0x04010203, then the VMAX will also give a different result after tail-predication

Indeed! Which is why we track for our zero'd false lanes. With vmax being a horizontal operations, we check that it operates upon on registers that we know have zero'd false lanes.

Ah yes, this was before my first coffee...!

[samparker]

Using predicate logic to determine validity.

[SjoerdMeijer]

I was going through my Phabricator list... is still relevant?

[samparker]

Maybe? Not something that I'll be continuing with though.

[SjoerdMeijer]

@dmgreen , interesting for you?

llvm/lib/Target/ARM/ARMInstrMVE.td

   let Inst{16} = 0b0;
   let Inst{15-12} = RdaDest{3-0};
   let Inst{8} = 0b1;
   let Inst{7} = bit_7;
   let Inst{6-5} = 0b00;
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = 0b0;
   let horizontalReduction = 1;
+  // Enable for unsigned max, or max abs, instructions.
+  let validForTailPredication = !and(!or(!eq(U, 1), !eq(bit_17, 0)), !eq(bit_7, 0));
 }
 
 multiclass MVE_VMINMAXV_p<string iname, bit notAbs, bit isMin,
                           MVEVectorVTInfo VTI, string intrBaseName> {
   def "": MVE_VMINMAXV<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
                        notAbs, isMin>;
   defvar Inst        = !cast<Instruction>(NAME);
   defvar unpred_intr = !cast<Intrinsic>(intrBaseName);

 }
 
 multiclass MVE_VMINMAXV_ty<string iname, bit isMin, string intrBaseName> {
   defm s8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16s8, intrBaseName>;
   defm s16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8s16, intrBaseName>;
   defm s32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4s32, intrBaseName>;
   defm u8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16u8, intrBaseName>;
   defm u16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8u16, intrBaseName>;
   defm u32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4u32, intrBaseName>;
 }
 
 defm MVE_VMINV : MVE_VMINMAXV_ty<"vminv", 1, "int_arm_mve_minv">;
 defm MVE_VMAXV : MVE_VMINMAXV_ty<"vmaxv", 0, "int_arm_mve_maxv">;
 
 let Predicates = [HasMVEInt] in {
   def : Pat<(i32 (vecreduce_smax (v16i8 MQPR:$src))),
             (i32 (MVE_VMAXVs8 (t2MVNi (i32 127)), $src))>;

 }
 
 defm MVE_VMINAV : MVE_VMINMAXAV_ty<"vminav", 1, "int_arm_mve_minav">;
 defm MVE_VMAXAV : MVE_VMINMAXAV_ty<"vmaxav", 0, "int_arm_mve_maxav">;
 
 class MVE_VMLAMLSDAV<string iname, string suffix, dag iops, string cstr,
                    bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0>
   : MVE_rDest<(outs tGPREven:$RdaDest), iops, NoItinerary, iname, suffix,
               "$RdaDest, $Qn, $Qm", cstr, []> {
   bits<4> RdaDest;
   bits<3> Qm;
   bits<3> Qn;
 
   let Inst{28} = bit_28;
   let Inst{22-20} = 0b111;
   let Inst{19-17} = Qn{2-0};
   let Inst{16} = sz;

llvm/test/CodeGen/Thumb2/LowOverheadLoops/unpredicated-max.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve %s -run-pass=arm-low-overhead-loops -o - | FileCheck %s
 
 --- |
-  define dso_local void @variant_max_use(i16* nocapture readonly %a, i16* %c, i32 %N) #0 {
+  define dso_local void @variant_smax_use(i16* nocapture readonly %a, i16* %c, i32 %N) #0 {
   entry:
     %cmp9 = icmp eq i32 %N, 0
     %tmp = add i32 %N, 3
     %tmp1 = lshr i32 %tmp, 2
     %tmp2 = shl nuw i32 %tmp1, 2
     %tmp3 = add i32 %tmp2, -4
     %tmp4 = lshr i32 %tmp3, 2
     %tmp5 = add nuw nsw i32 %tmp4, 1
vector.body:                                      ; preds = %vector.body, %vector.ph
     %tmp11 = icmp ne i32 %tmp10, 0
     %lsr.iv.next = add nsw i32 %lsr.iv1, -1
     br i1 %tmp11, label %vector.body, label %exit
 
   exit:                                             ; preds = %vector.body, %entry
     ret void
   }
 
+  define dso_local void @variant_umax_use(i16* nocapture readonly %a, i16* %c, i32 %N) #0 {
+  entry:
+    %cmp9 = icmp eq i32 %N, 0
+    %tmp = add i32 %N, 3
+    %tmp1 = lshr i32 %tmp, 2
+    %tmp2 = shl nuw i32 %tmp1, 2
+    %tmp3 = add i32 %tmp2, -4
+    %tmp4 = lshr i32 %tmp3, 2
+    %tmp5 = add nuw nsw i32 %tmp4, 1
+    br i1 %cmp9, label %exit, label %vector.ph
+
+  vector.ph:                                        ; preds = %entry
+    call void @llvm.set.loop.iterations.i32(i32 %tmp5)
+    br label %vector.body
+
+  vector.body:                                      ; preds = %vector.body, %vector.ph
+    %lsr.iv1 = phi i32 [ %lsr.iv.next, %vector.body ], [ %tmp5, %vector.ph ]
+    %lsr.iv = phi i16* [ %scevgep, %vector.body ], [ %a, %vector.ph ]
+    %lsr.iv.2 = phi i16* [ %scevgep.2, %vector.body ], [ %c, %vector.ph ]
+    %tmp7 = phi i32 [ %N, %vector.ph ], [ %tmp9, %vector.body ]
+    %lsr.iv17 = bitcast i16* %lsr.iv to <8 x i16>*
+    %tmp8 = call <8 x i1> @llvm.arm.mve.vctp16(i32 %tmp7)
+    %tmp9 = sub i32 %tmp7, 8
+    %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %lsr.iv17, i32 2, <8 x i1> %tmp8, <8 x i16> undef)
+    %max = tail call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> %wide.masked.load)
+    store i16 %max, i16* %lsr.iv.2
+    %scevgep = getelementptr i16, i16* %lsr.iv, i32 8
+    %scevgep.2 = getelementptr i16, i16* %lsr.iv.2, i32 1
+    %tmp10 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %lsr.iv1, i32 1)
+    %tmp11 = icmp ne i32 %tmp10, 0
+    %lsr.iv.next = add nsw i32 %lsr.iv1, -1
+    br i1 %tmp11, label %vector.body, label %exit
+
+  exit:                                             ; preds = %vector.body, %entry
+    ret void
+  }
+
   declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
   declare void @llvm.set.loop.iterations.i32(i32)
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
   declare <8 x i1> @llvm.arm.mve.vctp16(i32)
   declare i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16>)
+  declare i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16>)
 
 ...
 ---
-name:            variant_max_use
+name:            variant_smax_use
 alignment:       2
 tracksRegLiveness: true
 registers:       []
 liveins:
   - { reg: '$r0', virtual-reg: '' }
   - { reg: '$r1', virtual-reg: '' }
   - { reg: '$r2', virtual-reg: '' }
 frameInfo:
   stackSize:       8
   offsetAdjustment: 0
   maxAlignment:    4
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
       stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false,
       debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
   - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4,
       stack-id: default, callee-saved-register: '$r5', callee-saved-restored: true,
       debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
 callSites:       []
 constants:       []
 machineFunctionInfo: {}
 body:             |
-  ; CHECK-LABEL: name: variant_max_use
+  ; CHECK-LABEL: name: variant_smax_use
   ; CHECK: bb.0.entry:
   ; CHECK:   successors: %bb.1(0x80000000)
   ; CHECK:   liveins: $lr, $r0, $r1, $r2, $r5
   ; CHECK:   frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r5, killed $lr, implicit-def $sp, implicit $sp
   ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $r5, -8
   ; CHECK:   tCMPi8 renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
   ; CHECK:   t2IT 0, 8, implicit-def $itstate
-  ; CHECK:   tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r5, def $pc, implicit killed $itstate
+  ; CHECK:   frame-destroy tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r5, def $pc, implicit killed $itstate
   ; CHECK:   renamable $r3, dead $cpsr = tADDi3 renamable $r2, 3, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $r3 = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r12 = t2MOVi16 32768, 14 /* CC::al */, $noreg
   ; CHECK:   $r12 = t2MOVTi16 killed $r12, 65535, 14 /* CC::al */, $noreg
   ; CHECK:   dead $lr = t2DLS renamable $r3
   ; CHECK:   $r5 = tMOVr killed $r3, 14 /* CC::al */, $noreg
   ; CHECK: bb.1.vector.body:
   ; CHECK:   successors: %bb.1(0x7c000000), %bb.2(0x04000000)
   ; CHECK:   liveins: $r0, $r1, $r2, $r5, $r12
   ; CHECK:   $r3 = tMOVr $r12, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $vpr = MVE_VCTP16 renamable $r2, 0, $noreg
   ; CHECK:   MVE_VPST 8, implicit $vpr
   ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRHU16_post killed renamable $r0, 16, 1, killed renamable $vpr :: (load 16 from %ir.lsr.iv17, align 2)
   ; CHECK:   renamable $r3 = MVE_VMAXVs16 killed renamable $r3, killed renamable $q0, 0, $noreg
   ; CHECK:   $lr = tMOVr $r5, 14 /* CC::al */, $noreg
   ; CHECK:   early-clobber renamable $r1 = t2STRH_POST killed renamable $r3, killed renamable $r1, 2, 14 /* CC::al */, $noreg :: (store 2 into %ir.lsr.iv.2)
   ; CHECK:   renamable $r5, dead $cpsr = nsw tSUBi8 killed $r5, 1, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 8, 14 /* CC::al */, $noreg
   ; CHECK:   dead $lr = t2LEUpdate killed renamable $lr, %bb.1
   ; CHECK: bb.2.exit:
-  ; CHECK:   tPOP_RET 14 /* CC::al */, $noreg, def $r5, def $pc
+  ; CHECK:   frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r5, def $pc
   bb.0.entry:
     successors: %bb.1(0x80000000)
     liveins: $r0, $r1, $r2, $r5, $lr
 
     frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r5, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
     frame-setup CFI_INSTRUCTION offset $lr, -4
     frame-setup CFI_INSTRUCTION offset $r5, -8
     tCMPi8 renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
     t2IT 0, 8, implicit-def $itstate
-    tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r5, def $pc, implicit killed $itstate
+    frame-destroy tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r5, def $pc, implicit killed $itstate
     renamable $r3, dead $cpsr = tADDi3 renamable $r2, 3, 14 /* CC::al */, $noreg
     renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
     renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg
     renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
     renamable $r3 = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
     $r12 = t2MOVi16 32768, 14 /* CC::al */, $noreg
     $r12 = t2MOVTi16 killed $r12, 65535, 14 /* CC::al */, $noreg
     t2DoLoopStart renamable $r3
bb.1.vector.body:
     early-clobber renamable $r1 = t2STRH_POST killed renamable $r3, killed renamable $r1, 2, 14 /* CC::al */, $noreg :: (store 2 into %ir.lsr.iv.2)
     renamable $r5, dead $cpsr = nsw tSUBi8 killed $r5, 1, 14 /* CC::al */, $noreg
     renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 8, 14 /* CC::al */, $noreg
     renamable $lr = t2LoopDec killed renamable $lr, 1
     t2LoopEnd killed renamable $lr, %bb.1, implicit-def dead $cpsr
     tB %bb.2, 14 /* CC::al */, $noreg
 
   bb.2.exit:
-    tPOP_RET 14 /* CC::al */, $noreg, def $r5, def $pc
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r5, def $pc
+
+...
+---
+name:            variant_umax_use
+alignment:       2
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       8
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$r4', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  ; CHECK-LABEL: name: variant_umax_use
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $lr, $r0, $r1, $r2, $r4
+  ; CHECK:   frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r4, killed $lr, implicit-def $sp, implicit $sp
+  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $r4, -8
+  ; CHECK:   tCMPi8 renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   t2IT 0, 8, implicit-def $itstate
+  ; CHECK:   frame-destroy tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r4, def $pc, implicit killed $itstate
+  ; CHECK:   $lr = MVE_DLSTP_16 killed renamable $r2
+  ; CHECK: bb.1.vector.body:
+  ; CHECK:   successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+  ; CHECK:   liveins: $lr, $r0, $r1
+  ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRHU16_post killed renamable $r0, 16, 0, killed $noreg :: (load 16 from %ir.lsr.iv17, align 2)
+  ; CHECK:   renamable $r3 = MVE_VMAXVu16 killed renamable $r3, killed renamable $q0, 0, $noreg
+  ; CHECK:   early-clobber renamable $r1 = t2STRH_POST killed renamable $r3, killed renamable $r1, 2, 14 /* CC::al */, $noreg :: (store 2 into %ir.lsr.iv.2)
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.1
+  ; CHECK: bb.2.exit:
+  ; CHECK:   frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r4, def $pc
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r4, $lr
+
+    frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r4, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r4, -8
+    tCMPi8 renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    t2IT 0, 8, implicit-def $itstate
+    frame-destroy tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r4, def $pc, implicit killed $itstate
+    renamable $r3, dead $cpsr = tADDi3 renamable $r2, 3, 14 /* CC::al */, $noreg
+    renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
+    renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg
+    renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+    renamable $r3 = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
+    t2DoLoopStart renamable $r3
+    $r4 = tMOVr killed $r3, 14 /* CC::al */, $noreg
+
+  bb.1.vector.body:
+    successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+    liveins: $r0, $r1, $r2, $r4
+
+    renamable $r3, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
+    renamable $vpr = MVE_VCTP16 renamable $r2, 0, $noreg
+    MVE_VPST 8, implicit $vpr
+    renamable $r0, renamable $q0 = MVE_VLDRHU16_post killed renamable $r0, 16, 1, killed renamable $vpr :: (load 16 from %ir.lsr.iv17, align 2)
+    renamable $r3 = MVE_VMAXVu16 killed renamable $r3, killed renamable $q0, 0, $noreg
+    $lr = tMOVr $r4, 14 /* CC::al */, $noreg
+    early-clobber renamable $r1 = t2STRH_POST killed renamable $r3, killed renamable $r1, 2, 14 /* CC::al */, $noreg :: (store 2 into %ir.lsr.iv.2)
+    renamable $r4, dead $cpsr = nsw tSUBi8 killed $r4, 1, 14 /* CC::al */, $noreg
+    renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 8, 14 /* CC::al */, $noreg
+    renamable $lr = t2LoopDec killed renamable $lr, 1
+    t2LoopEnd killed renamable $lr, %bb.1, implicit-def dead $cpsr
+    tB %bb.2, 14 /* CC::al */, $noreg
+
+  bb.2.exit:
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r4, def $pc
 
 ...

llvm/unittests/Target/ARM/MachineInstrTest.cpp

     case MVE_VFMA_qr_Sf16:
     case MVE_VFMA_qr_Sf32:
     case MVE_VFMA_qr_f16:
     case MVE_VFMA_qr_f32:
     case MVE_VFMAf16:
     case MVE_VFMAf32:
     case MVE_VFMSf16:
     case MVE_VFMSf32:
+    case MVE_VMAXAVs16:
+    case MVE_VMAXAVs32:
+    case MVE_VMAXAVs8:
     case MVE_VMAXAs16:
     case MVE_VMAXAs32:
     case MVE_VMAXAs8:
     case MVE_VMAXs16:
     case MVE_VMAXs32:
     case MVE_VMAXs8:
     case MVE_VMAXu16:
     case MVE_VMAXu32:
     case MVE_VMAXu8:
+    case MVE_VMAXVu16:
+    case MVE_VMAXVu32:
+    case MVE_VMAXVu8:
     case MVE_VMINAs16:
     case MVE_VMINAs32:
     case MVE_VMINAs8:
     case MVE_VMINs16:
     case MVE_VMINs32:
     case MVE_VMINs8:
     case MVE_VMINu16:
     case MVE_VMINu32: