[ARM] MVE VMLAS

[dmgreen]

Summary

VMLAS Qda, Qn, Rm performs Qda = Qda * Qn + Rm, a similar pattern to the existing VMLA. The sinking of splat(Rm) into the loop is already performed, meaning we just need to select the pattern.

llvm/lib/Target/ARM/ARMInstrMVE.td

let Predicates = [HasMVEInt] in {
   def : Pat<(v8i16 (add (v8i16 MQPR:$src1),
                         (v8i16 (mul (v8i16 MQPR:$src2),
                                     (v8i16 (ARMvdup (i32 rGPR:$x))))))),
             (v8i16 (MVE_VMLA_qr_u16 $src1, $src2, $x))>;
   def : Pat<(v16i8 (add (v16i8 MQPR:$src1),
                         (v16i8 (mul (v16i8 MQPR:$src2),
                                     (v16i8 (ARMvdup (i32 rGPR:$x))))))),
             (v16i8 (MVE_VMLA_qr_u8 $src1, $src2, $x))>;
+
+  def : Pat<(v4i32 (add (v4i32 (ARMvdup (i32 rGPR:$x))),
+                        (v4i32 (mul (v4i32 MQPR:$src1),
+                                    (v4i32 MQPR:$src2))))),
+            (v4i32 (MVE_VMLAS_qr_u32 $src1, $src2, $x))>;
+  def : Pat<(v8i16 (add (v8i16 (ARMvdup (i32 rGPR:$x))),
+                        (v8i16 (mul (v8i16 MQPR:$src1),
+                                    (v8i16 MQPR:$src2))))),
+            (v8i16 (MVE_VMLAS_qr_u16 $src1, $src2, $x))>;
+  def : Pat<(v16i8 (add (v16i8 (ARMvdup (i32 rGPR:$x))),
+                        (v16i8 (mul (v16i8 MQPR:$src1),
+                                    (v16i8 MQPR:$src2))))),
+            (v16i8 (MVE_VMLAS_qr_u8 $src1, $src2, $x))>;
 }
 
 let Predicates = [HasMVEFloat] in {
   def MVE_VFMA_qr_f16  : MVE_VFMAMLA_qr<"vfma",  "f16", 0b1, 0b11, 0b0>;
   def MVE_VFMA_qr_f32  : MVE_VFMAMLA_qr<"vfma",  "f32", 0b0, 0b11, 0b0>;
   def MVE_VFMA_qr_Sf16 : MVE_VFMAMLA_qr<"vfmas", "f16", 0b1, 0b11, 0b1>;
   def MVE_VFMA_qr_Sf32 : MVE_VFMAMLA_qr<"vfmas", "f32", 0b0, 0b11, 0b1>;
 }

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll

 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB5_8
 ; CHECK-NEXT:  .LBB5_4: @ %vector.ph
 ; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB5_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u32 q0, [r0], #4
 ; CHECK-NEXT:    vldrb.u32 q1, [r1], #4
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, r2
-; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    vmlas.u32 q1, q0, r2
+; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB5_5
 ; CHECK-NEXT:    b .LBB5_11
 ; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r7, r12, #3
 ; CHECK-NEXT:    add.w r4, r3, #8
 ; CHECK-NEXT:    subs r7, #4
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    add.w lr, lr, r7, lsr #2

 ; CHECK-NEXT:    cmp.w r12, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r4, pc}
 ; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB6_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.s32 q0, [r0], #8
 ; CHECK-NEXT:    vldrh.s32 q1, [r1], #8
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, r2
-; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    vmlas.u32 q1, q0, r2
+; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB6_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %cmp10 = icmp eq i32 %N, 0
   br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry

 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB7_8
 ; CHECK-NEXT:  .LBB7_4: @ %vector.ph
 ; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB7_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u32 q0, [r0], #4
 ; CHECK-NEXT:    vldrb.u32 q1, [r1], #4
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, r2
-; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    vmlas.u32 q1, q0, r2
+; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB7_5
 ; CHECK-NEXT:    b .LBB7_11
 ; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r7, r12, #3
 ; CHECK-NEXT:    add.w r4, r3, #8
 ; CHECK-NEXT:    subs r7, #4
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    add.w lr, lr, r7, lsr #2

 ; CHECK-NEXT:    cmp.w r12, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r4, pc}
 ; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB8_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u32 q0, [r0], #8
 ; CHECK-NEXT:    vldrh.u32 q1, [r1], #8
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, r2
-; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    vmlas.u32 q1, q0, r2
+; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB8_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %cmp10 = icmp eq i32 %N, 0
   br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry

 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB9_8
 ; CHECK-NEXT:  .LBB9_4: @ %vector.ph
 ; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB9_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, r2
-; CHECK-NEXT:    vstrw.32 q0, [r3], #16
+; CHECK-NEXT:    vmlas.u32 q1, q0, r2
+; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB9_5
 ; CHECK-NEXT:    b .LBB9_11
 ; CHECK-NEXT:  .LBB9_6: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r5, r12, #3
 ; CHECK-NEXT:    add.w r4, r3, #8
 ; CHECK-NEXT:    subs r5, #4
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    add.w lr, r6, r5, lsr #2

llvm/test/CodeGen/Thumb2/mve-vmla.ll

 for.cond.cleanup:
   ret void
 }
 
 
 define arm_aapcs_vfpcc <4 x i32> @vmlasu32(<4 x i32> %A, <4 x i32> %B, i32 %X) nounwind {
 ; CHECK-LABEL: vmlasu32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmul.i32 q0, q0, q1
-; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmlas.u32 q0, q1, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = insertelement <4 x i32> undef, i32 %X, i32 0
   %1 = shufflevector <4 x i32> %0, <4 x i32> undef, <4 x i32> zeroinitializer
   %2 = mul nsw <4 x i32> %A, %B
   %3 = add nsw <4 x i32> %1, %2
   ret <4 x i32> %3
 }
 
 define arm_aapcs_vfpcc <4 x i32> @vmlasu32b(<4 x i32> %A, <4 x i32> %B, i32 %X) nounwind {
 ; CHECK-LABEL: vmlasu32b:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmul.i32 q0, q0, q1
-; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmlas.u32 q0, q1, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = insertelement <4 x i32> undef, i32 %X, i32 0
   %1 = shufflevector <4 x i32> %0, <4 x i32> undef, <4 x i32> zeroinitializer
   %2 = mul nsw <4 x i32> %A, %B
   %3 = add nsw <4 x i32> %2, %1
   ret <4 x i32> %3
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vmlasu16(<8 x i16> %A, <8 x i16> %B, i16 %X) nounwind {
 ; CHECK-LABEL: vmlasu16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
-; CHECK-NEXT:    vadd.i16 q0, q0, r0
+; CHECK-NEXT:    vmlas.u16 q0, q1, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = insertelement <8 x i16> undef, i16 %X, i32 0
   %1 = shufflevector <8 x i16> %0, <8 x i16> undef, <8 x i32> zeroinitializer
   %2 = mul nsw <8 x i16> %A, %B
   %3 = add nsw <8 x i16> %1, %2
   ret <8 x i16> %3
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vmlasu16b(<8 x i16> %A, <8 x i16> %B, i16 %X) nounwind {
 ; CHECK-LABEL: vmlasu16b:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
-; CHECK-NEXT:    vadd.i16 q0, q0, r0
+; CHECK-NEXT:    vmlas.u16 q0, q1, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = insertelement <8 x i16> undef, i16 %X, i32 0
   %1 = shufflevector <8 x i16> %0, <8 x i16> undef, <8 x i32> zeroinitializer
   %2 = mul nsw <8 x i16> %A, %B
   %3 = add nsw <8 x i16> %2, %1
   ret <8 x i16> %3
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vmlasu8(<16 x i8> %A, <16 x i8> %B, i8 %X) nounwind {
 ; CHECK-LABEL: vmlasu8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmul.i8 q0, q0, q1
-; CHECK-NEXT:    vadd.i8 q0, q0, r0
+; CHECK-NEXT:    vmlas.u8 q0, q1, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = insertelement <16 x i8> undef, i8 %X, i32 0
   %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> zeroinitializer
   %2 = mul nsw <16 x i8> %A, %B
   %3 = add nsw <16 x i8> %1, %2
   ret <16 x i8> %3
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vmlasu8b(<16 x i8> %A, <16 x i8> %B, i8 %X) nounwind {
 ; CHECK-LABEL: vmlasu8b:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmul.i8 q0, q0, q1
-; CHECK-NEXT:    vadd.i8 q0, q0, r0
+; CHECK-NEXT:    vmlas.u8 q0, q1, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = insertelement <16 x i8> undef, i8 %X, i32 0
   %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> zeroinitializer
   %2 = mul nsw <16 x i8> %A, %B
   %3 = add nsw <16 x i8> %2, %1
   ret <16 x i8> %3
 }
 
 define void @vmlas32_in_loop(i32* %s1, i32 %x, i32* %d, i32 %n) {
 ; CHECK-LABEL: vmlas32_in_loop:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:  .LBB15_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, r1
-; CHECK-NEXT:    vstrb.8 q0, [r2], #16
+; CHECK-NEXT:    vmlas.u32 q1, q0, r1
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB15_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
 entry:
   %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %x, i32 0
   %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
   br label %vector.body
 

 define void @vmlas16_in_loop(i16* %s1, i16 %x, i16* %d, i32 %n) {
 ; CHECK-LABEL: vmlas16_in_loop:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:  .LBB16_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r2]
 ; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
 ; CHECK-NEXT:    subs r3, #8
-; CHECK-NEXT:    vmul.i16 q0, q1, q0
-; CHECK-NEXT:    vadd.i16 q0, q0, r1
-; CHECK-NEXT:    vstrb.8 q0, [r2], #16
+; CHECK-NEXT:    vmlas.u16 q1, q0, r1
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB16_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
 entry:
   %broadcast.splatinsert11 = insertelement <8 x i16> undef, i16 %x, i32 0
   %broadcast.splat12 = shufflevector <8 x i16> %broadcast.splatinsert11, <8 x i16> undef, <8 x i32> zeroinitializer
   br label %vector.body
 

 define void @vmlas8_in_loop(i8* %s1, i8 %x, i8* %d, i32 %n) {
 ; CHECK-LABEL: vmlas8_in_loop:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:  .LBB17_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r2]
 ; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
 ; CHECK-NEXT:    subs r3, #16
-; CHECK-NEXT:    vmul.i8 q0, q1, q0
-; CHECK-NEXT:    vadd.i8 q0, q0, r1
-; CHECK-NEXT:    vstrb.8 q0, [r2], #16
+; CHECK-NEXT:    vmlas.u8 q1, q0, r1
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB17_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
 entry:
   %broadcast.splatinsert11 = insertelement <16 x i8> undef, i8 %x, i32 0
   %broadcast.splat12 = shufflevector <16 x i8> %broadcast.splatinsert11, <16 x i8> undef, <16 x i32> zeroinitializer
   br label %vector.body