[ARM] VQADD instructions

[dmgreen]

Summary

This selects MVE VQADD from the vector llvm.sadd.sat or llvm.uadd.sat intrinsics.

The signed versions seem hard to get from C, but the unsigned are simple enough. And both are available from llvm, obviously.

[SjoerdMeijer]

Looks like a nice bit of isel to me.

llvm/lib/Target/ARM/ARMInstrMVE.td

1563

And looking at this, I almost start to like tablegen. :-)

[samparker]

Just wondering, who generates the intrinsics? From the little that I remember from the last time I looked, I thought it was clang but it that is was x86 specific? Do we have some hooks somewhere saying that we support them?

[dmgreen]

The usat version can be idiom recognised from C:
https://godbolt.org/z/9knBnP
ssat is more difficult like I mentioned, but I don't think it should be impossible to do the same thing there too.

llvm/lib/Target/ARM/ARMInstrMVE.td

1563

This is from D67158. The idea is that an intrinsic can be easily added in here, same as for vadds in that patch.

[samparker]

Great!

llvm/lib/Target/ARM/ARMISelLowering.cpp

for (auto VT : IntTypes) {
     setOperationAction(ISD::ABS, VT, Legal);
     setOperationAction(ISD::SETCC, VT, Custom);
     setOperationAction(ISD::MLOAD, VT, Custom);
     setOperationAction(ISD::MSTORE, VT, Legal);
     setOperationAction(ISD::CTLZ, VT, Legal);
     setOperationAction(ISD::CTTZ, VT, Custom);
     setOperationAction(ISD::BITREVERSE, VT, Legal);
     setOperationAction(ISD::BSWAP, VT, Legal);
+    setOperationAction(ISD::SADDSAT, VT, Legal);
+    setOperationAction(ISD::UADDSAT, VT, Legal);
 
     // No native support for these.
     setOperationAction(ISD::UDIV, VT, Expand);
     setOperationAction(ISD::SDIV, VT, Expand);
     setOperationAction(ISD::UREM, VT, Expand);
     setOperationAction(ISD::SREM, VT, Expand);
     setOperationAction(ISD::CTPOP, VT, Expand);
 

llvm/lib/Target/ARM/ARMInstrMVE.td

def : Pat<(v16i8 (sub (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
             (v16i8 (MVE_VSUBi8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
   def : Pat<(v8i16 (sub (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
             (v8i16 (MVE_VSUBi16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
   def : Pat<(v4i32 (sub (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
             (v4i32 (MVE_VSUBi32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
 }
 
 class MVE_VQADDSUB<string iname, string suffix, bit U, bit subtract,
-                   bits<2> size, list<dag> pattern=[]>
-  : MVE_int<iname, suffix, size, pattern> {
+                   bits<2> size, ValueType vt>
+  : MVE_int<iname, suffix, size, []> {
 
   let Inst{28} = U;
   let Inst{25-23} = 0b110;
   let Inst{16} = 0b0;
   let Inst{12-10} = 0b000;
   let Inst{9} = subtract;
   let Inst{8} = 0b0;
   let Inst{4} = 0b1;
   let Inst{0} = 0b0;
+
+  ValueType VT = vt;
+}
+
+class MVE_VQADD<string suffix, bit U, bits<2> size, ValueType VT>
+  : MVE_VQADDSUB<"vqadd", suffix, U, 0b0, size, VT>;
+class MVE_VQSUB<string suffix, bit U, bits<2> size, ValueType VT>
+  : MVE_VQADDSUB<"vqsub", suffix, U, 0b1, size, VT>;
+
+def MVE_VQADDs8  : MVE_VQADD<"s8",  0b0, 0b00, v16i8>;
+def MVE_VQADDs16 : MVE_VQADD<"s16", 0b0, 0b01, v8i16>;
+def MVE_VQADDs32 : MVE_VQADD<"s32", 0b0, 0b10, v4i32>;
+def MVE_VQADDu8  : MVE_VQADD<"u8",  0b1, 0b00, v16i8>;
+def MVE_VQADDu16 : MVE_VQADD<"u16", 0b1, 0b01, v8i16>;
+def MVE_VQADDu32 : MVE_VQADD<"u32", 0b1, 0b10, v4i32>;
+
+def MVE_VQSUBs8  : MVE_VQSUB<"s8",  0b0, 0b00, v16i8>;
+def MVE_VQSUBs16 : MVE_VQSUB<"s16", 0b0, 0b01, v8i16>;
+def MVE_VQSUBs32 : MVE_VQSUB<"s32", 0b0, 0b10, v4i32>;
+def MVE_VQSUBu8  : MVE_VQSUB<"u8",  0b1, 0b00, v16i8>;
+def MVE_VQSUBu16 : MVE_VQSUB<"u16", 0b1, 0b01, v8i16>;
+def MVE_VQSUBu32 : MVE_VQSUB<"u32", 0b1, 0b10, v4i32>;
+
+let Predicates = [HasMVEInt] in {
+  foreach instr = [MVE_VQADDu8, MVE_VQADDu16, MVE_VQADDu32] in
+    foreach VT = [instr.VT] in
+      def : Pat<(VT (uaddsat (VT MQPR:$Qm), (VT MQPR:$Qn))),
+                (VT (instr (VT MQPR:$Qm), (VT MQPR:$Qn)))>;
+  foreach instr = [MVE_VQADDs8, MVE_VQADDs16, MVE_VQADDs32] in
+    foreach VT = [instr.VT] in
+      def : Pat<(VT (saddsat (VT MQPR:$Qm), (VT MQPR:$Qn))),
+                (VT (instr (VT MQPR:$Qm), (VT MQPR:$Qn)))>;
 }
 
-class MVE_VQADD<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
-  : MVE_VQADDSUB<"vqadd", suffix, U, 0b0, size, pattern>;
-class MVE_VQSUB<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
-  : MVE_VQADDSUB<"vqsub", suffix, U, 0b1, size, pattern>;
-
-def MVE_VQADDs8  : MVE_VQADD<"s8",  0b0, 0b00>;
-def MVE_VQADDs16 : MVE_VQADD<"s16", 0b0, 0b01>;
-def MVE_VQADDs32 : MVE_VQADD<"s32", 0b0, 0b10>;
-def MVE_VQADDu8  : MVE_VQADD<"u8",  0b1, 0b00>;
-def MVE_VQADDu16 : MVE_VQADD<"u16", 0b1, 0b01>;
-def MVE_VQADDu32 : MVE_VQADD<"u32", 0b1, 0b10>;
-
-def MVE_VQSUBs8  : MVE_VQSUB<"s8",  0b0, 0b00>;
-def MVE_VQSUBs16 : MVE_VQSUB<"s16", 0b0, 0b01>;
-def MVE_VQSUBs32 : MVE_VQSUB<"s32", 0b0, 0b10>;
-def MVE_VQSUBu8  : MVE_VQSUB<"u8",  0b1, 0b00>;
-def MVE_VQSUBu16 : MVE_VQSUB<"u16", 0b1, 0b01>;
-def MVE_VQSUBu32 : MVE_VQSUB<"u32", 0b1, 0b10>;
 
 class MVE_VABD_int<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
   : MVE_int<"vabd", suffix, size, pattern> {
 
   let Inst{28} = U;
   let Inst{25-23} = 0b110;
   let Inst{16} = 0b0;
   let Inst{12-8} = 0b00111;

llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc <16 x i8> @sadd_int8_t(<16 x i8> %src1, <16 x i8> %src2) {
 ; CHECK-LABEL: sadd_int8_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vadd.i8 q2, q0, q1
-; CHECK-NEXT:    vmov.i8 q3, #0x80
-; CHECK-NEXT:    vcmp.s8 lt, q2, zr
-; CHECK-NEXT:    vmov.i8 q4, #0x7f
-; CHECK-NEXT:    vpsel q3, q4, q3
-; CHECK-NEXT:    vcmp.s8 gt, q0, q2
-; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.s8 lt, q1, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmsr p0, r0
-; CHECK-NEXT:    vpsel q0, q3, q2
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vqadd.s8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
   ret <16 x i8> %0
 }
 
 define arm_aapcs_vfpcc <8 x i16> @sadd_int16_t(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: sadd_int16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vadd.i16 q2, q0, q1
-; CHECK-NEXT:    vmov.i16 q3, #0x8000
-; CHECK-NEXT:    vcmp.s16 lt, q2, zr
-; CHECK-NEXT:    vmvn.i16 q4, #0x8000
-; CHECK-NEXT:    vpsel q3, q4, q3
-; CHECK-NEXT:    vcmp.s16 gt, q0, q2
-; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.s16 lt, q1, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmsr p0, r0
-; CHECK-NEXT:    vpsel q0, q3, q2
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vqadd.s16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
   ret <8 x i16> %0
 }
 
 define arm_aapcs_vfpcc <4 x i32> @sadd_int32_t(<4 x i32> %src1, <4 x i32> %src2) {
 ; CHECK-LABEL: sadd_int32_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vadd.i32 q2, q0, q1
-; CHECK-NEXT:    vmov.i32 q3, #0x80000000
-; CHECK-NEXT:    vcmp.s32 lt, q2, zr
-; CHECK-NEXT:    vmvn.i32 q4, #0x80000000
-; CHECK-NEXT:    vpsel q3, q4, q3
-; CHECK-NEXT:    vcmp.s32 gt, q0, q2
-; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.s32 lt, q1, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    vmsr p0, r0
-; CHECK-NEXT:    vpsel q0, q3, q2
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vqadd.s32 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
   ret <4 x i32> %0
 }
 
 define arm_aapcs_vfpcc <2 x i64> @sadd_int64_t(<2 x i64> %src1, <2 x i64> %src2) {
 ; CHECK-LABEL: sadd_int64_t:

 entry:
   %0 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
   ret <2 x i64> %0
 }
 
 define arm_aapcs_vfpcc <16 x i8> @uadd_int8_t(<16 x i8> %src1, <16 x i8> %src2) {
 ; CHECK-LABEL: uadd_int8_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q2, q1
-; CHECK-NEXT:    vmin.u8 q0, q0, q2
-; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vqadd.u8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
   ret <16 x i8> %0
 }
 
 define arm_aapcs_vfpcc <8 x i16> @uadd_int16_t(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: uadd_int16_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q2, q1
-; CHECK-NEXT:    vmin.u16 q0, q0, q2
-; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vqadd.u16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
   ret <8 x i16> %0
 }
 
 define arm_aapcs_vfpcc <4 x i32> @uadd_int32_t(<4 x i32> %src1, <4 x i32> %src2) {
 ; CHECK-LABEL: uadd_int32_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q2, q1
-; CHECK-NEXT:    vmin.u32 q0, q0, q2
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vqadd.u32 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
   ret <4 x i32> %0
 }
 
 define arm_aapcs_vfpcc <2 x i64> @uadd_int64_t(<2 x i64> %src1, <2 x i64> %src2) {
 ; CHECK-LABEL: uadd_int64_t: