[ARM][MVE] Add VHADD and VHSUB patterns

[samparker]

Summary

Add patterns, for two vector operands, that use a normal add node along with an arm vshr imm node.

[dmgreen]

We had these patterns before and took them out because they were not correct. My understanding is that these instructions do trunc(shift(add(sext(a), sext(b)), 1)). They internally operate in a higher bitwidth than we natively have.

See rGe9e1daf2b9e800c00a3c08db53650c1569288a1b for the revert, which includes some examples in the commit message. I think I left the tests in for them though.

Maybe there's something we can do here with no wrap flags? Or patterns with extensions in them?

[samparker]

Ah yes, now I see '[esize:1]' has the result bit range. That's a bit annoying, but quite cool... I think using no wrap flags sounds neater, as I can just add a PatFrag. I guess I should emit the i8 patterns though as they won't trigger?

[samparker]

Now checking for non wrap flags on the add and sub.

[dmgreen]

Seems OK, from what I can tell. And http://volta.cs.utah.edu:8080/z/8mUFbh says this is OK. My fuzzer really isn't going to like it, but that's a separate issue!

llvm/lib/Target/ARM/ARMInstrMVE.td

2089

Can you add a comment about how this instruction is using a higher bitwidth type, and the nowrap makes sure we don't overflow when only dealing with lower bitwidth?

[samparker]

Moved the PatFrags together and added a comment.

[dmgreen]

LGTM thanks.

llvm/lib/Target/ARM/ARMInstrMVE.td

 class MVE_VHADD_<string suffix, bit U, bits<2> size,
               list<dag> pattern=[]>
   : MVE_VHADDSUB<"vhadd", suffix, U, 0b0, size, pattern>;
 class MVE_VHSUB_<string suffix, bit U, bits<2> size,
               list<dag> pattern=[]>
   : MVE_VHADDSUB<"vhsub", suffix, U, 0b1, size, pattern>;
 
 multiclass MVE_VHADD_m<MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int> {
+                      SDNode unpred_op, Intrinsic pred_int, SDNode shift_op> {
   def "" : MVE_VHADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEInt] in {
     // Unpredicated add-and-divide-by-two
     def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned))),
               (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
 
+    def : Pat<(VTI.Vec (shift_op (add (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
+              (Inst MQPR:$Qm, MQPR:$Qn)>;
+
     // Predicated add-and-divide-by-two
     def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned),
                             (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
               (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
                              ARMVCCThen, (VTI.Pred VCCR:$mask),
                              (VTI.Vec MQPR:$inactive)))>;
   }
 }
 
-multiclass MVE_VHADD<MVEVectorVTInfo VTI>
-  : MVE_VHADD_m<VTI, int_arm_mve_vhadd, int_arm_mve_hadd_predicated>;
+multiclass MVE_VHADD<MVEVectorVTInfo VTI, SDNode shift_op>
+  : MVE_VHADD_m<VTI, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, shift_op>;
 
-defm MVE_VHADDs8  : MVE_VHADD<MVE_v16s8>;
-defm MVE_VHADDs16 : MVE_VHADD<MVE_v8s16>;
-defm MVE_VHADDs32 : MVE_VHADD<MVE_v4s32>;
-defm MVE_VHADDu8  : MVE_VHADD<MVE_v16u8>;
-defm MVE_VHADDu16 : MVE_VHADD<MVE_v8u16>;
-defm MVE_VHADDu32 : MVE_VHADD<MVE_v4u32>;
+defm MVE_VHADDs8  : MVE_VHADD<MVE_v16s8, ARMvshrsImm>;
+defm MVE_VHADDs16 : MVE_VHADD<MVE_v8s16, ARMvshrsImm>;
+defm MVE_VHADDs32 : MVE_VHADD<MVE_v4s32, ARMvshrsImm>;
+defm MVE_VHADDu8  : MVE_VHADD<MVE_v16u8, ARMvshruImm>;
+defm MVE_VHADDu16 : MVE_VHADD<MVE_v8u16, ARMvshruImm>;
+defm MVE_VHADDu32 : MVE_VHADD<MVE_v4u32, ARMvshruImm>;
 
 multiclass MVE_VHSUB_m<MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int> {
+                      SDNode unpred_op, Intrinsic pred_int, SDNode shift_op> {
   def "" : MVE_VHSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEInt] in {
     // Unpredicated subtract-and-divide-by-two
     def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
                             (i32 VTI.Unsigned))),
               (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
 
+    def : Pat<(VTI.Vec (shift_op (sub (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
+              (Inst MQPR:$Qm, MQPR:$Qn)>;
+
     // Predicated subtract-and-divide-by-two
     def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
                             (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
                             (VTI.Vec MQPR:$inactive))),
               (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
                              ARMVCCThen, (VTI.Pred VCCR:$mask),
                              (VTI.Vec MQPR:$inactive)))>;
   }
 }
 
-multiclass MVE_VHSUB<MVEVectorVTInfo VTI>
-  : MVE_VHSUB_m<VTI, int_arm_mve_vhsub, int_arm_mve_hsub_predicated>;
+multiclass MVE_VHSUB<MVEVectorVTInfo VTI, SDNode shift_op>
+  : MVE_VHSUB_m<VTI, int_arm_mve_vhsub, int_arm_mve_hsub_predicated, shift_op>;
 
-defm MVE_VHSUBs8  : MVE_VHSUB<MVE_v16s8>;
-defm MVE_VHSUBs16 : MVE_VHSUB<MVE_v8s16>;
-defm MVE_VHSUBs32 : MVE_VHSUB<MVE_v4s32>;
-defm MVE_VHSUBu8  : MVE_VHSUB<MVE_v16u8>;
-defm MVE_VHSUBu16 : MVE_VHSUB<MVE_v8u16>;
-defm MVE_VHSUBu32 : MVE_VHSUB<MVE_v4u32>;
+defm MVE_VHSUBs8  : MVE_VHSUB<MVE_v16s8, ARMvshrsImm>;
+defm MVE_VHSUBs16 : MVE_VHSUB<MVE_v8s16, ARMvshrsImm>;
+defm MVE_VHSUBs32 : MVE_VHSUB<MVE_v4s32, ARMvshrsImm>;
+defm MVE_VHSUBu8  : MVE_VHSUB<MVE_v16u8, ARMvshruImm>;
+defm MVE_VHSUBu16 : MVE_VHSUB<MVE_v8u16, ARMvshruImm>;
+defm MVE_VHSUBu32 : MVE_VHSUB<MVE_v4u32, ARMvshruImm>;
 
 class MVE_VDUP<string suffix, bit B, bit E, list<dag> pattern=[]>
   : MVE_p<(outs MQPR:$Qd), (ins rGPR:$Rt), NoItinerary,
           "vdup", suffix, "$Qd, $Rt", vpred_r, "", pattern> {
   bits<4> Qd;
   bits<4> Rt;
 
   let Inst{28} = 0b0;

llvm/test/CodeGen/Thumb2/mve-halving.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @vhadds_v16i8(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: vhadds_v16i8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhadd.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %add = add <16 x i8> %x, %y
+  %half = ashr <16 x i8> %add, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %half
+}
+define arm_aapcs_vfpcc <16 x i8> @vhaddu_v16i8(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: vhaddu_v16i8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhadd.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %add = add <16 x i8> %x, %y
+  %half = lshr <16 x i8> %add, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %half
+}
+define arm_aapcs_vfpcc <8 x i16> @vhadds_v8i16(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: vhadds_v8i16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhadd.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %add = add <8 x i16> %x, %y
+  %half = ashr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %half
+}
+define arm_aapcs_vfpcc <8 x i16> @vhaddu_v8i16(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: vhaddu_v8i16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhadd.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %add = add <8 x i16> %x, %y
+  %half = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %half
+}
+define arm_aapcs_vfpcc <4 x i32> @vhadds_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: vhadds_v4i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhadd.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %add = add <4 x i32> %x, %y
+  %half = ashr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %half
+}
+define arm_aapcs_vfpcc <4 x i32> @vhaddu_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: vhaddu_v4i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhadd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %add = add <4 x i32> %x, %y
+  %half = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %half
+}
+define arm_aapcs_vfpcc <16 x i8> @vhsubs_v16i8(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: vhsubs_v16i8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhsub.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub <16 x i8> %x, %y
+  %half = ashr <16 x i8> %sub, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %half
+}
+define arm_aapcs_vfpcc <16 x i8> @vhsubu_v16i8(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: vhsubu_v16i8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhsub.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub <16 x i8> %x, %y
+  %half = lshr <16 x i8> %sub, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  ret <16 x i8> %half
+}
+define arm_aapcs_vfpcc <8 x i16> @vhsubs_v8i16(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: vhsubs_v8i16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhsub.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub <8 x i16> %x, %y
+  %half = ashr <8 x i16> %sub, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %half
+}
+define arm_aapcs_vfpcc <8 x i16> @vhsubu_v8i16(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: vhsubu_v8i16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhsub.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub <8 x i16> %x, %y
+  %half = lshr <8 x i16> %sub, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  ret <8 x i16> %half
+}
+define arm_aapcs_vfpcc <4 x i32> @vhsubs_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: vhsubs_v4i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhsub.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub <4 x i32> %x, %y
+  %half = ashr <4 x i32> %sub, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %half
+}
+define arm_aapcs_vfpcc <4 x i32> @vhsubu_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: vhsubu_v4i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vhsub.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+  %sub = sub <4 x i32> %x, %y
+  %half = lshr <4 x i32> %sub, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %half
+}