[ARM] Single source VMOVNT

[dmgreen]

Summary

Our current lowering of VMOVNT goes via a shuffle vector of the form <0, N, 2, N+2, 4, N+4, ..>. That can of course also be a single input shuffle of the form <0, 0, 2, 2, 4, 4, ..>, where we use a VMOVNT to insert a vector into the top lanes of itself. This adds lowering of that case, re-using the existing isVMOVNMask.

llvm/lib/Target/ARM/ARMISelLowering.cpp

static bool isReverseMask(ArrayRef<int> M, EVT VT) {
   // Look for <15, ..., 3, -1, 1, 0>.
   for (unsigned i = 0; i != NumElts; ++i)
     if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
       return false;
 
   return true;
 }
 
-static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top) {
+static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
   unsigned NumElts = VT.getVectorNumElements();
   // Make sure the mask has the right size.
   if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
-      return false;
 
   // If Top
   //   Look for <0, N, 2, N+2, 4, N+4, ..>.
   //   This inserts Input2 into Input1
   // else if not Top
   //   Look for <0, N+1, 2, N+3, 4, N+5, ..>
   //   This inserts Input1 into Input2
   unsigned Offset = Top ? 0 : 1;
-  for (unsigned i = 0; i < NumElts; i+=2) {
+  unsigned N = SingleSource ? 0 : NumElts;
+  for (unsigned i = 0; i < NumElts; i += 2) {
     if (M[i] >= 0 && M[i] != (int)i)
       return false;
-    if (M[i+1] >= 0 && M[i+1] != (int)(NumElts + i + Offset))
+    if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset))
       return false;
   }
 
   return true;
 }
 
 // Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
 // from a pair of inputs. For example:
else if (Subtarget->hasNEON() &&
            (isVEXTMask(M, VT, ReverseVEXT, Imm) ||
             isVTBLMask(M, VT) ||
             isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF)))
     return true;
   else if (Subtarget->hasNEON() && (VT == MVT::v8i16 || VT == MVT::v16i8) &&
            isReverseMask(M, VT))
     return true;
   else if (Subtarget->hasMVEIntegerOps() &&
-           (isVMOVNMask(M, VT, 0) || isVMOVNMask(M, VT, 1)))
+           (isVMOVNMask(M, VT, true, false) ||
+            isVMOVNMask(M, VT, false, false) || isVMOVNMask(M, VT, true, true)))
     return true;
   else
     return false;
 }
 
 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
 /// the specified operations to build the shuffle.
 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
if (ST->hasNEON()) {
               ShuffleMask, VT, WhichResult, isV_UNDEF)) {
         if (isV_UNDEF)
           V2 = V1;
         return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
             .getValue(WhichResult);
       }
     }
     if (ST->hasMVEIntegerOps()) {
-      if (isVMOVNMask(ShuffleMask, VT, 0))
+      if (isVMOVNMask(ShuffleMask, VT, false, false))
         return DAG.getNode(ARMISD::VMOVN, dl, VT, V2, V1,
                            DAG.getConstant(0, dl, MVT::i32));
-      if (isVMOVNMask(ShuffleMask, VT, 1))
+      if (isVMOVNMask(ShuffleMask, VT, true, false))
         return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V2,
                            DAG.getConstant(1, dl, MVT::i32));
+      if (isVMOVNMask(ShuffleMask, VT, true, true))
+        return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V1,
+                           DAG.getConstant(1, dl, MVT::i32));
     }
 
     // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
     // shuffles that produce a result larger than their operands with:
     //   shuffle(concat(v1, undef), concat(v2, undef))
     // ->
     //   shuffle(concat(v1, v2), undef)
     // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).

llvm/test/CodeGen/Thumb2/mve-vmovn.ll

entry:
   %strided.vec = shufflevector <4 x i32> %src1, <4 x i32> %src2, <8 x i32> <i32 4, i32 0, i32 5, i32 1, i32 6, i32 2, i32 7, i32 3>
   %out = trunc <8 x i32> %strided.vec to <8 x i16>
   ret <8 x i16> %out
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vmovn32_trunc3(<4 x i32> %src1) {
 ; CHECK-LABEL: vmovn32_trunc3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vmov.16 q0[1], r0
-; CHECK-NEXT:    vmov r0, s5
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s6
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s7
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vmovnt.i32 q0, q0
 ; CHECK-NEXT:    bx lr
 ;
 ; CHECKBE-LABEL: vmovn32_trunc3:
 ; CHECKBE:       @ %bb.0: @ %entry
-; CHECKBE-NEXT:    vrev64.32 q2, q0
-; CHECKBE-NEXT:    vmov r0, s8
-; CHECKBE-NEXT:    vmov.16 q1[0], r0
-; CHECKBE-NEXT:    vmov.16 q1[1], r0
-; CHECKBE-NEXT:    vmov r0, s9
-; CHECKBE-NEXT:    vmov.16 q1[2], r0
-; CHECKBE-NEXT:    vmov.16 q1[3], r0
-; CHECKBE-NEXT:    vmov r0, s10
-; CHECKBE-NEXT:    vmov.16 q1[4], r0
-; CHECKBE-NEXT:    vmov.16 q1[5], r0
-; CHECKBE-NEXT:    vmov r0, s11
-; CHECKBE-NEXT:    vmov.16 q1[6], r0
-; CHECKBE-NEXT:    vmov.16 q1[7], r0
+; CHECKBE-NEXT:    vrev64.32 q1, q0
+; CHECKBE-NEXT:    vmovnt.i32 q1, q1
 ; CHECKBE-NEXT:    vrev64.16 q0, q1
 ; CHECKBE-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <4 x i32> %src1, <4 x i32> undef, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>
   %out = trunc <8 x i32> %strided.vec to <8 x i16>
   ret <8 x i16> %out
 }
 
entry:
   %strided.vec = shufflevector <8 x i16> %src1, <8 x i16> %src2, <16 x i32> <i32 8, i32 0, i32 9, i32 1, i32 10, i32 2, i32 11, i32 3, i32 12, i32 4, i32 13, i32 5, i32 14, i32 6, i32 15, i32 7>
   %out = trunc <16 x i16> %strided.vec to <16 x i8>
   ret <16 x i8> %out
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vmovn16_trunc3(<8 x i16> %src1) {
 ; CHECK-LABEL: vmovn16_trunc3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vmov.8 q0[0], r0
-; CHECK-NEXT:    vmov.8 q0[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[1]
-; CHECK-NEXT:    vmov.8 q0[2], r0
-; CHECK-NEXT:    vmov.8 q0[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.8 q0[4], r0
-; CHECK-NEXT:    vmov.8 q0[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.8 q0[6], r0
-; CHECK-NEXT:    vmov.8 q0[7], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-NEXT:    vmov.8 q0[8], r0
-; CHECK-NEXT:    vmov.8 q0[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-NEXT:    vmov.8 q0[10], r0
-; CHECK-NEXT:    vmov.8 q0[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov.8 q0[12], r0
-; CHECK-NEXT:    vmov.8 q0[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.8 q0[14], r0
-; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vmovnt.i16 q0, q0
 ; CHECK-NEXT:    bx lr
 ;
 ; CHECKBE-LABEL: vmovn16_trunc3:
 ; CHECKBE:       @ %bb.0: @ %entry
-; CHECKBE-NEXT:    vrev64.16 q2, q0
-; CHECKBE-NEXT:    vmov.u16 r0, q2[0]
-; CHECKBE-NEXT:    vmov.8 q1[0], r0
-; CHECKBE-NEXT:    vmov.8 q1[1], r0
-; CHECKBE-NEXT:    vmov.u16 r0, q2[1]
-; CHECKBE-NEXT:    vmov.8 q1[2], r0
-; CHECKBE-NEXT:    vmov.8 q1[3], r0
-; CHECKBE-NEXT:    vmov.u16 r0, q2[2]
-; CHECKBE-NEXT:    vmov.8 q1[4], r0
-; CHECKBE-NEXT:    vmov.8 q1[5], r0
-; CHECKBE-NEXT:    vmov.u16 r0, q2[3]
-; CHECKBE-NEXT:    vmov.8 q1[6], r0
-; CHECKBE-NEXT:    vmov.8 q1[7], r0
-; CHECKBE-NEXT:    vmov.u16 r0, q2[4]
-; CHECKBE-NEXT:    vmov.8 q1[8], r0
-; CHECKBE-NEXT:    vmov.8 q1[9], r0
-; CHECKBE-NEXT:    vmov.u16 r0, q2[5]
-; CHECKBE-NEXT:    vmov.8 q1[10], r0
-; CHECKBE-NEXT:    vmov.8 q1[11], r0
-; CHECKBE-NEXT:    vmov.u16 r0, q2[6]
-; CHECKBE-NEXT:    vmov.8 q1[12], r0
-; CHECKBE-NEXT:    vmov.8 q1[13], r0
-; CHECKBE-NEXT:    vmov.u16 r0, q2[7]
-; CHECKBE-NEXT:    vmov.8 q1[14], r0
-; CHECKBE-NEXT:    vmov.8 q1[15], r0
+; CHECKBE-NEXT:    vrev64.16 q1, q0
+; CHECKBE-NEXT:    vmovnt.i16 q1, q1
 ; CHECKBE-NEXT:    vrev64.8 q0, q1
 ; CHECKBE-NEXT:    bx lr
 entry:
   %strided.vec = shufflevector <8 x i16> %src1, <8 x i16> undef, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
   %out = trunc <16 x i16> %strided.vec to <16 x i8>
   ret <16 x i8> %out
 }
 

 entry:
   %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
   ret <8 x i16> %out
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vmovn16_single_t(<8 x i16> %src1) {
 ; CHECK-LABEL: vmovn16_single_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vins.f16 s0, s0
-; CHECK-NEXT:    vins.f16 s1, s1
-; CHECK-NEXT:    vins.f16 s2, s2
-; CHECK-NEXT:    vins.f16 s3, s3
+; CHECK-NEXT:    vmovnt.i32 q0, q0
 ; CHECK-NEXT:    bx lr
 ;
 ; CHECKBE-LABEL: vmovn16_single_t:
 ; CHECKBE:       @ %bb.0: @ %entry
 ; CHECKBE-NEXT:    vrev64.16 q1, q0
-; CHECKBE-NEXT:    vins.f16 s5, s5
-; CHECKBE-NEXT:    vins.f16 s4, s4
-; CHECKBE-NEXT:    vins.f16 s6, s6
-; CHECKBE-NEXT:    vins.f16 s7, s7
+; CHECKBE-NEXT:    vmovnt.i32 q1, q1
 ; CHECKBE-NEXT:    vrev64.16 q0, q1
 ; CHECKBE-NEXT:    bx lr
 entry:
   %out = shufflevector <8 x i16> %src1, <8 x i16> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   ret <8 x i16> %out
 }
 
 

 entry:
   %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
   ret <16 x i8> %out
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vmovn8_single_t(<16 x i8> %src1) {
 ; CHECK-LABEL: vmovn8_single_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    vmov.8 q0[0], r0
-; CHECK-NEXT:    vmov.8 q0[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.8 q0[2], r0
-; CHECK-NEXT:    vmov.8 q0[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.8 q0[4], r0
-; CHECK-NEXT:    vmov.8 q0[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.8 q0[6], r0
-; CHECK-NEXT:    vmov.8 q0[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.8 q0[8], r0
-; CHECK-NEXT:    vmov.8 q0[9], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.8 q0[10], r0
-; CHECK-NEXT:    vmov.8 q0[11], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.8 q0[12], r0
-; CHECK-NEXT:    vmov.8 q0[13], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.8 q0[14], r0
-; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vmovnt.i16 q0, q0
 ; CHECK-NEXT:    bx lr
 ;
 ; CHECKBE-LABEL: vmovn8_single_t:
 ; CHECKBE:       @ %bb.0: @ %entry
-; CHECKBE-NEXT:    vrev64.8 q2, q0
-; CHECKBE-NEXT:    vmov.u8 r0, q2[0]
-; CHECKBE-NEXT:    vmov.8 q1[0], r0
-; CHECKBE-NEXT:    vmov.8 q1[1], r0
-; CHECKBE-NEXT:    vmov.u8 r0, q2[2]
-; CHECKBE-NEXT:    vmov.8 q1[2], r0
-; CHECKBE-NEXT:    vmov.8 q1[3], r0
-; CHECKBE-NEXT:    vmov.u8 r0, q2[4]
-; CHECKBE-NEXT:    vmov.8 q1[4], r0
-; CHECKBE-NEXT:    vmov.8 q1[5], r0
-; CHECKBE-NEXT:    vmov.u8 r0, q2[6]
-; CHECKBE-NEXT:    vmov.8 q1[6], r0
-; CHECKBE-NEXT:    vmov.8 q1[7], r0
-; CHECKBE-NEXT:    vmov.u8 r0, q2[8]
-; CHECKBE-NEXT:    vmov.8 q1[8], r0
-; CHECKBE-NEXT:    vmov.8 q1[9], r0
-; CHECKBE-NEXT:    vmov.u8 r0, q2[10]
-; CHECKBE-NEXT:    vmov.8 q1[10], r0
-; CHECKBE-NEXT:    vmov.8 q1[11], r0
-; CHECKBE-NEXT:    vmov.u8 r0, q2[12]
-; CHECKBE-NEXT:    vmov.8 q1[12], r0
-; CHECKBE-NEXT:    vmov.8 q1[13], r0
-; CHECKBE-NEXT:    vmov.u8 r0, q2[14]
-; CHECKBE-NEXT:    vmov.8 q1[14], r0
-; CHECKBE-NEXT:    vmov.8 q1[15], r0
+; CHECKBE-NEXT:    vrev64.8 q1, q0
+; CHECKBE-NEXT:    vmovnt.i16 q1, q1
 ; CHECKBE-NEXT:    vrev64.8 q0, q1
 ; CHECKBE-NEXT:    bx lr
 entry:
   %out = shufflevector <16 x i8> %src1, <16 x i8> undef, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
   ret <16 x i8> %out
 }