[ARM] Combines for VMOVN

[dmgreen]

Summary

This adds two combines for VMOVN, one to fold VMOVN[tb](c, VQMOVNb(a, b)) => VQMOVN[tb](c, b), the other to perform demand bits analysis on the lanes of a VMOVN. We know that only the bottom lanes of the second operand and the top or bottom lanes of the Qd operand are needed in the result, depending on if the VMOVN is bottom or top.

[samparker]

llvm/lib/Target/ARM/ARMISelLowering.cpp

14442

Would it be possible to use APInt::getSplat() instead? This is looking a bit cryptic.

[ctetreau]

llvm/lib/Target/ARM/ARMISelLowering.cpp

14441

getVectorNumElements is going away soon. Please do the cast.

See: D77278

[efriedma]

llvm/lib/Target/ARM/ARMISelLowering.cpp

14441

This is EVT::getVectorNumElements(); that's not going away, and there is no such cast.

[ctetreau]

llvm/lib/Target/ARM/ARMISelLowering.cpp

14441

Ahh, EVT... My old nemesis...

Please disregard.

[dmgreen]

Updated to use Splat's

[dmgreen]

ping

[samparker]

LGTM.

llvm/lib/Target/ARM/ARMISelLowering.cpp

if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
     return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B);
   if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B))
     return Create64bitNode(ARMISD::VMLALVs, {A, B});
   if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B))
     return Create64bitNode(ARMISD::VMLALVu, {A, B});
   return SDValue();
 }
 
+static SDValue PerformVMOVNCombine(SDNode *N,
+                                   TargetLowering::DAGCombinerInfo &DCI) {
+  SDValue Op0 = N->getOperand(0);
+  SDValue Op1 = N->getOperand(1);
+  unsigned IsTop = N->getConstantOperandVal(2);
+
+  // VMOVNt(c, VQMOVNb(a, b)) => VQMOVNt(c, b)
+  // VMOVNb(c, VQMOVNb(a, b)) => VQMOVNb(c, b)
+  if ((Op1->getOpcode() == ARMISD::VQMOVNs ||
+       Op1->getOpcode() == ARMISD::VQMOVNu) &&
+      Op1->getConstantOperandVal(2) == 0)
+    return DCI.DAG.getNode(Op1->getOpcode(), SDLoc(Op1), N->getValueType(0),
+                           Op0, Op1->getOperand(1), N->getOperand(2));
+
+  // Only the bottom lanes from Qm (Op1) and either the top or bottom lanes from
+  // Qd (Op0) are demanded from a VMOVN, depending on whether we are inserting
+  // into the top or bottom lanes.
+  unsigned NumElts = N->getValueType(0).getVectorNumElements();
+  APInt Op1DemandedElts = APInt::getSplat(NumElts, APInt::getLowBitsSet(2, 1));
+  APInt Op0DemandedElts =
+      IsTop ? Op1DemandedElts
+            : APInt::getSplat(NumElts, APInt::getHighBitsSet(2, 1));
+
+  APInt KnownUndef, KnownZero;
+  const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
+  if (TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, KnownUndef,
+                                     KnownZero, DCI))
+    return SDValue(N, 0);
+  if (TLI.SimplifyDemandedVectorElts(Op1, Op1DemandedElts, KnownUndef,
+                                     KnownZero, DCI))
+    return SDValue(N, 0);
+
+  return SDValue();
+}
+
 static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG) {
   SDLoc DL(N);
   SDValue Op0 = N->getOperand(0);
   SDValue Op1 = N->getOperand(1);
 
   // Turn X << -C -> X >> C and viceversa. The negative shifts can come up from
   // uses of the intrinsics.
   if (auto C = dyn_cast<ConstantSDNode>(N->getOperand(2))) {
SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ARMISD::PREDICATE_CAST:
     return PerformPREDICATE_CASTCombine(N, DCI);
   case ARMISD::VECTOR_REG_CAST:
     return PerformVECTOR_REG_CASTCombine(N, DCI, Subtarget);
   case ARMISD::VCMP:
     return PerformVCMPCombine(N, DCI, Subtarget);
   case ISD::VECREDUCE_ADD:
     return PerformVECREDUCE_ADDCombine(N, DCI.DAG, Subtarget);
+  case ARMISD::VMOVN:
+    return PerformVMOVNCombine(N, DCI);
   case ARMISD::ASRL:
   case ARMISD::LSRL:
   case ARMISD::LSLL:
     return PerformLongShiftCombine(N, DCI.DAG);
   case ARMISD::SMULWB: {
     unsigned BitWidth = N->getValueType(0).getSizeInBits();
     APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
     if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))

llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll

 ; CHECK-NEXT:    add.w r6, r1, r5, lsl #1
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB7_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
 ; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
 ; CHECK-NEXT:    vmullt.s16 q2, q1, q0
 ; CHECK-NEXT:    vmullb.s16 q0, q1, q0
-; CHECK-NEXT:    vshr.s32 q2, q2, #15
 ; CHECK-NEXT:    vshr.s32 q0, q0, #15
-; CHECK-NEXT:    vqmovnb.s32 q2, q2
+; CHECK-NEXT:    vshr.s32 q2, q2, #15
 ; CHECK-NEXT:    vqmovnb.s32 q0, q0
-; CHECK-NEXT:    vmovlb.s16 q2, q2
 ; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vmovnt.i32 q0, q2
+; CHECK-NEXT:    vqmovnt.s32 q0, q2
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB7_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r5, r3
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r4, r5, r6, pc}
 ; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader21
 ; CHECK-NEXT:    movw r0, #32768

 ; CHECK-NEXT:    vmov.16 q5[7], r4
 ; CHECK-NEXT:    vptt.i16 ne, q5, zr
 ; CHECK-NEXT:    vldrht.u16 q5, [r0], #16
 ; CHECK-NEXT:    vldrht.u16 q6, [r1], #16
 ; CHECK-NEXT:    vmullt.s16 q7, q6, q5
 ; CHECK-NEXT:    vmullb.s16 q5, q6, q5
 ; CHECK-NEXT:    vshr.s32 q7, q7, #15
 ; CHECK-NEXT:    vshr.s32 q5, q5, #15
-; CHECK-NEXT:    vqmovnb.s32 q7, q7
 ; CHECK-NEXT:    vqmovnb.s32 q5, q5
-; CHECK-NEXT:    vmovlb.s16 q7, q7
 ; CHECK-NEXT:    vmovlb.s16 q5, q5
-; CHECK-NEXT:    vmovnt.i32 q5, q7
+; CHECK-NEXT:    vqmovnt.s32 q5, q7
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrht.16 q5, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB10_2
 ; CHECK-NEXT:  .LBB10_3: @ %for.cond.cleanup
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.4:

 ; CHECK-NEXT:    adds r6, r1, r5
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB16_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
 ; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
 ; CHECK-NEXT:    vmullt.s8 q2, q1, q0
 ; CHECK-NEXT:    vmullb.s8 q0, q1, q0
-; CHECK-NEXT:    vshr.s16 q2, q2, #7
 ; CHECK-NEXT:    vshr.s16 q0, q0, #7
-; CHECK-NEXT:    vqmovnb.s16 q2, q2
+; CHECK-NEXT:    vshr.s16 q2, q2, #7
 ; CHECK-NEXT:    vqmovnb.s16 q0, q0
-; CHECK-NEXT:    vmovlb.s8 q2, q2
 ; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vmovnt.i16 q0, q2
+; CHECK-NEXT:    vqmovnt.s16 q0, q2
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB16_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r5, r3
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r4, r5, r6, pc}
 ; CHECK-NEXT:  .LBB16_6: @ %for.body.preheader23
 ; CHECK-NEXT:    sub.w lr, r3, r5

 ; CHECK-NEXT:    vmov.8 q7[14], r4
 ; CHECK-NEXT:    vmov.u16 r4, q0[7]
 ; CHECK-NEXT:    vmov.8 q7[15], r4
 ; CHECK-NEXT:    vptt.i8 ne, q7, zr
 ; CHECK-NEXT:    vldrbt.u8 q0, [r0], #16
 ; CHECK-NEXT:    vldrbt.u8 q4, [r1], #16
 ; CHECK-NEXT:    vmullt.s8 q5, q4, q0
 ; CHECK-NEXT:    vmullb.s8 q0, q4, q0
-; CHECK-NEXT:    vshr.s16 q5, q5, #7
 ; CHECK-NEXT:    vshr.s16 q0, q0, #7
-; CHECK-NEXT:    vqmovnb.s16 q5, q5
+; CHECK-NEXT:    vshr.s16 q5, q5, #7
 ; CHECK-NEXT:    vqmovnb.s16 q0, q0
-; CHECK-NEXT:    vmovlb.s8 q5, q5
 ; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vmovnt.i16 q0, q5
+; CHECK-NEXT:    vqmovnt.s16 q0, q5
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrbt.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB19_2
 ; CHECK-NEXT:  .LBB19_3: @ %for.cond.cleanup
 ; CHECK-NEXT:    add sp, #56
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:    .p2align 4

llvm/test/CodeGen/Thumb2/mve-vqmovn-combine.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc <8 x i16> @vqmovni32_sminmax_t1(<4 x i32> %s0, <8 x i16> %src1) {
 ; CHECK-LABEL: vqmovni32_sminmax_t1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vqmovnb.s32 q0, q0
-; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vmovnt.i32 q1, q0
+; CHECK-NEXT:    vqmovnt.s32 q1, q0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp sgt <4 x i32> %s0, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
   %s1 = select <4 x i1> %c1, <4 x i32> %s0, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
   %c2 = icmp slt <4 x i32> %s1, <i32 32767, i32 32767, i32 32767, i32 32767>
   %s2 = select <4 x i1> %c2, <4 x i32> %s1, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
   %src2 = bitcast <4 x i32> %s2 to <8 x i16>
   %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
   ret <8 x i16> %out
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vqmovni32_sminmax_t2(<4 x i32> %s0, <8 x i16> %src1) {
 ; CHECK-LABEL: vqmovni32_sminmax_t2:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vqmovnb.s32 q0, q0
-; CHECK-NEXT:    vmovlb.s16 q0, q0
 ; CHECK-NEXT:    vmovnt.i32 q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp sgt <4 x i32> %s0, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
   %s1 = select <4 x i1> %c1, <4 x i32> %s0, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
   %c2 = icmp slt <4 x i32> %s1, <i32 32767, i32 32767, i32 32767, i32 32767>
   %s2 = select <4 x i1> %c2, <4 x i32> %s1, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
   %src2 = bitcast <4 x i32> %s2 to <8 x i16>
   %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 8, i32 0, i32 10, i32 2, i32 12, i32 4, i32 14, i32 6>
   ret <8 x i16> %out
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vqmovni32_sminmax_b1(<4 x i32> %s0, <8 x i16> %src1) {
 ; CHECK-LABEL: vqmovni32_sminmax_b1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vqmovnb.s32 q0, q0
-; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vmovnb.i32 q1, q0
+; CHECK-NEXT:    vqmovnb.s32 q1, q0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp sgt <4 x i32> %s0, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
   %s1 = select <4 x i1> %c1, <4 x i32> %s0, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
   %c2 = icmp slt <4 x i32> %s1, <i32 32767, i32 32767, i32 32767, i32 32767>
   %s2 = select <4 x i1> %c2, <4 x i32> %s1, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
   %src2 = bitcast <4 x i32> %s2 to <8 x i16>
entry:
   %out = shufflevector <8 x i16> %src1, <8 x i16> %src2, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
   ret <8 x i16> %out
 }
 
 
 define arm_aapcs_vfpcc <16 x i8> @vqmovni16_sminmax_t1(<8 x i16> %s0, <16 x i8> %src1) {
 ; CHECK-LABEL: vqmovni16_sminmax_t1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vqmovnb.s16 q0, q0
-; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vmovnt.i16 q1, q0
+; CHECK-NEXT:    vqmovnt.s16 q1, q0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp sgt <8 x i16> %s0, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
   %s1 = select <8 x i1> %c1, <8 x i16> %s0, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
   %c2 = icmp slt <8 x i16> %s1, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %s2 = select <8 x i1> %c2, <8 x i16> %s1, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %src2 = bitcast <8 x i16> %s2 to <16 x i8>
   %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
   ret <16 x i8> %out
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vqmovni16_sminmax_t2(<8 x i16> %s0, <16 x i8> %src1) {
 ; CHECK-LABEL: vqmovni16_sminmax_t2:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vqmovnb.s16 q0, q0
-; CHECK-NEXT:    vmovlb.s8 q0, q0
 ; CHECK-NEXT:    vmovnt.i16 q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp sgt <8 x i16> %s0, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
   %s1 = select <8 x i1> %c1, <8 x i16> %s0, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
   %c2 = icmp slt <8 x i16> %s1, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %s2 = select <8 x i1> %c2, <8 x i16> %s1, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %src2 = bitcast <8 x i16> %s2 to <16 x i8>
   %out = shufflevector <16 x i8> %src1, <16 x i8> %src2, <16 x i32> <i32 16, i32 0, i32 18, i32 2, i32 20, i32 4, i32 22, i32 6, i32 24, i32 8, i32 26, i32 10, i32 28, i32 12, i32 30, i32 14>
   ret <16 x i8> %out
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vqmovni16_sminmax_b1(<8 x i16> %s0, <16 x i8> %src1) {
 ; CHECK-LABEL: vqmovni16_sminmax_b1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vqmovnb.s16 q0, q0
-; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vmovnb.i16 q1, q0
+; CHECK-NEXT:    vqmovnb.s16 q1, q0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp sgt <8 x i16> %s0, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
   %s1 = select <8 x i1> %c1, <8 x i16> %s0, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
   %c2 = icmp slt <8 x i16> %s1, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %s2 = select <8 x i1> %c2, <8 x i16> %s1, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
   %src2 = bitcast <8 x i16> %s2 to <16 x i8>