[ARM] Always replace FP16 bitcasts with VMOVhr or VMOVrh

[dmgreen]

Summary

This changes the logic with lowering fp16 bitcasts to always produce either a VMOVhr or a VMOVrh, instead of only trying to do it with certain surrounding nodes. To perform the same optimisations demand bits and known bits information has been added for them.

[efriedma]

LGTM

llvm/lib/Target/ARM/ARMISelLowering.cpp

13002–13008

"register"

llvm/lib/Target/ARM/ARMISelLowering.cpp

 /// operand type is illegal (e.g., v2f32 for a target that doesn't support
 /// vectors), since the legalizer won't know what to do with that.
 static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
                              const ARMSubtarget *Subtarget) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDLoc dl(N);
   SDValue Op = N->getOperand(0);
 
-  // This function is only supposed to be called for i64 types, either as the
-  // source or destination of the bit convert.
+  // This function is only supposed to be called for i16 and i64 types, either
+  // as the source or destination of the bit convert.
   EVT SrcVT = Op.getValueType();
   EVT DstVT = N->getValueType(0);
-  const bool HasFullFP16 = Subtarget->hasFullFP16();
 
   if (SrcVT == MVT::i16 && DstVT == MVT::f16) {
-    if (!HasFullFP16)
-      return SDValue();
-    // SoftFP: read half-precision arguments:
-    //
-    // t2: i32,ch = ...
-    //        t7: i16 = truncate t2 <~~~~ Op
-    //      t8: f16 = bitcast t7    <~~~~ N
-    //
-    if (Op.getOperand(0).getValueType() == MVT::i32)
-      return DAG.getNode(ARMISD::VMOVhr, SDLoc(Op),
-                         MVT::f16, Op.getOperand(0));
-
+    if (!Subtarget->hasFullFP16())
       return SDValue();
+    // f16 bitcast i16 -> VMOVhr
+    return DAG.getNode(ARMISD::VMOVhr, SDLoc(N), MVT::f16,
+                       DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op));
   }
 
-  // Half-precision return values
   if (SrcVT == MVT::f16 && DstVT == MVT::i16) {
-    if (!HasFullFP16)
-      return SDValue();
-    //
-    //          t11: f16 = fadd t8, t10
-    //        t12: i16 = bitcast t11       <~~~ SDNode N
-    //      t13: i32 = zero_extend t12
-    //    t16: ch,glue = CopyToReg t0, Register:i32 %r0, t13
-    //  t17: ch = ARMISD::RET_FLAG t16, Register:i32 %r0, t16:1
-    //
-    // transform this into:
-    //
-    //    t20: i32 = ARMISD::VMOVrh t11
-    //  t16: ch,glue = CopyToReg t0, Register:i32 %r0, t20
-    //
-    auto ZeroExtend = N->use_begin();
-    if (N->use_size() != 1 || ZeroExtend->getOpcode() != ISD::ZERO_EXTEND ||
-        ZeroExtend->getValueType(0) != MVT::i32)
-      return SDValue();
-
-    auto Copy = ZeroExtend->use_begin();
-    if (Copy->getOpcode() == ISD::CopyToReg &&
-        Copy->use_begin()->getOpcode() == ARMISD::RET_FLAG) {
-      SDValue Cvt = DAG.getNode(ARMISD::VMOVrh, SDLoc(Op), MVT::i32, Op);
-      DAG.ReplaceAllUsesWith(*ZeroExtend, &Cvt);
-      return Cvt;
-    }
+    if (!Subtarget->hasFullFP16())
       return SDValue();
+    // i16 bitcast f16 -> VMOVrh
+    return DAG.getNode(ISD::TRUNCATE, SDLoc(N), MVT::i16,
+                       DAG.getNode(ARMISD::VMOVrh, SDLoc(N), MVT::i32, Op));
   }
 
   if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
     return SDValue();
 
   // Turn i64->f64 into VMOVDRR.
   if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) {
     // Do not force values to GPRs (this is what VMOVDRR does for the inputs)

 
 static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
   // FullFP16: half values are passed in S-registers, and we don't
   // need any of the bitcast and moves:
   //
   //     t2: f32,ch = CopyFromReg t0, Register:f32 %0
   //   t5: i32 = bitcast t2
   // t18: f16 = ARMISD::VMOVhr t5
-  SDValue BC = N->getOperand(0);
-  if (BC->getOpcode() != ISD::BITCAST)
-    return SDValue();
-  SDValue Copy = BC->getOperand(0);
-  if (Copy.getValueType() != MVT::f32 || Copy->getOpcode() != ISD::CopyFromReg)
-    return SDValue();
-
+  SDValue Op0 = N->getOperand(0);
+  if (Op0->getOpcode() == ISD::BITCAST) {
+    SDValue Copy = Op0->getOperand(0);
+    if (Copy.getValueType() == MVT::f32 &&
+        Copy->getOpcode() == ISD::CopyFromReg) {
       SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1)};
-  SDValue NewCopy = DCI.DAG.getNode(ISD::CopyFromReg, SDLoc(N), MVT::f16, Ops);
+      SDValue NewCopy =
+          DCI.DAG.getNode(ISD::CopyFromReg, SDLoc(N), MVT::f16, Ops);
       return NewCopy;
     }
+  }
+
+  // Only the bottom 16 bits of the source register are used.
+  APInt DemandedMask = APInt::getLowBitsSet(32, 16);
+  const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
+  if (TLI.SimplifyDemandedBits(Op0, DemandedMask, DCI))
+    return SDValue(N, 0);
+
+  return SDValue();
+}
 
 static SDValue PerformVMOVrhCombine(SDNode *N,
                                     TargetLowering::DAGCombinerInfo &DCI) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
 
   // fold (VMOVrh (load x)) -> (zextload (i16*)x)
   if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse()) {
case ARMISD::VGETLANEu: {
     if (Op.getOpcode() == ARMISD::VGETLANEs)
       Known = Known.sext(DstSz);
     else {
       Known = Known.zext(DstSz);
     }
     assert(DstSz == Known.getBitWidth());
     break;
   }
+  case ARMISD::VMOVrh: {
+    KnownBits KnownOp = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
+    assert(KnownOp.getBitWidth() == 16);
+    Known = KnownOp.zext(32);
+    break;
+  }
   }
 }
 
 bool
 ARMTargetLowering::targetShrinkDemandedConstant(SDValue Op,
                                                 const APInt &DemandedAPInt,
                                                 TargetLoweringOpt &TLO) const {
   // Delay optimization, so we don't have to deal with illegal types, or block

llvm/test/CodeGen/Thumb2/mve-intrinsics/vminvq.ll

 entry:
   %0 = tail call i32 @llvm.arm.mve.maxav.v4i32(i32 %a, <4 x i32> %b)
   ret i32 %0
 }
 
 define arm_aapcs_vfpcc float @test_vminnmvq_f16(float %a.coerce, <8 x half> %b) {
 ; CHECK-LABEL: test_vminnmvq_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vminnmv.f16 r0, q1
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast float %a.coerce to i32
   %tmp.0.extract.trunc = trunc i32 %0 to i16
   %1 = bitcast i16 %tmp.0.extract.trunc to half
   %2 = tail call half @llvm.arm.mve.minnmv.f16.v8f16(half %1, <8 x half> %b)
   %3 = bitcast half %2 to i16
   %tmp2.0.insert.ext = zext i16 %3 to i32

 entry:
   %0 = tail call float @llvm.arm.mve.minnmv.f32.v4f32(float %a, <4 x float> %b)
   ret float %0
 }
 
 define arm_aapcs_vfpcc float @test_vminnmavq_f16(float %a.coerce, <8 x half> %b) {
 ; CHECK-LABEL: test_vminnmavq_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vminnmav.f16 r0, q1
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast float %a.coerce to i32
   %tmp.0.extract.trunc = trunc i32 %0 to i16
   %1 = bitcast i16 %tmp.0.extract.trunc to half
   %2 = tail call half @llvm.arm.mve.minnmav.f16.v8f16(half %1, <8 x half> %b)
   %3 = bitcast half %2 to i16
   %tmp2.0.insert.ext = zext i16 %3 to i32

 entry:
   %0 = tail call float @llvm.arm.mve.minnmav.f32.v4f32(float %a, <4 x float> %b)
   ret float %0
 }
 
 define arm_aapcs_vfpcc float @test_vmaxnmvq_f16(float %a.coerce, <8 x half> %b) {
 ; CHECK-LABEL: test_vmaxnmvq_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmaxnmv.f16 r0, q1
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast float %a.coerce to i32
   %tmp.0.extract.trunc = trunc i32 %0 to i16
   %1 = bitcast i16 %tmp.0.extract.trunc to half
   %2 = tail call half @llvm.arm.mve.maxnmv.f16.v8f16(half %1, <8 x half> %b)
   %3 = bitcast half %2 to i16
   %tmp2.0.insert.ext = zext i16 %3 to i32

 entry:
   %0 = tail call float @llvm.arm.mve.maxnmv.f32.v4f32(float %a, <4 x float> %b)
   ret float %0
 }
 
 define arm_aapcs_vfpcc float @test_vmaxnmavq_f16(float %a.coerce, <8 x half> %b) {
 ; CHECK-LABEL: test_vmaxnmavq_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmaxnmav.f16 r0, q1
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast float %a.coerce to i32
   %tmp.0.extract.trunc = trunc i32 %0 to i16
   %1 = bitcast i16 %tmp.0.extract.trunc to half
   %2 = tail call half @llvm.arm.mve.maxnmav.f16.v8f16(half %1, <8 x half> %b)
   %3 = bitcast half %2 to i16
   %tmp2.0.insert.ext = zext i16 %3 to i32
entry:
   %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
   %2 = tail call i32 @llvm.arm.mve.maxav.predicated.v4i32.v4i1(i32 %a, <4 x i32> %b, <4 x i1> %1)
   ret i32 %2
 }
 
 define arm_aapcs_vfpcc float @test_vminnmvq_p_f16(float %a.coerce, <8 x half> %b, i16 zeroext %p) {
 ; CHECK-LABEL: test_vminnmvq_p_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vminnmvt.f16 r1, q1
 ; CHECK-NEXT:    vmov s0, r1
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast float %a.coerce to i32
   %tmp.0.extract.trunc = trunc i32 %0 to i16
   %1 = bitcast i16 %tmp.0.extract.trunc to half
   %2 = zext i16 %p to i32
   %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
   %4 = tail call half @llvm.arm.mve.minnmv.predicated.f16.v8f16.v8i1(half %1, <8 x half> %b, <8 x i1> %3)
entry:
   %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
   %2 = tail call float @llvm.arm.mve.minnmv.predicated.f32.v4f32.v4i1(float %a, <4 x float> %b, <4 x i1> %1)
   ret float %2
 }
 
 define arm_aapcs_vfpcc float @test_vminnmavq_p_f16(float %a.coerce, <8 x half> %b, i16 zeroext %p) {
 ; CHECK-LABEL: test_vminnmavq_p_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vminnmavt.f16 r1, q1
 ; CHECK-NEXT:    vmov s0, r1
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast float %a.coerce to i32
   %tmp.0.extract.trunc = trunc i32 %0 to i16
   %1 = bitcast i16 %tmp.0.extract.trunc to half
   %2 = zext i16 %p to i32
   %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
   %4 = tail call half @llvm.arm.mve.minnmav.predicated.f16.v8f16.v8i1(half %1, <8 x half> %b, <8 x i1> %3)
entry:
   %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
   %2 = tail call float @llvm.arm.mve.minnmav.predicated.f32.v4f32.v4i1(float %a, <4 x float> %b, <4 x i1> %1)
   ret float %2
 }
 
 define arm_aapcs_vfpcc float @test_vmaxnmvq_p_f16(float %a.coerce, <8 x half> %b, i16 zeroext %p) {
 ; CHECK-LABEL: test_vmaxnmvq_p_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vmaxnmvt.f16 r1, q1
 ; CHECK-NEXT:    vmov s0, r1
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast float %a.coerce to i32
   %tmp.0.extract.trunc = trunc i32 %0 to i16
   %1 = bitcast i16 %tmp.0.extract.trunc to half
   %2 = zext i16 %p to i32
   %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
   %4 = tail call half @llvm.arm.mve.maxnmv.predicated.f16.v8f16.v8i1(half %1, <8 x half> %b, <8 x i1> %3)
entry:
   %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
   %2 = tail call float @llvm.arm.mve.maxnmv.predicated.f32.v4f32.v4i1(float %a, <4 x float> %b, <4 x i1> %1)
   ret float %2
 }
 
 define arm_aapcs_vfpcc float @test_vmaxnmavq_p_f16(float %a.coerce, <8 x half> %b, i16 zeroext %p) {
 ; CHECK-LABEL: test_vmaxnmavq_p_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vmaxnmavt.f16 r1, q1
 ; CHECK-NEXT:    vmov s0, r1
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vmov s0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast float %a.coerce to i32
   %tmp.0.extract.trunc = trunc i32 %0 to i16
   %1 = bitcast i16 %tmp.0.extract.trunc to half
   %2 = zext i16 %p to i32
   %3 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %2)
   %4 = tail call half @llvm.arm.mve.maxnmav.predicated.f16.v8f16.v8i1(half %1, <8 x half> %b, <8 x i1> %3)

llvm/test/CodeGen/Thumb2/mve-vdup.ll

entry:
   %3 = insertelement <8 x half> undef, half %2, i32 0
   %out = shufflevector <8 x half> %3, <8 x half> undef, <8 x i32> zeroinitializer
   ret <8 x half> %out
 }
 
 define arm_aapcs_vfpcc <8 x half> @vdup_f16_bc(half* %src1, half* %src2) {
 ; CHECK-LABEL: vdup_f16_bc:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vldr.16 s2, [r0]
 ; CHECK-NEXT:    vadd.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r0, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r0, s0
 ; CHECK-NEXT:    vdup.16 q0, r0
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load half, half *%src1, align 2
   %1 = load half, half *%src2, align 2
   %2 = fadd half %0, %1
   %bc = bitcast half %2 to i16
   %3 = insertelement <8 x i16> undef, i16 %bc, i32 0
   %out = shufflevector <8 x i16> %3, <8 x i16> undef, <8 x i32> zeroinitializer
entry:
   %outbc = bitcast <4 x i32> %out to <4 x float>
   %ext = extractelement <4 x float> %outbc, i32 2
   ret float %ext
 }
 
 define arm_aapcs_vfpcc half @vdup_f16_extract(half* %src1, half* %src2) {
 ; CHECK-LABEL: vdup_f16_extract:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldr.16 s0, [r2]
 ; CHECK-NEXT:    vldr.16 s2, [r1]
 ; CHECK-NEXT:    vadd.f16 s0, s2, s0
-; CHECK-NEXT:    vstr.16 s0, [sp, #2]
-; CHECK-NEXT:    ldrh.w r1, [sp, #2]
+; CHECK-NEXT:    vmov.f16 r1, s0
 ; CHECK-NEXT:    vdup.16 q0, r1
 ; CHECK-NEXT:    vstr.16 s1, [r0]
-; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load half, half *%src1, align 2
   %1 = load half, half *%src2, align 2
   %2 = fadd half %0, %1
   %bc = bitcast half %2 to i16
   %3 = insertelement <8 x i16> undef, i16 %bc, i32 0
   %out = shufflevector <8 x i16> %3, <8 x i16> undef, <8 x i32> zeroinitializer
   %outbc = bitcast <8 x i16> %out to <8 x half>
   %ext = extractelement <8 x half> %outbc, i32 2
   ret half %ext
 }