[X86][FP16] Only generate approximate rsqrt when Reciprocal is true for half type

[pengfei]

Summary

We have reasonable fast sqrt and accurate rsqrt for half type due to the
limited fractions. So neither do we need multi steps refinement for
rsqrt nor replace sqrt by rsqrt.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

23243

Do we have test coverage for this path? A quick grep didn't find anything other than the intrinsics test files.

[pengfei]

llvm/lib/Target/X86/X86ISelLowering.cpp

23243

We have many tests in both changed files to cover the path, e.g. test_sqrt_ph_512_fast
You can't find elsewhere simply because this path is for f16 only and we don't have the support prior to FP16 enabling.

[RKSimon]

LGTM

llvm/lib/Target/X86/X86ISelLowering.cpp

23243

Got it - thanks.

llvm/lib/Target/X86/X86ISelLowering.cpp

static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
   SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
   return Sub.getValue(1);
 }
 
 /// Check if replacement of SQRT with RSQRT should be disabled.
 bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
 
+  // We don't need to replace SQRT with RSQRT for half type.
+  if (VT.getScalarType() == MVT::f16)
+    return true;
+
   // We never want to use both SQRT and RSQRT instructions for the same input.
   if (DAG.getNodeIfExists(X86ISD::FRSQRT, DAG.getVTList(VT), Op))
     return false;
 
   if (VT.isVector())
     return Subtarget.hasFastVectorFSQRT();
   return Subtarget.hasFastScalarFSQRT();
 }
if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
     SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op);
     if (RefinementSteps == 0 && !Reciprocal)
       Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate);
     return Estimate;
   }
 
   if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&
       Subtarget.hasFP16()) {
+    assert(Reciprocal && "Don't replace SQRT with RSQRT for half type");
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = 0;
 
     if (VT == MVT::f16) {
       SDValue Zero = DAG.getIntPtrConstant(0, DL);
       SDValue Undef = DAG.getUNDEF(MVT::v8f16);
       Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
       Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op);

llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll

; CHECK-NEXT:    retq
   %1 = call fast half @llvm.sqrt.f16(half %a0)
   %2 = fdiv fast half %a1, %1
   ret half %2
 }
 
 define half @test_sqrt_sh3(half %a0, half %a1) {
 ; CHECK-LABEL: test_sqrt_sh3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; CHECK-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; CHECK-NEXT:    vcmpltsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %k1
-; CHECK-NEXT:    vrsqrtsh %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
+; CHECK-NEXT:    vsqrtsh %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %1 = call fast half @llvm.sqrt.f16(half %a0)
   ret half %1
 }
 
 declare half @llvm.sqrt.f16(half)
 
 define <8 x half> @test_sqrt_sh_r(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask) {

llvm/test/CodeGen/X86/avx512fp16vl-intrinsics.ll

; CHECK-NEXT:    retq
   %1 = call fast <8 x half> @llvm.sqrt.v8f16(<8 x half> %a0)
   %2 = fdiv fast <8 x half> %a1, %1
   ret <8 x half> %2
 }
 
 define <8 x half> @test_sqrt_ph_128_fast2(<8 x half> %a0, <8 x half> %a1) {
 ; CHECK-LABEL: test_sqrt_ph_128_fast2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; CHECK-NEXT:    vpand %xmm1, %xmm0, %xmm1
-; CHECK-NEXT:    vcmpgeph {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %xmm1, %k1
-; CHECK-NEXT:    vrsqrtph %xmm0, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vsqrtph %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %1 = call fast <8 x half> @llvm.sqrt.v8f16(<8 x half> %a0)
   ret <8 x half> %1
 }
 
 define <8 x half> @test_mask_sqrt_ph_128(<8 x half> %a0, <8 x half> %passthru, i8 %mask) {
 ; CHECK-LABEL: test_mask_sqrt_ph_128:
 ; CHECK:       # %bb.0: