[X86] Optimize fdiv with reciprocal instructions for half type

[pengfei]

Fix scalar problem.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

23161

This is a SCALAR_TO_VECTOR node

23203

SCALAR_TO_VECTOR

[pengfei]

Thanks Simon. I had seen the discussion about INSERT_VECTOR_ELT and SCALAR_TO_VECTOR, but forgot the conclusion. Do you remember the reason why we prefer the latter?

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

23156

Should we keep the user RefinementStep if it isn't ReciprocalEstimate::Unspecified?

23198

Same question as above.

[pengfei]

Keep the user RefinementStep when ReciprocalEstimate::Unspecified.

llvm/lib/Target/X86/X86ISelLowering.cpp

23156

Yes, we should respect user RefinementStep here. Thanks Craig.

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

23197

You need to know the type is legal. v64f16 will get through this, but type legalization won't be able to split a v64f16 X86ISD::RCP14 node.

[pengfei]

Address review comment.

llvm/lib/Target/X86/X86ISelLowering.cpp

23197

Yes. Thanks Craig!

[craig.topper]

LGTM

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

 ///     does not require additional intermediate precision]
 /// For the last iteration, put numerator N into it to gain more precision:
 ///   Result = N X_i + X_i (N - N A X_i)
 SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op,
                                       SDNodeFlags Flags) {
   if (LegalDAG)
     return SDValue();
 
-  // TODO: Handle half and/or extended types?
+  // TODO: Handle extended types?
   EVT VT = Op.getValueType();
-  if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64)
+  if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 &&
+      VT.getScalarType() != MVT::f64)
     return SDValue();
 
   // If estimates are explicitly disabled for this function, we're done.
   MachineFunction &MF = DAG.getMachineFunction();
   int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF);
   if (Enabled == TLI.ReciprocalEstimate::Disabled)
     return SDValue();
 

 /// Build code to calculate either rsqrt(Op) or sqrt(Op). In the latter case
 /// Op*rsqrt(Op) is actually computed, so additional postprocessing is needed if
 /// Op can be zero.
 SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags,
                                            bool Reciprocal) {
   if (LegalDAG)
     return SDValue();
 
-  // TODO: Handle half and/or extended types?
+  // TODO: Handle extended types?
   EVT VT = Op.getValueType();
-  if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64)
+  if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 &&
+      VT.getScalarType() != MVT::f64)
     return SDValue();
 
   // If estimates are explicitly disabled for this function, we're done.
   MachineFunction &MF = DAG.getMachineFunction();
   int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF);
   if (Enabled == TLI.ReciprocalEstimate::Disabled)
     return SDValue();
 

llvm/lib/Target/X86/X86ISelLowering.cpp

if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = 1;
 
     UseOneConstNR = false;
     // There is no FSQRT for 512-bits, but there is RSQRT14.
     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;
     return DAG.getNode(Opcode, SDLoc(Op), VT, Op);
   }
+
+  if (VT.getScalarType() == MVT::f16 && Subtarget.hasFP16()) {
+    if (VT == MVT::f16)
+      return SDValue();
+
+    RefinementSteps = 0;
+    return DAG.getNode(X86ISD::RSQRT14, SDLoc(Op), VT, Op);
+  }
   return SDValue();
 }
 
 /// The minimum architected relative accuracy is 2^-12. We need one
 /// Newton-Raphson step to have a good float result (24 bits of precision).
 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
                                             int Enabled,
                                             int &RefinementSteps) const {
if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
 
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = 1;
 
     // There is no FSQRT for 512-bits, but there is RCP14.
     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;
     return DAG.getNode(Opcode, SDLoc(Op), VT, Op);
   }
+
+  if (VT.getScalarType() == MVT::f16 && Subtarget.hasFP16()) {
+    if (VT == MVT::f16)
+      return SDValue();
+
+    RefinementSteps = 0;
+    return DAG.getNode(X86ISD::RCP14, SDLoc(Op), VT, Op);
+  }
   return SDValue();
 }
 
 /// If we have at least two divisions that use the same divisor, convert to
 /// multiplication by a reciprocal. This may need to be adjusted for a given
 /// CPU if a division's cost is not at least twice the cost of a multiplication.
 /// This is because we still need one division to calculate the reciprocal and
 /// then we need two multiplies by that reciprocal as replacements for the

llvm/test/CodeGen/X86/avx512fp16-arith-vl-intrinsics.ll

 ; CHECK-LABEL: test_int_x86_avx512fp16_div_ph_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivph %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    retq
   %res = fdiv <16 x half> %x1, %x2
   ret <16 x half> %res
 }
 
+define <16 x half> @test_int_x86_avx512fp16_div_ph_256_fast(<16 x half> %x1, <16 x half> %x2) {
+; CHECK-LABEL: test_int_x86_avx512fp16_div_ph_256_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrcpph %ymm1, %ymm1
+; CHECK-NEXT:    vmulph %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = fdiv fast <16 x half> %x1, %x2
+  ret <16 x half> %res
+}
+
 define <16 x half> @test_int_x86_avx512fp16_mask_div_ph_256(<16 x half> %x1, <16 x half> %x2, <16 x half> %src, i16 %mask, <16 x half>* %ptr) {
 ; CHECK-LABEL: test_int_x86_avx512fp16_mask_div_ph_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovaps %ymm2, %ymm3
 ; CHECK-NEXT:    vdivph %ymm1, %ymm0, %ymm3 {%k1}
 ; CHECK-NEXT:    vdivph (%rsi), %ymm0, %ymm2 {%k1}
 ; CHECK-NEXT:    vdivph %ymm2, %ymm3, %ymm0

 ; CHECK-LABEL: test_int_x86_avx512fp16_div_ph_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivph %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = fdiv <8 x half> %x1, %x2
   ret <8 x half> %res
 }
 
+define <8 x half> @test_int_x86_avx512fp16_div_ph_128_fast(<8 x half> %x1, <8 x half> %x2) {
+; CHECK-LABEL: test_int_x86_avx512fp16_div_ph_128_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrcpph %xmm1, %xmm1
+; CHECK-NEXT:    vmulph %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = fdiv fast <8 x half> %x1, %x2
+  ret <8 x half> %res
+}
+
 define <8 x half> @test_int_x86_avx512fp16_mask_div_ph_128(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, <8 x half>* %ptr) {
 ; CHECK-LABEL: test_int_x86_avx512fp16_mask_div_ph_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm3
 ; CHECK-NEXT:    vdivph %xmm1, %xmm0, %xmm3 {%k1}
 ; CHECK-NEXT:    vdivph (%rsi), %xmm0, %xmm2 {%k1}
 ; CHECK-NEXT:    vdivph %xmm2, %xmm3, %xmm0

llvm/test/CodeGen/X86/avx512fp16-arith.ll

 ; CHECK-LABEL: vdivph_512_test:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vdivph %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    retq
   %x = fdiv  <32 x half> %i, %j
   ret <32 x half> %x
 }
 
+define <32 x half> @vdivph_512_test_fast(<32 x half> %i, <32 x half> %j) nounwind readnone {
+; CHECK-LABEL: vdivph_512_test_fast:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vrcpph %zmm1, %zmm1
+; CHECK-NEXT:    vmulph %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %x = fdiv fast <32 x half> %i, %j
+  ret <32 x half> %x
+}
+
 define half @add_sh(half %i, half %j, half* %x.ptr) nounwind readnone {
 ; CHECK-LABEL: add_sh:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vaddsh %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vaddsh (%rdi), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %x = load half, half* %x.ptr
   %y = fadd  half %i, %j

 ; CHECK-NEXT:    vdivsh (%rdi), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %x = load half, half* %x.ptr
   %y = fdiv  half %i, %j
   %r = fdiv  half %y, %x
   ret half %r
 }
 
+; FIXME: We may need to support scalar as well.
+define half @div_sh_3(half %i, half %j) nounwind readnone {
+; CHECK-LABEL: div_sh_3:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vdivsh %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %r = fdiv fast half %i, %j
+  ret half %r
+}
+
 define i1 @cmp_une_sh(half %x, half %y) {
 ; CHECK-LABEL: cmp_une_sh:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    vcmpneqsh %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
 entry:

llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll

 ; CHECK-LABEL: test_sqrt_ph_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtph %zmm0, %zmm0
 ; CHECK-NEXT:    retq
   %1 = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
   ret <32 x half> %1
 }
 
+define <32 x half> @test_sqrt_ph_512_fast(<32 x half> %a0, <32 x half> %a1) {
+; CHECK-LABEL: test_sqrt_ph_512_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrsqrtph %zmm0, %zmm0
+; CHECK-NEXT:    vmulph %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %1 = call fast <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
+  %2 = fdiv fast <32 x half> %a1, %1
+  ret <32 x half> %2
+}
+
 define <32 x half> @test_mask_sqrt_ph_512(<32 x half> %a0, <32 x half> %passthru, i32 %mask) {
 ; CHECK-LABEL: test_mask_sqrt_ph_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vsqrtph %zmm0, %zmm1 {%k1}
 ; CHECK-NEXT:    vmovaps %zmm1, %zmm0
 ; CHECK-NEXT:    retq
   %1 = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)

llvm/test/CodeGen/X86/avx512fp16vl-intrinsics.ll

 ; CHECK-LABEL: test_sqrt_ph_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtph %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %1 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> %a0)
   ret <8 x half> %1
 }
 
+define <8 x half> @test_sqrt_ph_128_fast(<8 x half> %a0, <8 x half> %a1) {
+; CHECK-LABEL: test_sqrt_ph_128_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrsqrtph %xmm0, %xmm0
+; CHECK-NEXT:    vmulph %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = call fast <8 x half> @llvm.sqrt.v8f16(<8 x half> %a0)
+  %2 = fdiv fast <8 x half> %a1, %1
+  ret <8 x half> %2
+}
+
 define <8 x half> @test_mask_sqrt_ph_128(<8 x half> %a0, <8 x half> %passthru, i8 %mask) {
 ; CHECK-LABEL: test_mask_sqrt_ph_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vsqrtph %xmm0, %xmm1 {%k1}
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %1 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> %a0)

 ; CHECK-LABEL: test_sqrt_ph_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtph %ymm0, %ymm0
 ; CHECK-NEXT:    retq
   %1 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> %a0)
   ret <16 x half> %1
 }
 
+define <16 x half> @test_sqrt_ph_256_fast(<16 x half> %a0, <16 x half> %a1) {
+; CHECK-LABEL: test_sqrt_ph_256_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrsqrtph %ymm0, %ymm0
+; CHECK-NEXT:    vmulph %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %1 = call fast <16 x half> @llvm.sqrt.v16f16(<16 x half> %a0)
+  %2 = fdiv fast <16 x half> %a1, %1
+  ret <16 x half> %2
+}
+
 define <16 x half> @test_mask_sqrt_ph_256(<16 x half> %a0, <16 x half> %passthru, i16 %mask) {
 ; CHECK-LABEL: test_mask_sqrt_ph_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vsqrtph %ymm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovaps %ymm1, %ymm0
 ; CHECK-NEXT:    retq
   %1 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> %a0)