[X86] Optimize fdiv with reciprocal instructions for half type

[pengfei]

Fix scalar problem.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

23187

This is a SCALAR_TO_VECTOR node

23237

SCALAR_TO_VECTOR

[pengfei]

Thanks Simon. I had seen the discussion about INSERT_VECTOR_ELT and SCALAR_TO_VECTOR, but forgot the conclusion. Do you remember the reason why we prefer the latter?

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

23182

Should we keep the user RefinementStep if it isn't ReciprocalEstimate::Unspecified?

23232

Same question as above.

[pengfei]

Keep the user RefinementStep when ReciprocalEstimate::Unspecified.

llvm/lib/Target/X86/X86ISelLowering.cpp

23182

Yes, we should respect user RefinementStep here. Thanks Craig.

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

23231

You need to know the type is legal. v64f16 will get through this, but type legalization won't be able to split a v64f16 X86ISD::RCP14 node.

[pengfei]

Address review comment.

llvm/lib/Target/X86/X86ISelLowering.cpp

23231

Yes. Thanks Craig!

[craig.topper]

LGTM

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

 //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //

 ///     does not require additional intermediate precision]
 /// For the last iteration, put numerator N into it to gain more precision:
 ///   Result = N X_i + X_i (N - N A X_i)
 SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op,
                                       SDNodeFlags Flags) {
   if (LegalDAG)
     return SDValue();
 
-  // TODO: Handle half and/or extended types?
+  // TODO: Handle extended types?
   EVT VT = Op.getValueType();
-  if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64)
+  if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 &&
+      VT.getScalarType() != MVT::f64)
     return SDValue();
 
   // If estimates are explicitly disabled for this function, we're done.
   MachineFunction &MF = DAG.getMachineFunction();
   int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF);
   if (Enabled == TLI.ReciprocalEstimate::Disabled)
     return SDValue();
 

 /// Build code to calculate either rsqrt(Op) or sqrt(Op). In the latter case
 /// Op*rsqrt(Op) is actually computed, so additional postprocessing is needed if
 /// Op can be zero.
 SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags,
                                            bool Reciprocal) {
   if (LegalDAG)
     return SDValue();
 
-  // TODO: Handle half and/or extended types?
+  // TODO: Handle extended types?
   EVT VT = Op.getValueType();
-  if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64)
+  if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 &&
+      VT.getScalarType() != MVT::f64)
     return SDValue();
 
   // If estimates are explicitly disabled for this function, we're done.
   MachineFunction &MF = DAG.getMachineFunction();
   int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF);
   if (Enabled == TLI.ReciprocalEstimate::Disabled)
     return SDValue();
 

llvm/lib/Target/X86/X86ISelLowering.cpp

 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //

 
 /// The minimum architected relative accuracy is 2^-12. We need one
 /// Newton-Raphson step to have a good float result (24 bits of precision).
 SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
                                            SelectionDAG &DAG, int Enabled,
                                            int &RefinementSteps,
                                            bool &UseOneConstNR,
                                            bool Reciprocal) const {
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
 
   // SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps.
   // It is likely not profitable to do this for f64 because a double-precision
   // rsqrt estimate with refinement on x86 prior to FMA requires at least 16
   // instructions: convert to single, rsqrtss, convert back to double, refine
   // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
   // along with FMA, this could be a throughput win.
   // TODO: SQRT requires SSE2 to prevent the introduction of an illegal v4i32
   // after legalize types.
   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
       (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) ||
       (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) ||
       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = 1;
 
     UseOneConstNR = false;
     // There is no FSQRT for 512-bits, but there is RSQRT14.
     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;
-    return DAG.getNode(Opcode, SDLoc(Op), VT, Op);
+    return DAG.getNode(Opcode, DL, VT, Op);
+  }
+
+  if (VT.getScalarType() == MVT::f16 && Subtarget.hasFP16()) {
+    if (RefinementSteps == ReciprocalEstimate::Unspecified)
+      RefinementSteps = 0;
+
+    if (VT == MVT::f16) {
+      SDValue Zero = DAG.getIntPtrConstant(0, DL);
+      SDValue Undef = DAG.getUNDEF(MVT::v8f16);
+      Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
+      Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op);
+      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
+    }
+
+    return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op);
   }
   return SDValue();
 }
 
 /// The minimum architected relative accuracy is 2^-12. We need one
 /// Newton-Raphson step to have a good float result (24 bits of precision).
 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
                                             int Enabled,
                                             int &RefinementSteps) const {
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
 
   // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
   // It is likely not profitable to do this for f64 because a double-precision
   // reciprocal estimate with refinement on x86 prior to FMA requires
   // 15 instructions: convert to single, rcpss, convert back to double, refine
   // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
   // along with FMA, this could be a throughput win.
 
   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
       (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
     // Enable estimate codegen with 1 refinement step for vector division.
     // Scalar division estimates are disabled because they break too much
     // real-world code. These defaults are intended to match GCC behavior.
     if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)
       return SDValue();
 
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = 1;
 
     // There is no FSQRT for 512-bits, but there is RCP14.
     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;
-    return DAG.getNode(Opcode, SDLoc(Op), VT, Op);
+    return DAG.getNode(Opcode, DL, VT, Op);
+  }
+
+  if (VT.getScalarType() == MVT::f16 && Subtarget.hasFP16()) {
+    if (RefinementSteps == ReciprocalEstimate::Unspecified)
+      RefinementSteps = 0;
+
+    if (VT == MVT::f16) {
+      SDValue Zero = DAG.getIntPtrConstant(0, DL);
+      SDValue Undef = DAG.getUNDEF(MVT::v8f16);
+      Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
+      Op = DAG.getNode(X86ISD::RCP14S, DL, MVT::v8f16, Undef, Op);
+      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
+    }
+
+    return DAG.getNode(X86ISD::RCP14, DL, VT, Op);
   }
   return SDValue();
 }
 
 /// If we have at least two divisions that use the same divisor, convert to
 /// multiplication by a reciprocal. This may need to be adjusted for a given
 /// CPU if a division's cost is not at least twice the cost of a multiplication.
 /// This is because we still need one division to calculate the reciprocal and

llvm/test/CodeGen/X86/avx512fp16-arith-vl-intrinsics.ll

 ; CHECK-LABEL: test_int_x86_avx512fp16_div_ph_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivph %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    retq
   %res = fdiv <16 x half> %x1, %x2
   ret <16 x half> %res
 }
 
+define <16 x half> @test_int_x86_avx512fp16_div_ph_256_fast(<16 x half> %x1, <16 x half> %x2) {
+; CHECK-LABEL: test_int_x86_avx512fp16_div_ph_256_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrcpph %ymm1, %ymm1
+; CHECK-NEXT:    vmulph %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = fdiv fast <16 x half> %x1, %x2
+  ret <16 x half> %res
+}
+
 define <16 x half> @test_int_x86_avx512fp16_mask_div_ph_256(<16 x half> %x1, <16 x half> %x2, <16 x half> %src, i16 %mask, <16 x half>* %ptr) {
 ; CHECK-LABEL: test_int_x86_avx512fp16_mask_div_ph_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovaps %ymm2, %ymm3
 ; CHECK-NEXT:    vdivph %ymm1, %ymm0, %ymm3 {%k1}
 ; CHECK-NEXT:    vdivph (%rsi), %ymm0, %ymm2 {%k1}
 ; CHECK-NEXT:    vdivph %ymm2, %ymm3, %ymm0

 ; CHECK-LABEL: test_int_x86_avx512fp16_div_ph_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivph %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = fdiv <8 x half> %x1, %x2
   ret <8 x half> %res
 }
 
+define <8 x half> @test_int_x86_avx512fp16_div_ph_128_fast(<8 x half> %x1, <8 x half> %x2) {
+; CHECK-LABEL: test_int_x86_avx512fp16_div_ph_128_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrcpph %xmm1, %xmm1
+; CHECK-NEXT:    vmulph %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %res = fdiv fast <8 x half> %x1, %x2
+  ret <8 x half> %res
+}
+
 define <8 x half> @test_int_x86_avx512fp16_mask_div_ph_128(<8 x half> %x1, <8 x half> %x2, <8 x half> %src, i8 %mask, <8 x half>* %ptr) {
 ; CHECK-LABEL: test_int_x86_avx512fp16_mask_div_ph_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm3
 ; CHECK-NEXT:    vdivph %xmm1, %xmm0, %xmm3 {%k1}
 ; CHECK-NEXT:    vdivph (%rsi), %xmm0, %xmm2 {%k1}
 ; CHECK-NEXT:    vdivph %xmm2, %xmm3, %xmm0

llvm/test/CodeGen/X86/avx512fp16-arith.ll

 ; CHECK-LABEL: vdivph_512_test:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vdivph %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    retq
   %x = fdiv  <32 x half> %i, %j
   ret <32 x half> %x
 }
 
+define <32 x half> @vdivph_512_test_fast(<32 x half> %i, <32 x half> %j) nounwind readnone {
+; CHECK-LABEL: vdivph_512_test_fast:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vrcpph %zmm1, %zmm1
+; CHECK-NEXT:    vmulph %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %x = fdiv fast <32 x half> %i, %j
+  ret <32 x half> %x
+}
+
 define half @add_sh(half %i, half %j, half* %x.ptr) nounwind readnone {
 ; CHECK-LABEL: add_sh:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vaddsh %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vaddsh (%rdi), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %x = load half, half* %x.ptr
   %y = fadd  half %i, %j

 ; CHECK-NEXT:    vdivsh (%rdi), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %x = load half, half* %x.ptr
   %y = fdiv  half %i, %j
   %r = fdiv  half %y, %x
   ret half %r
 }
 
+define half @div_sh_3(half %i, half %j) nounwind readnone {
+; CHECK-LABEL: div_sh_3:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vrcpsh %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vmulsh %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %r = fdiv fast half %i, %j
+  ret half %r
+}
+
 define i1 @cmp_une_sh(half %x, half %y) {
 ; CHECK-LABEL: cmp_une_sh:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    vcmpneqsh %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
 entry:

llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll

 ; CHECK-LABEL: test_sqrt_ph_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtph %zmm0, %zmm0
 ; CHECK-NEXT:    retq
   %1 = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
   ret <32 x half> %1
 }
 
+define <32 x half> @test_sqrt_ph_512_fast(<32 x half> %a0, <32 x half> %a1) {
+; CHECK-LABEL: test_sqrt_ph_512_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrsqrtph %zmm0, %zmm0
+; CHECK-NEXT:    vmulph %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %1 = call fast <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)
+  %2 = fdiv fast <32 x half> %a1, %1
+  ret <32 x half> %2
+}
+
 define <32 x half> @test_mask_sqrt_ph_512(<32 x half> %a0, <32 x half> %passthru, i32 %mask) {
 ; CHECK-LABEL: test_mask_sqrt_ph_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vsqrtph %zmm0, %zmm1 {%k1}
 ; CHECK-NEXT:    vmovaps %zmm1, %zmm0
 ; CHECK-NEXT:    retq
   %1 = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %a0)

 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vsqrtsh %xmm1, %xmm0, %xmm2 {%k1}
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask, i32 4)
   ret <8 x half> %res
 }
 
+define half @test_sqrt_sh2(half %a0, half %a1) {
+; CHECK-LABEL: test_sqrt_sh2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrsqrtsh %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vmulsh %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = call fast half @llvm.sqrt.f16(half %a0)
+  %2 = fdiv fast half %a1, %1
+  ret half %2
+}
+
+declare half @llvm.sqrt.f16(half)
+
 define <8 x half> @test_sqrt_sh_r(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask) {
 ; CHECK-LABEL: test_sqrt_sh_r:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vsqrtsh {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %a0, <8 x half> %a1, <8 x half> %a2, i8 %mask, i32 10)

llvm/test/CodeGen/X86/avx512fp16vl-intrinsics.ll

 ; CHECK-LABEL: test_sqrt_ph_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtph %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %1 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> %a0)
   ret <8 x half> %1
 }
 
+define <8 x half> @test_sqrt_ph_128_fast(<8 x half> %a0, <8 x half> %a1) {
+; CHECK-LABEL: test_sqrt_ph_128_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrsqrtph %xmm0, %xmm0
+; CHECK-NEXT:    vmulph %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = call fast <8 x half> @llvm.sqrt.v8f16(<8 x half> %a0)
+  %2 = fdiv fast <8 x half> %a1, %1
+  ret <8 x half> %2
+}
+
 define <8 x half> @test_mask_sqrt_ph_128(<8 x half> %a0, <8 x half> %passthru, i8 %mask) {
 ; CHECK-LABEL: test_mask_sqrt_ph_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vsqrtph %xmm0, %xmm1 {%k1}
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %1 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> %a0)

 ; CHECK-LABEL: test_sqrt_ph_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtph %ymm0, %ymm0
 ; CHECK-NEXT:    retq
   %1 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> %a0)
   ret <16 x half> %1
 }
 
+define <16 x half> @test_sqrt_ph_256_fast(<16 x half> %a0, <16 x half> %a1) {
+; CHECK-LABEL: test_sqrt_ph_256_fast:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrsqrtph %ymm0, %ymm0
+; CHECK-NEXT:    vmulph %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %1 = call fast <16 x half> @llvm.sqrt.v16f16(<16 x half> %a0)
+  %2 = fdiv fast <16 x half> %a1, %1
+  ret <16 x half> %2
+}
+
 define <16 x half> @test_mask_sqrt_ph_256(<16 x half> %a0, <16 x half> %passthru, i16 %mask) {
 ; CHECK-LABEL: test_mask_sqrt_ph_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vsqrtph %ymm0, %ymm1 {%k1}
 ; CHECK-NEXT:    vmovaps %ymm1, %ymm0
 ; CHECK-NEXT:    retq
   %1 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> %a0)