[X86][FP16] Combine two steps conversions into direct conversion

[pengfei]

Summary

When both v8i64 and v4f16 are not legal in a v8i64->v8f16 conversion, legalizer will breaks it into v8i64->v4i64->v4f32->v8f32->v8f16.

Given we support v4i64->v8f16, we can combine them with a shuffle instruction.

[pengfei]

Fix a mistake.

[LuoYuanke]

llvm/lib/Target/X86/X86ISelLowering.cpp

56704

Should we check the number of operands of concat be 2?

56717

Not sure we can transform the node for strict fp operation.

56719

Should we make sure opc is a xint_to_fp node, otherwise return SDValue()?

56728

Ditto.

56736

Why move the check after line 56701?

[pengfei]

llvm/lib/Target/X86/X86ISelLowering.cpp

56704

concat has at least 2 operands. Given we have checked the dest is v8f16 and both source come from v4i64, I think we don't need to check it.

56717

The transform is strict fp compliance. There are totally two different exceptions in this case: precision and overflow. A value out of range of f16 will set both exceptions. A precision exception in i64->f32 always means the value out of range of f16. So we won't discard of introduce any exceptions.

56719

We make sure it's xint_to_fp because we have checked the result type is f32 and input type is i64.

56736

Reuse the code in 56693 line to check src and dst type.

[LuoYuanke]

LGTM

llvm/lib/Target/X86/X86ISelLowering.cpp

static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
 static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG,
                                const X86Subtarget &Subtarget) {
   if (!Subtarget.hasF16C() || Subtarget.useSoftFloat())
     return SDValue();
 
-  if (Subtarget.hasFP16())
-    return SDValue();
-
   bool IsStrict = N->isStrictFPOpcode();
   EVT VT = N->getValueType(0);
   SDValue Src = N->getOperand(IsStrict ? 1 : 0);
   EVT SrcVT = Src.getValueType();
 
   if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
       SrcVT.getVectorElementType() != MVT::f32)
     return SDValue();
 
+  SDLoc dl(N);
+
+  SDValue Cvt, Chain;
   unsigned NumElts = VT.getVectorNumElements();
-  if (NumElts == 1 || !isPowerOf2_32(NumElts))
+  if (Subtarget.hasFP16()) {
+    // Combine (v8f16 fp_round(concat_vectors(v4f32 (xint_to_fp v4i64), ..)))
+    // into (v8f16 vector_shuffle(v8f16 (CVTXI2P v4i64), ..))
+    if (NumElts == 8 && Src.getOpcode() == ISD::CONCAT_VECTORS) {
+      SDValue Cvt0, Cvt1;
+      SDValue Op0 = Src.getOperand(0);
+      SDValue Op1 = Src.getOperand(1);
+      bool IsOp0Strict = Op0->isStrictFPOpcode();
+      if (Op0.getOpcode() != Op1.getOpcode() ||
+          Op0.getOperand(IsOp0Strict ? 1 : 0).getValueType() != MVT::v4i64 ||
+          Op1.getOperand(IsOp0Strict ? 1 : 0).getValueType() != MVT::v4i64) {
         return SDValue();
+      }
+      int Mask[8] = {0, 1, 2, 3, 8, 9, 10, 11};
+      if (IsStrict) {
+        assert(IsOp0Strict && "Op0 must be strict node");
+        unsigned Opc = Op0.getOpcode() == ISD::STRICT_SINT_TO_FP
+                           ? X86ISD::STRICT_CVTSI2P
+                           : X86ISD::STRICT_CVTUI2P;
+        Cvt0 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
+                           {Op0.getOperand(0), Op0.getOperand(1)});
+        Cvt1 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
+                           {Op1.getOperand(0), Op1.getOperand(1)});
+        Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
+        return DAG.getMergeValues({Cvt, Cvt0.getValue(1)}, dl);
+      }
+      unsigned Opc = Op0.getOpcode() == ISD::SINT_TO_FP ? X86ISD::CVTSI2P
+                                                        : X86ISD::CVTUI2P;
+      Cvt0 = DAG.getNode(Opc, dl, MVT::v8f16, Op0.getOperand(0));
+      Cvt1 = DAG.getNode(Opc, dl, MVT::v8f16, Op1.getOperand(0));
+      return Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
+    }
+    return SDValue();
+  }
 
-  SDLoc dl(N);
+  if (NumElts == 1 || !isPowerOf2_32(NumElts))
+    return SDValue();
 
   // Widen to at least 4 input elements.
   if (NumElts < 4)
     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
                       DAG.getConstantFP(0.0, dl, SrcVT));
 
   // Destination is v8i16 with at least 8 elements.
-  EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
-                               std::max(8U, NumElts));
-  SDValue Cvt, Chain;
+  EVT CvtVT =
+      EVT::getVectorVT(*DAG.getContext(), MVT::i16, std::max(8U, NumElts));
   SDValue Rnd = DAG.getTargetConstant(4, dl, MVT::i32);
   if (IsStrict) {
     Cvt = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {CvtVT, MVT::Other},
                       {N->getOperand(0), Src, Rnd});
     Chain = Cvt.getValue(1);
   } else {
     Cvt = DAG.getNode(X86ISD::CVTPS2PH, dl, CvtVT, Src, Rnd);
   }

llvm/test/CodeGen/X86/avx512fp16-cvt.ll

 ; X86-NEXT:    retl
   %a = fptrunc fp128 %x to half
   ret half %a
 }
 
 define <8 x half> @s64tof16(<8 x i64> %a) #0 {
 ; CHECK-LABEL: s64tof16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvtqq2ps %ymm0, %xmm0
-; CHECK-NEXT:    vcvtqq2ps %ymm1, %xmm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-NEXT:    vcvtps2phx %ymm0, %xmm0
+; CHECK-NEXT:    vcvtqq2ph %ymm1, %xmm1
+; CHECK-NEXT:    vcvtqq2ph %ymm0, %xmm0
+; CHECK-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = sitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %1
 }
 
 define <8 x half> @u64tof16(<8 x i64> %a) #0 {
 ; CHECK-LABEL: u64tof16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvtuqq2ps %ymm0, %xmm0
-; CHECK-NEXT:    vcvtuqq2ps %ymm1, %xmm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-NEXT:    vcvtps2phx %ymm0, %xmm0
+; CHECK-NEXT:    vcvtuqq2ph %ymm1, %xmm1
+; CHECK-NEXT:    vcvtuqq2ph %ymm0, %xmm0
+; CHECK-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = uitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %1
 }
 
 attributes #0 = { "min-legal-vector-width"="256" "prefer-vector-width"="256" }

llvm/test/CodeGen/X86/vec-strict-inttofp-256-fp16.ll

%result = call <4 x half> @llvm.experimental.constrained.uitofp.v4f16.v4i64(<4 x i64> %x,
                                                               metadata !"round.dynamic",
                                                               metadata !"fpexcept.strict") #0
   ret <4 x half> %result
 }
 
 define <8 x half> @sitofp_v8i64_v8f16(<8 x i64> %x) #1 {
 ; CHECK-LABEL: sitofp_v8i64_v8f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvtqq2ps %ymm0, %xmm0
-; CHECK-NEXT:    vcvtqq2ps %ymm1, %xmm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-NEXT:    vcvtps2phx %ymm0, %xmm0
+; CHECK-NEXT:    vcvtqq2ph %ymm1, %xmm1
+; CHECK-NEXT:    vcvtqq2ph %ymm0, %xmm0
+; CHECK-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
  %result = call <8 x half> @llvm.experimental.constrained.sitofp.v8f16.v8i64(<8 x i64> %x,
                                                               metadata !"round.dynamic",
                                                               metadata !"fpexcept.strict") #0
   ret <8 x half> %result
 }
 
 define <8 x half> @uitofp_v8i64_v8f16(<8 x i64> %x) #1 {
 ; CHECK-LABEL: uitofp_v8i64_v8f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvtuqq2ps %ymm0, %xmm0
-; CHECK-NEXT:    vcvtuqq2ps %ymm1, %xmm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-NEXT:    vcvtps2phx %ymm0, %xmm0
+; CHECK-NEXT:    vcvtuqq2ph %ymm1, %xmm1
+; CHECK-NEXT:    vcvtuqq2ph %ymm0, %xmm0
+; CHECK-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret{{[l|q]}}
  %result = call <8 x half> @llvm.experimental.constrained.uitofp.v8f16.v8i64(<8 x i64> %x,
                                                               metadata !"round.dynamic",
                                                               metadata !"fpexcept.strict") #0
   ret <8 x half> %result
 }
 
 attributes #0 = { strictfp }
 attributes #1 = { strictfp "min-legal-vector-width"="256" "prefer-vector-width"="256" }