[X86][ISel] Lowering FROUND(f16) and FROUNDEVEN(f16)

[FreddyYe]

Summary

When AVX512FP16 is enabled, FROUND(f16) cannot be dealt with
TypeLegalize, and no libcall in libm is ready for fround(f16) now.
FROUNDEVEN(f16) has related instruction in AVX512FP16.

[FreddyYe]

Example reproduce: https://gcc.godbolt.org/z/czrY74sxv

[craig.topper]

llvm/test/CodeGen/X86/fp-roundeven.ll

45

Why can’t this just be vrndscalesh?

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

1954

What about strict_fround?

Where are tests for strict_froundeven?

[FreddyYe]

llvm/test/CodeGen/X86/fp-roundeven.ll

45

Yes, It can. It was a mistake. I'll update, including the summary.

[FreddyYe]

llvm/lib/Target/X86/X86ISelLowering.cpp

1954

I used a report_fatal_error to deal with strict_fround.
Added.

[FreddyYe]

Thanks for review! Addressed comments.

[FreddyYe]

nfc refine.

[pengfei]

llvm/lib/Target/X86/X86ISelLowering.cpp

1954–1957

How about the vector type?

22503

Can we customize STRICT_FROUND here too? I think you just need to chain the FP nodes togerther.

[FreddyYe]

llvm/lib/Target/X86/X86ISelLowering.cpp

1954–1957

For now, vector expands to vrndscalesh. While vrndscaleph can be emited. I'll refine, too.

22503

@craig.topper created this function originally. He mentions

Constrained intrinsics would use STRICT_FROUND which won't go through this code.

at https://reviews.llvm.org/D73607. Hi Craig, can we chain the FP nodes here?

[pengfei]

llvm/lib/Target/X86/X86ISelLowering.cpp

22503

Oh, I see. Can we return SDValue() so that we can call lib function finally?

[FreddyYe]

llvm/lib/Target/X86/X86ISelLowering.cpp

22503

No, that causes abort for now since round(fp16) is not handled yet in LLVM. I suppose libm hasn't supoprted yet?

[pengfei]

llvm/lib/Target/X86/X86ISelLowering.cpp

22503

Right, we don't have lib functions for FP16. I think it's OK to report error for now.

[craig.topper]

LGTM

[FreddyYe]

Refine tests.

llvm/lib/Target/X86/X86ISelLowering.cpp

if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
     // AVX512_FP16 scalar operations
     setGroup(MVT::f16);
     addRegisterClass(MVT::f16,    &X86::FR16XRegClass);
     setOperationAction(ISD::SELECT_CC,            MVT::f16, Expand);
     setOperationAction(ISD::BR_CC,                MVT::f16, Expand);
     setOperationAction(ISD::SETCC,                MVT::f16, Custom);
     setOperationAction(ISD::STRICT_FSETCC,        MVT::f16, Custom);
     setOperationAction(ISD::STRICT_FSETCCS,       MVT::f16, Custom);
+    setOperationAction(ISD::FROUND,               MVT::f16, Custom);
+    setOperationAction(ISD::STRICT_FROUND,        MVT::f16, Custom);
+    setOperationAction(ISD::FROUNDEVEN,           MVT::f16, Legal);
+    setOperationAction(ISD::STRICT_FROUNDEVEN,    MVT::f16, Legal);
     setOperationAction(ISD::FP_ROUND,             MVT::f16, Custom);
     setOperationAction(ISD::STRICT_FP_ROUND,      MVT::f16, Custom);
     setOperationAction(ISD::STRICT_FP_EXTEND,     MVT::f32, Legal);
     if (isTypeLegal(MVT::f80)) {
       setOperationAction(ISD::FP_EXTEND,          MVT::f80, Custom);
       setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::f80, Custom);
     }
 
SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
   return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
 }
 
 /// ISD::FROUND is defined to round to nearest with ties rounding away from 0.
 /// This mode isn't supported in hardware on X86. But as long as we aren't
 /// compiling with trapping math, we can emulate this with
 /// floor(X + copysign(nextafter(0.5, 0.0), X)).
 static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
+  if (Op.getOpcode() == ISD::STRICT_FROUND &&
+      Op.getSimpleValueType() == MVT::f16)
+    report_fatal_error("For now cannot emit strict round(fp16) at backend for "
+                       "lacking library support.");
   SDValue N0 = Op.getOperand(0);
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
 
   // N0 += copysign(nextafter(0.5, 0.0), N0)
   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
   bool Ignored;
   APFloat Point5Pred = APFloat(0.5f);
SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FP16_TO_FP:
   case ISD::STRICT_FP16_TO_FP:  return LowerFP16_TO_FP(Op, DAG);
   case ISD::FP_TO_FP16:
   case ISD::STRICT_FP_TO_FP16:  return LowerFP_TO_FP16(Op, DAG);
   case ISD::LOAD:               return LowerLoad(Op, Subtarget, DAG);
   case ISD::STORE:              return LowerStore(Op, Subtarget, DAG);
   case ISD::FADD:
   case ISD::FSUB:               return lowerFaddFsub(Op, DAG);
+  case ISD::STRICT_FROUND:
   case ISD::FROUND:             return LowerFROUND(Op, DAG);
   case ISD::FABS:
   case ISD::FNEG:               return LowerFABSorFNEG(Op, DAG);
   case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
   case ISD::FGETSIGN:           return LowerFGETSIGN(Op, DAG);
   case ISD::LRINT:
   case ISD::LLRINT:             return LowerLRINT_LLRINT(Op, DAG);
   case ISD::SETCC:

llvm/test/CodeGen/X86/fp-round.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE41
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=AVX512FP16
+
+define half @round_f16(half %h) {
+; SSE2-LABEL: round_f16:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    pushq %rax
+; SSE2-NEXT:    .cfi_def_cfa_offset 16
+; SSE2-NEXT:    movzwl %di, %edi
+; SSE2-NEXT:    callq ___extendhfsf2
+; SSE2-NEXT:    callq _roundf
+; SSE2-NEXT:    callq ___truncsfhf2
+; SSE2-NEXT:    popq %rcx
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: round_f16:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    pushq %rax
+; SSE41-NEXT:    .cfi_def_cfa_offset 16
+; SSE41-NEXT:    movzwl %di, %edi
+; SSE41-NEXT:    callq ___extendhfsf2
+; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; SSE41-NEXT:    andps %xmm0, %xmm1
+; SSE41-NEXT:    orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    addss %xmm0, %xmm1
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    roundss $11, %xmm1, %xmm0
+; SSE41-NEXT:    callq ___truncsfhf2
+; SSE41-NEXT:    popq %rcx
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: round_f16:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    pushq %rax
+; AVX1-NEXT:    .cfi_def_cfa_offset 16
+; AVX1-NEXT:    movzwl %di, %edi
+; AVX1-NEXT:    callq ___extendhfsf2
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX1-NEXT:    vorps %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    callq ___truncsfhf2
+; AVX1-NEXT:    popq %rcx
+; AVX1-NEXT:    retq
+;
+; AVX512FP16-LABEL: round_f16:
+; AVX512FP16:       # %bb.0:
+; AVX512FP16-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512FP16-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1,4.9976E-1]
+; AVX512FP16-NEXT:    vpternlogq $248, %xmm1, %xmm0, %xmm2
+; AVX512FP16-NEXT:    vaddsh %xmm2, %xmm0, %xmm0
+; AVX512FP16-NEXT:    vrndscalesh $11, %xmm0, %xmm0, %xmm0
+; AVX512FP16-NEXT:    retq
+entry:
+  %a = call half @llvm.round.f16(half %h)
+  ret half %a
+}
 
 define float @round_f32(float %x) {
 ; SSE2-LABEL: round_f32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    jmp _roundf ## TAILCALL
 ;
 ; SSE41-LABEL: round_f32:
 ; SSE41:       ## %bb.0:

 ; AVX512-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1
 ; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vrndscalepd $11, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %a = call <8 x double> @llvm.round.v8f64(<8 x double> %x)
   ret <8 x double> %a
 }
 
+declare half @llvm.round.f16(half)
 declare float @llvm.round.f32(float)
 declare double @llvm.round.f64(double)
 declare <4 x float> @llvm.round.v4f32(<4 x float>)
 declare <2 x double> @llvm.round.v2f64(<2 x double>)
 declare <8 x float> @llvm.round.v8f32(<8 x float>)
 declare <4 x double> @llvm.round.v4f64(<4 x double>)
 declare <16 x float> @llvm.round.v16f32(<16 x float>)
 declare <8 x double> @llvm.round.v8f64(<8 x double>)

llvm/test/CodeGen/X86/fp-roundeven.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE41
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=AVX512FP16
+
+define half @roundeven_f16(half %h) {
+; SSE-LABEL: roundeven_f16:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    .cfi_def_cfa_offset 16
+; SSE-NEXT:    movzwl %di, %edi
+; SSE-NEXT:    callq ___extendhfsf2
+; SSE-NEXT:    callq _roundevenf
+; SSE-NEXT:    callq ___truncsfhf2
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    retq
+;
+; SSE4_1-LABEL: roundeven_f16:
+; SSE4_1:       ## %bb.0:
+; SSE4_1-NEXT:    pushq %rax
+; SSE4_1-NEXT:    .cfi_def_cfa_offset 16
+; SSE4_1-NEXT:    movzwl %di, %edi
+; SSE4_1-NEXT:    callq ___extendhfsf2
+; SSE4_1-NEXT:    roundss $8, %xmm0, %xmm0
+; SSE4_1-NEXT:    callq ___truncsfhf2
+; SSE4_1-NEXT:    popq %rcx
+; SSE4_1-NEXT:    retq
+;
+; AVX1-LABEL: roundeven_f16:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    pushq %rax
+; AVX1-NEXT:    .cfi_def_cfa_offset 16
+; AVX1-NEXT:    movzwl %di, %edi
+; AVX1-NEXT:    callq ___extendhfsf2
+; AVX1-NEXT:    vroundss $8, %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    callq ___truncsfhf2
+; AVX1-NEXT:    popq %rcx
+; AVX1-NEXT:    retq
+;
+; AVX512FP16-LABEL: roundeven_f16:
+; AVX512FP16:       # %bb.0:
+; AVX512FP16-NEXT:    vrndscalesh     $8, %xmm0, %xmm0, %xmm0
+; AVX512FP16-NEXT:    retq
+entry:
+  %a = call half @llvm.roundeven.f16(half %h)
+  ret half %a
+}
 
 define float @roundeven_f32(float %x) {
 ; SSE2-LABEL: roundeven_f32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    jmp _roundevenf ## TAILCALL
 ;
 ; SSE41-LABEL: roundeven_f32:
 ; SSE41:       ## %bb.0:

 ; AVX512-LABEL: roundeven_v8f64:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vrndscalepd $8, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %a = call <8 x double> @llvm.roundeven.v8f64(<8 x double> %x)
   ret <8 x double> %a
 }
 
+declare half @llvm.roundeven.f16(half)
 declare float @llvm.roundeven.f32(float)
 declare double @llvm.roundeven.f64(double)
 declare <4 x float> @llvm.roundeven.v4f32(<4 x float>)
 declare <2 x double> @llvm.roundeven.v2f64(<2 x double>)
 declare <8 x float> @llvm.roundeven.v8f32(<8 x float>)
 declare <4 x double> @llvm.roundeven.v4f64(<4 x double>)
 declare <16 x float> @llvm.roundeven.v16f32(<16 x float>)
 declare <8 x double> @llvm.roundeven.v8f64(<8 x double>)

llvm/test/CodeGen/X86/fp-strict-scalar-round-fp16.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16 -O3 | FileCheck %s --check-prefixes=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 -O3 | FileCheck %s --check-prefixes=X64
 
 declare half @llvm.experimental.constrained.ceil.f16(half, metadata)
 declare half @llvm.experimental.constrained.floor.f16(half, metadata)
 declare half @llvm.experimental.constrained.trunc.f16(half, metadata)
 declare half @llvm.experimental.constrained.rint.f16(half, metadata, metadata)
 declare half @llvm.experimental.constrained.nearbyint.f16(half, metadata, metadata)
+declare half @llvm.experimental.constrained.roundeven.f16(half, metadata)
 
 define half @fceil32(half %f) #0 {
 ; X86-LABEL: fceil32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vrndscalesh $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: fceil32:

 ; X64-NEXT:    vrndscalesh $12, %xmm0, %xmm0, %xmm0
 ; X64-NEXT:    retq
   %res = call half @llvm.experimental.constrained.nearbyint.f16(
                         half %f,
                         metadata !"round.dynamic", metadata !"fpexcept.strict") #0
   ret half %res
 }
 
+define half @froundeven16(half %f) #0 {
+; X86-LABEL: froundeven16:
+; X86:       # %bb.0:
+; X86-NEXT:    vrndscalesh $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: froundeven16:
+; X64:       # %bb.0:
+; X64-NEXT:    vrndscalesh $8, %xmm0, %xmm0, %xmm0
+; X64-NEXT:    retq
+
+  %res = call half @llvm.experimental.constrained.roundeven.f16(
+                        half %f, metadata !"fpexcept.strict") #0
+  ret half %res
+}
+
 attributes #0 = { strictfp }