Index: lib/Target/AMDGPU/AMDGPUISelLowering.h =================================================================== --- lib/Target/AMDGPU/AMDGPUISelLowering.h +++ lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -47,7 +47,7 @@ SDValue LowerFRINT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerFROUND32(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const; Index: lib/Target/AMDGPU/AMDGPUISelLowering.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -1733,32 +1733,37 @@ } // XXX - May require not supporting f32 denormals? -SDValue AMDGPUTargetLowering::LowerFROUND32(SDValue Op, SelectionDAG &DAG) const { + +// Don't handle v2f16. The extra instructions to scalarize and repack around the +// compare and vselect end up producing worse code than scalarizing the whole +// operation. +SDValue AMDGPUTargetLowering::LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const { SDLoc SL(Op); SDValue X = Op.getOperand(0); + EVT VT = Op.getValueType(); - SDValue T = DAG.getNode(ISD::FTRUNC, SL, MVT::f32, X); + SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X); // TODO: Should this propagate fast-math-flags? - SDValue Diff = DAG.getNode(ISD::FSUB, SL, MVT::f32, X, T); + SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T); - SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, MVT::f32, Diff); + SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff); - const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f32); - const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); - const SDValue Half = DAG.getConstantFP(0.5, SL, MVT::f32); + const SDValue Zero = DAG.getConstantFP(0.0, SL, VT); + const SDValue One = DAG.getConstantFP(1.0, SL, VT); + const SDValue Half = DAG.getConstantFP(0.5, SL, VT); - SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f32, One, X); + SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X); EVT SetCCVT = - getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32); + getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE); - SDValue Sel = DAG.getNode(ISD::SELECT, SL, MVT::f32, Cmp, SignOne, Zero); + SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero); - return DAG.getNode(ISD::FADD, SL, MVT::f32, T, Sel); + return DAG.getNode(ISD::FADD, SL, VT, T, Sel); } SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const { @@ -1821,8 +1826,8 @@ SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const { EVT VT = Op.getValueType(); - if (VT == MVT::f32) - return LowerFROUND32(Op, DAG); + if (VT == MVT::f32 || VT == MVT::f16) + return LowerFROUND32_16(Op, DAG); if (VT == MVT::f64) return LowerFROUND64(Op, DAG); Index: lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- lib/Target/AMDGPU/SIISelLowering.cpp +++ lib/Target/AMDGPU/SIISelLowering.cpp @@ -365,6 +365,7 @@ setOperationAction(ISD::FP_TO_UINT, MVT::f16, Promote); setOperationAction(ISD::SINT_TO_FP, MVT::f16, Promote); setOperationAction(ISD::UINT_TO_FP, MVT::f16, Promote); + setOperationAction(ISD::FROUND, MVT::f16, Custom); // F16 - VOP2 Actions. setOperationAction(ISD::BR_CC, MVT::f16, Expand); Index: test/CodeGen/AMDGPU/llvm.rint.f16.ll =================================================================== --- test/CodeGen/AMDGPU/llvm.rint.f16.ll +++ test/CodeGen/AMDGPU/llvm.rint.f16.ll @@ -1,5 +1,6 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s -; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=VI -check-prefix=GFX89 %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s declare half @llvm.rint.f16(half %a) declare <2 x half> @llvm.rint.v2f16(<2 x half> %a) @@ -9,7 +10,7 @@ ; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] ; SI: v_rndne_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]] ; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] -; VI: v_rndne_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GFX89: v_rndne_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] ; GCN: buffer_store_short v[[R_F16]] ; GCN: s_endpgm define void @rint_f16( @@ -25,16 +26,26 @@ ; GCN-LABEL: {{^}}rint_v2f16 ; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] ; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] -; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] ; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] ; SI: v_rndne_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]] ; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] ; SI: v_rndne_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]] ; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] + +; VI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] ; VI: v_rndne_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] ; VI: v_rndne_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] -; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] -; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_0]] + +; VI: v_and_b32_e32 v[[R_F16_0]], 0xffff, v[[R_F16_0]] +; SIVI: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; SIVI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_0]] + +; GFX9: v_rndne_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] +; GFX9: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GFX9: v_rndne_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] +; GFX9: v_pack_b32_f16 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_1]] + ; GCN: buffer_store_dword v[[R_V2_F16]] ; GCN: s_endpgm define void @rint_v2f16( Index: test/CodeGen/AMDGPU/llvm.round.ll =================================================================== --- test/CodeGen/AMDGPU/llvm.round.ll +++ test/CodeGen/AMDGPU/llvm.round.ll @@ -1,18 +1,19 @@ -; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}round_f32: -; SI-DAG: s_load_dword [[SX:s[0-9]+]] -; SI-DAG: s_brev_b32 [[K:s[0-9]+]], -2{{$}} -; SI-DAG: v_trunc_f32_e32 [[TRUNC:v[0-9]+]], [[SX]] -; SI-DAG: v_sub_f32_e32 [[SUB:v[0-9]+]], [[SX]], [[TRUNC]] -; SI-DAG: v_mov_b32_e32 [[VX:v[0-9]+]], [[SX]] -; SI: v_bfi_b32 [[COPYSIGN:v[0-9]+]], [[K]], 1.0, [[VX]] -; SI: v_cmp_ge_f32_e64 vcc, |[[SUB]]|, 0.5 -; SI: v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, [[VX]] -; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], [[SEL]], [[TRUNC]] -; SI: buffer_store_dword [[RESULT]] +; GCN-DAG: s_load_dword [[SX:s[0-9]+]] +; GCN-DAG: s_brev_b32 [[K:s[0-9]+]], -2{{$}} +; GCN-DAG: v_trunc_f32_e32 [[TRUNC:v[0-9]+]], [[SX]] +; GCN-DAG: v_sub_f32_e32 [[SUB:v[0-9]+]], [[SX]], [[TRUNC]] +; GCN-DAG: v_mov_b32_e32 [[VX:v[0-9]+]], [[SX]] +; GCN: v_bfi_b32 [[COPYSIGN:v[0-9]+]], [[K]], 1.0, [[VX]] +; GCN: v_cmp_ge_f32_e64 vcc, |[[SUB]]|, 0.5 +; GCN: v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, [[VX]] +; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], [[SEL]], [[TRUNC]] +; GCN: buffer_store_dword [[RESULT]] ; R600: TRUNC {{.*}}, [[ARG:KC[0-9]\[[0-9]+\]\.[XYZW]]] ; R600-DAG: ADD {{.*}}, @@ -32,7 +33,7 @@ ; compiler doesn't crash. ; FUNC-LABEL: {{^}}round_v2f32: -; SI: s_endpgm +; GCN: s_endpgm ; R600: CF_END define void @round_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %in) #0 { %result = call <2 x float> @llvm.round.v2f32(<2 x float> %in) #1 @@ -41,7 +42,7 @@ } ; FUNC-LABEL: {{^}}round_v4f32: -; SI: s_endpgm +; GCN: s_endpgm ; R600: CF_END define void @round_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %in) #0 { %result = call <4 x float> @llvm.round.v4f32(<4 x float> %in) #1 @@ -50,7 +51,7 @@ } ; FUNC-LABEL: {{^}}round_v8f32: -; SI: s_endpgm +; GCN: s_endpgm ; R600: CF_END define void @round_v8f32(<8 x float> addrspace(1)* %out, <8 x float> %in) #0 { %result = call <8 x float> @llvm.round.v8f32(<8 x float> %in) #1 @@ -58,10 +59,51 @@ ret void } +; FUNC-LABEL: {{^}}round_f16: +; GFX89-DAG: s_load_dword [[SX:s[0-9]+]] +; GFX89-DAG: s_movk_i32 [[K:s[0-9]+]], 0x7fff{{$}} +; GFX89-DAG: v_mov_b32_e32 [[VX:v[0-9]+]], [[SX]] +; GFX89-DAG: v_mov_b32_e32 [[BFI_K:v[0-9]+]], 0x3c00 +; GFX89: v_bfi_b32 [[COPYSIGN:v[0-9]+]], [[K]], [[BFI_K]], [[VX]] + +; GFX89: v_trunc_f16_e32 [[TRUNC:v[0-9]+]], [[SX]] +; GFX89: v_sub_f16_e32 [[SUB:v[0-9]+]], [[SX]], [[TRUNC]] +; GFX89: v_cmp_ge_f16_e64 vcc, |[[SUB]]|, 0.5 +; GFX89: v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, [[COPYSIGN]] +; GFX89: v_add_f16_e32 [[RESULT:v[0-9]+]], [[SEL]], [[TRUNC]] +; GFX89: buffer_store_short [[RESULT]] +define void @round_f16(half addrspace(1)* %out, i32 %x.arg) #0 { + %x.arg.trunc = trunc i32 %x.arg to i16 + %x = bitcast i16 %x.arg.trunc to half + %result = call half @llvm.round.f16(half %x) #1 + store half %result, half addrspace(1)* %out + ret void +} + +; Should be scalarized +; FUNC-LABEL: {{^}}round_v2f16: +; GFX89-DAG: s_movk_i32 [[K:s[0-9]+]], 0x7fff{{$}} +; GFX89-DAG: v_mov_b32_e32 [[BFI_K:v[0-9]+]], 0x3c00 +; GFX89: v_bfi_b32 [[COPYSIGN0:v[0-9]+]], [[K]], [[BFI_K]], +; GFX89: v_bfi_b32 [[COPYSIGN1:v[0-9]+]], [[K]], [[BFI_K]], + +; GFX9: v_pack_b32_f16 +define void @round_v2f16(<2 x half> addrspace(1)* %out, i32 %in.arg) #0 { + %in = bitcast i32 %in.arg to <2 x half> + %result = call <2 x half> @llvm.round.v2f16(<2 x half> %in) + store <2 x half> %result, <2 x half> addrspace(1)* %out + ret void +} + declare float @llvm.round.f32(float) #1 declare <2 x float> @llvm.round.v2f32(<2 x float>) #1 declare <4 x float> @llvm.round.v4f32(<4 x float>) #1 declare <8 x float> @llvm.round.v8f32(<8 x float>) #1 +declare half @llvm.round.f16(half) #1 +declare <2 x half> @llvm.round.v2f16(<2 x half>) #1 +declare <4 x half> @llvm.round.v4f16(<4 x half>) #1 +declare <8 x half> @llvm.round.v8f16(<8 x half>) #1 + attributes #0 = { nounwind } attributes #1 = { nounwind readnone }