AMDGPU: Simplify f16 to i64 custom lowering

[Petar.Avramovic]

Summary

Range that f16 can represent fits into i32.
Lower as f16->i32->i64 instead of f16->f32->i64
since f32->i64 has long expansion.

[arsenm]

llvm/test/CodeGen/AMDGPU/fptosi.f16.ll

108

Why are there test changes only for the vector cases?

[Petar.Avramovic]

Context was missing.

llvm/test/CodeGen/AMDGPU/fptosi.f16.ll

108

Tests only check for a few instructions and here there was no v_cvt_f32_f16_e32.
I think that they test if there was a f16 -> f32 followed by long f32 -> i64 expansion but don't check expansion.
Now f16->i32 is selected as f16->f32->i32 and f16->f32 part is still there.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

2726–2727

I don't think this needs to be limited to Subtarget->has16BitInsts()

llvm/test/CodeGen/AMDGPU/fptosi.f16.ll

45

These checks should be strengthened to show the new expansion

[Petar.Avramovic]

This update also simplifies f16 -> i64 on subtargets without has16BitInsts().
Such targets immediately promote f16 to f32 whenever they encounter f16 def using fp16_to_fp node. Recognize fp16_to_fp input to f32 to i64 and do the same thing as for f16 to i64 conversion. This gives similar(small difference for f16 vectors) results for subtargets with or without has16BitInsts() since f16->i32 gets selected like f16->f32->i32.
Update tests with more detailed checks.

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

 
 SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDValue Src = Op.getOperand(0);
 
   // TODO: Factor out code common with LowerFP_TO_UINT.
 
   EVT SrcVT = Src.getValueType();
-  if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
+  if (SrcVT == MVT::f16 ||
+      (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
     SDLoc DL(Op);
 
-    SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
-    SDValue FpToInt32 =
-        DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
-
-    return FpToInt32;
+    SDValue FpToInt32 = DAG.getNode(Op.getOpcode(), DL, MVT::i32, Src);
+    return DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, FpToInt32);
   }
 
   if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
     return LowerFP64_TO_INT(Op, DAG, true);
 
   return SDValue();
 }
 
 SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDValue Src = Op.getOperand(0);
 
   // TODO: Factor out code common with LowerFP_TO_SINT.
 
   EVT SrcVT = Src.getValueType();
-  if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
+  if (SrcVT == MVT::f16 ||
+      (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
     SDLoc DL(Op);
 
-    SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
-    SDValue FpToInt32 =
-        DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
-
-    return FpToInt32;
+    SDValue FpToUInt32 = DAG.getNode(Op.getOpcode(), DL, MVT::i32, Src);
+    return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, FpToUInt32);
   }
 
   if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
     return LowerFP64_TO_INT(Op, DAG, false);
 
   return SDValue();
 }
 

llvm/test/CodeGen/AMDGPU/fptosi.f16.ll

entry:
   store i32 %r.val, i32 addrspace(1)* %r
   ret void
 }
 
 ; Need to make sure we promote f16 to f32 when converting f16 to i64. Existing
 ; test checks code generated for 'i64 = fp_to_sint f32'.
 
 ; GCN-LABEL: {{^}}fptosi_f16_to_i64
-; GCN: buffer_load_ushort
-; GCN: v_cvt_f32_f16_e32
+; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
+; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
+; GCN: v_cvt_i32_f32_e32 v[[R_I64_Low:[0-9]+]], v[[A_F32]]
+; GCN: v_ashrrev_i32_e32 v[[R_I64_High:[0-9]+]], 31, v[[R_I64_Low]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[R_I64_Low]]{{\:}}[[R_I64_High]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @fptosi_f16_to_i64(
     i64 addrspace(1)* %r,
     half addrspace(1)* %a) {
 entry:
   %a.val = load half, half addrspace(1)* %a
   %r.val = fptosi half %a.val to i64
   store i64 %r.val, i64 addrspace(1)* %r
   ret void
entry:
   store <2 x i32> %r.val, <2 x i32> addrspace(1)* %r
   ret void
 }
 
 ; Need to make sure we promote f16 to f32 when converting f16 to i64. Existing
 ; test checks code generated for 'i64 = fp_to_sint f32'.
 
 ; GCN-LABEL: {{^}}fptosi_v2f16_to_v2i64
-; GCN: buffer_load_dword
-; GCN: v_cvt_f32_f16_e32
-; SI: v_cvt_f32_f16_e32
-; VI: v_cvt_f32_f16_sdwa
+; GCN: buffer_load_dword v[[A_F16_0:[0-9]+]]
+; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_F16_0]]
+; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
+; SI: v_cvt_i32_f32_e32 v[[R_I64_0_Low:[0-9]+]], v[[A_F32_0]]
+; SI: v_ashrrev_i32_e32 v[[R_I64_0_High:[0-9]+]], 31, v[[R_I64_0_Low]]
+; SI: v_cvt_i32_f32_e32 v[[R_I64_1_Low:[0-9]+]], v[[A_F32_1]]
+; SI: v_ashrrev_i32_e32 v[[R_I64_1_High:[0-9]+]], 31, v[[R_I64_1_Low]]
+; VI: v_cvt_f32_f16_sdwa v[[A_F32_1:[0-9]+]], v[[A_F16_0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; VI: v_cvt_i32_f32_e32 v[[R_I64_1_Low:[0-9]+]], v[[A_F32_1]]
+; VI: v_cvt_i32_f32_e32 v[[R_I64_0_Low:[0-9]+]], v[[A_F32_0]]
+; VI: v_ashrrev_i32_e32 v[[R_I64_1_High:[0-9]+]], 31, v[[R_I64_1_Low]]
+; VI: v_ashrrev_i32_e32 v[[R_I64_0_High:[0-9]+]], 31, v[[R_I64_0_Low]]
+; GCN: buffer_store_dwordx4 v{{\[}}[[R_I64_0_Low]]{{\:}}[[R_I64_1_High]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @fptosi_v2f16_to_v2i64(
     <2 x i64> addrspace(1)* %r,
     <2 x half> addrspace(1)* %a) {
 entry:
   %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
   %r.val = fptosi <2 x half> %a.val to <2 x i64>
   store <2 x i64> %r.val, <2 x i64> addrspace(1)* %r
   ret void
 }

llvm/test/CodeGen/AMDGPU/fptoui.f16.ll

entry:
   store i32 %r.val, i32 addrspace(1)* %r
   ret void
 }
 
 ; Need to make sure we promote f16 to f32 when converting f16 to i64. Existing
 ; test checks code generated for 'i64 = fp_to_uint f32'.
 
 ; GCN-LABEL: {{^}}fptoui_f16_to_i64
-; GCN: buffer_load_ushort
-; GCN: v_cvt_f32_f16_e32
+; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
+; GCN: v_mov_b32_e32 v[[R_I64_High:[0-9]+]], 0
+; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
+; GCN: v_cvt_u32_f32_e32 v[[R_I64_Low:[0-9]+]], v[[A_F32]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[R_I64_Low]]{{\:}}[[R_I64_High]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @fptoui_f16_to_i64(
     i64 addrspace(1)* %r,
     half addrspace(1)* %a) {
 entry:
   %a.val = load half, half addrspace(1)* %a
   %r.val = fptoui half %a.val to i64
   store i64 %r.val, i64 addrspace(1)* %r
entry:
   store <2 x i32> %r.val, <2 x i32> addrspace(1)* %r
   ret void
 }
 
 ; Need to make sure we promote f16 to f32 when converting f16 to i64. Existing
 ; test checks code generated for 'i64 = fp_to_uint f32'.
 
 ; GCN-LABEL: {{^}}fptoui_v2f16_to_v2i64
-; GCN: buffer_load_dword
-; GCN: v_cvt_f32_f16_e32
-; SI: v_cvt_f32_f16_e32
-; VI: v_cvt_f32_f16_sdwa
+; GCN: buffer_load_dword v[[A_F16_0:[0-9]+]]
+; GCN: v_mov_b32_e32 v[[R_I64_1_High:[0-9]+]], 0
+; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_F16_0]]
+; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
+; SI: v_cvt_u32_f32_e32 v[[R_I64_0_Low:[0-9]+]], v[[A_F32_0]]
+; SI: v_cvt_u32_f32_e32 v[[R_I64_1_Low:[0-9]+]], v[[A_F32_1]]
+; VI: v_cvt_f32_f16_sdwa v[[A_F32_1:[0-9]+]], v[[A_F16_0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; VI: v_cvt_u32_f32_e32 v[[R_I64_1_Low:[0-9]+]], v[[A_F32_1]]
+; VI: v_cvt_u32_f32_e32 v[[R_I64_0_Low:[0-9]+]], v[[A_F32_0]]
+; GCN: v_mov_b32_e32 v[[R_I64_0_High:[0-9]+]], 0
+; GCN: buffer_store_dwordx4 v{{\[}}[[R_I64_0_Low]]{{\:}}[[R_I64_1_High]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @fptoui_v2f16_to_v2i64(
     <2 x i64> addrspace(1)* %r,
     <2 x half> addrspace(1)* %a) {
 entry:
   %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
   %r.val = fptoui <2 x half> %a.val to <2 x i64>
   store <2 x i64> %r.val, <2 x i64> addrspace(1)* %r
   ret void
 }