[AMDGPU] Implement f16 fcopysign and fcopysign(f32, f64)

[arsenm]

lib/Target/AMDGPU/SIISelLowering.cpp

313–314 ↗

(On Diff #83722)

This shouldn't be necessary. This will only be false for R600 targets and the default is expand for illegal types

lib/Target/AMDGPU/SIInstructions.td

680–683 ↗

(On Diff #83722)

It is possible to have mismatched FP types for src0 and src1. If you can come up with a testcases combined with the fp casts you might need that too

test/CodeGen/AMDGPU/fcopysign.f16.ll

1 ↗

(On Diff #83722)

No -mcpu=SI

[kzhuravl]

Address review feedback

[arsenm]

lib/Target/AMDGPU/SIInstructions.td

688 ↗

(On Diff #84074)

This should use e64

696 ↗

(On Diff #84074)

e64

708 ↗

(On Diff #84074)

e64

test/CodeGen/AMDGPU/fcopysign.f16.ll

1 ↗

(On Diff #84074)

This should also have an SI run line. The -enable-unsafe-fp-math should also be dropped

[kzhuravl]

test/CodeGen/AMDGPU/fcopysign.f16.ll

1 ↗

(On Diff #84074)

Should check lines also be added for SI? Or run line is sufficient?

[kzhuravl]

test/CodeGen/AMDGPU/fcopysign.f16.ll

1 ↗

(On Diff #84074)

Nevermind. I got confused by the original comment.

[kzhuravl]

Address review feedback + implement fcopysign(f32, f64)

[arsenm]

LGTM

llvm/trunk/lib/Target/AMDGPU/AMDGPUInstructions.td

multiclass BFIPatterns <Instruction BFI_INT,
   >;
 
   def : Pat <
     (fcopysign f32:$src0, f32:$src1),
     (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0, $src1)
   >;
 
   def : Pat <
+    (f32 (fcopysign f32:$src0, f64:$src1)),
+    (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0,
+             (i32 (EXTRACT_SUBREG $src1, sub1)))
+  >;
+
+  def : Pat <
     (f64 (fcopysign f64:$src0, f64:$src1)),
     (REG_SEQUENCE RC64,
       (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
       (BFI_INT (LoadImm32 (i32 0x7fffffff)),
                (i32 (EXTRACT_SUBREG $src0, sub1)),
                (i32 (EXTRACT_SUBREG $src1, sub1))), sub1)
   >;
 

llvm/trunk/lib/Target/AMDGPU/SIInstructions.td

(REG_SEQUENCE VReg_64,
     (i32 (EXTRACT_SUBREG f64:$src, sub0)),
     sub0,
     (V_XOR_B32_e32 (i32 (EXTRACT_SUBREG f64:$src, sub1)),
                    (i32 (V_MOV_B32_e32 (i32 0x80000000)))),
     sub1)
 >;
 
 def : Pat <
+  (fcopysign f16:$src0, f16:$src1),
+  (V_BFI_B32 (S_MOV_B32 (i32 0x00007fff)), $src0, $src1)
+>;
+
+def : Pat <
+  (fcopysign f32:$src0, f16:$src1),
+  (V_BFI_B32 (S_MOV_B32 (i32 0x7fffffff)), $src0,
+             (V_LSHLREV_B32_e64 (i32 16), $src1))
+>;
+
+def : Pat <
+  (fcopysign f64:$src0, f16:$src1),
+  (REG_SEQUENCE SReg_64,
+    (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
+    (V_BFI_B32 (S_MOV_B32 (i32 0x7fffffff)), (i32 (EXTRACT_SUBREG $src0, sub1)),
+               (V_LSHLREV_B32_e64 (i32 16), $src1)), sub1)
+>;
+
+def : Pat <
+  (fcopysign f16:$src0, f32:$src1),
+  (V_BFI_B32 (S_MOV_B32 (i32 0x00007fff)), $src0,
+             (V_LSHRREV_B32_e64 (i32 16), $src1))
+>;
+
+def : Pat <
+  (fcopysign f16:$src0, f64:$src1),
+  (V_BFI_B32 (S_MOV_B32 (i32 0x00007fff)), $src0,
+             (V_LSHRREV_B32_e64 (i32 16), (EXTRACT_SUBREG $src1, sub1)))
+>;
+
+def : Pat <
   (fneg f16:$src),
   (V_XOR_B32_e32 $src, (V_MOV_B32_e32 (i32 0x00008000)))
 >;
 
 def : Pat <
   (fabs f16:$src),
   (V_AND_B32_e64 $src, (V_MOV_B32_e32 (i32 0x00007fff)))
 >;

llvm/trunk/test/CodeGen/AMDGPU/fcopysign.f16.ll

+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=GCN -check-prefix=VI %s
+
+declare half @llvm.copysign.f16(half, half)
+declare float @llvm.copysign.f32(float, float)
+declare double @llvm.copysign.f64(double, double)
+declare <2 x half> @llvm.copysign.v2f16(<2 x half>, <2 x half>)
+declare <3 x half> @llvm.copysign.v3f16(<3 x half>, <3 x half>)
+declare <4 x half> @llvm.copysign.v4f16(<4 x half>, <4 x half>)
+
+; FUNC-LABEL: {{^}}test_copysign_f16:
+; SI: buffer_load_ushort v[[MAG:[0-9]+]]
+; SI: buffer_load_ushort v[[SIGN:[0-9]+]]
+; SI: s_brev_b32 s[[CONST:[0-9]+]], -2
+; SI: v_cvt_f32_f16_e32 v[[MAG_F32:[0-9]+]], v[[MAG]]
+; SI: v_cvt_f32_f16_e32 v[[SIGN_F32:[0-9]+]], v[[SIGN]]
+; SI: v_bfi_b32 v[[OUT_F32:[0-9]+]], s[[CONST]], v[[MAG_F32]], v[[SIGN_F32]]
+; SI: v_cvt_f16_f32_e32 v[[OUT:[0-9]+]], v[[OUT_F32]]
+; VI: buffer_load_ushort v[[SIGN:[0-9]+]]
+; VI: buffer_load_ushort v[[MAG:[0-9]+]]
+; VI: s_movk_i32 s[[CONST:[0-9]+]], 0x7fff
+; VI: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG]], v[[SIGN]]
+; GCN: buffer_store_short v[[OUT]]
+; GCN: s_endpgm
+define void @test_copysign_f16(
+  half addrspace(1)* %arg_out,
+  half addrspace(1)* %arg_mag,
+  half addrspace(1)* %arg_sign) {
+entry:
+  %mag = load half, half addrspace(1)* %arg_mag
+  %sign = load half, half addrspace(1)* %arg_sign
+  %out = call half @llvm.copysign.f16(half %mag, half %sign)
+  store half %out, half addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_out_f32_mag_f16_sign_f32:
+; GCN-DAG: buffer_load_ushort v[[MAG:[0-9]+]]
+; GCN-DAG: buffer_load_dword v[[SIGN:[0-9]+]]
+; GCN-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
+; GCN-DAG: v_cvt_f32_f16_e32 v[[MAG_EXT:[0-9]+]], v[[MAG]]
+; GCN: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG_EXT]], v[[SIGN]]
+; GCN: buffer_store_dword v[[OUT]]
+; GCN: s_endpgm
+define void @test_copysign_out_f32_mag_f16_sign_f32(
+  float addrspace(1)* %arg_out,
+  half addrspace(1)* %arg_mag,
+  float addrspace(1)* %arg_sign) {
+entry:
+  %mag = load half, half addrspace(1)* %arg_mag
+  %mag.ext = fpext half %mag to float
+  %sign = load float, float addrspace(1)* %arg_sign
+  %out = call float @llvm.copysign.f32(float %mag.ext, float %sign)
+  store float %out, float addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_out_f64_mag_f16_sign_f64:
+; GCN-DAG: buffer_load_ushort v[[MAG:[0-9]+]]
+; GCN-DAG: buffer_load_dwordx2 v{{\[}}[[SIGN_LO:[0-9]+]]:[[SIGN_HI:[0-9]+]]{{\]}}
+; GCN-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
+; GCN-DAG: v_cvt_f32_f16_e32 v[[MAG_EXT:[0-9]+]], v[[MAG]]
+; GCN-DAG: v_cvt_f64_f32_e32 v{{\[}}[[MAG_EXT_LO:[0-9]+]]:[[MAG_EXT_HI:[0-9]+]]{{\]}}, v[[MAG_EXT]]
+; GCN: v_bfi_b32 v[[OUT_HI:[0-9]+]], s[[CONST]], v[[MAG_EXT_HI]], v[[SIGN_HI]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[MAG_EXT_LO]]:[[OUT_HI]]{{\]}}
+; GCN: s_endpgm
+define void @test_copysign_out_f64_mag_f16_sign_f64(
+  double addrspace(1)* %arg_out,
+  half addrspace(1)* %arg_mag,
+  double addrspace(1)* %arg_sign) {
+entry:
+  %mag = load half, half addrspace(1)* %arg_mag
+  %mag.ext = fpext half %mag to double
+  %sign = load double, double addrspace(1)* %arg_sign
+  %out = call double @llvm.copysign.f64(double %mag.ext, double %sign)
+  store double %out, double addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_out_f32_mag_f32_sign_f16:
+; GCN-DAG: buffer_load_dword v[[MAG:[0-9]+]]
+; GCN-DAG: buffer_load_ushort v[[SIGN:[0-9]+]]
+; GCN-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
+; SI-DAG: v_cvt_f32_f16_e32 v[[SIGN_F32:[0-9]+]], v[[SIGN]]
+; SI: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG]], v[[SIGN_F32]]
+; VI-DAG: v_lshlrev_b32_e32 v[[SIGN_SHIFT:[0-9]+]], 16, v[[SIGN]]
+; VI: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG]], v[[SIGN_SHIFT]]
+; GCN: buffer_store_dword v[[OUT]]
+; GCN: s_endpgm
+define void @test_copysign_out_f32_mag_f32_sign_f16(
+  float addrspace(1)* %arg_out,
+  float addrspace(1)* %arg_mag,
+  half addrspace(1)* %arg_sign) {
+entry:
+  %mag = load float, float addrspace(1)* %arg_mag
+  %sign = load half, half addrspace(1)* %arg_sign
+  %sign.ext = fpext half %sign to float
+  %out = call float @llvm.copysign.f32(float %mag, float %sign.ext)
+  store float %out, float addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_out_f64_mag_f64_sign_f16:
+; GCN-DAG: buffer_load_dwordx2 v{{\[}}[[MAG_LO:[0-9]+]]:[[MAG_HI:[0-9]+]]{{\]}}
+; GCN-DAG: buffer_load_ushort v[[SIGN:[0-9]+]]
+; GCN-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
+; SI-DAG: v_cvt_f32_f16_e32 v[[SIGN_F32:[0-9]+]], v[[SIGN]]
+; SI: v_bfi_b32 v[[OUT_HI:[0-9]+]], s[[CONST]], v[[MAG_HI]], v[[SIGN_F32]]
+; VI-DAG: v_lshlrev_b32_e32 v[[SIGN_SHIFT:[0-9]+]], 16, v[[SIGN]]
+; VI: v_bfi_b32 v[[OUT_HI:[0-9]+]], s[[CONST]], v[[MAG_HI]], v[[SIGN_SHIFT]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[MAG_LO]]:[[OUT_HI]]{{\]}}
+; GCN: s_endpgm
+define void @test_copysign_out_f64_mag_f64_sign_f16(
+  double addrspace(1)* %arg_out,
+  double addrspace(1)* %arg_mag,
+  half addrspace(1)* %arg_sign) {
+entry:
+  %mag = load double, double addrspace(1)* %arg_mag
+  %sign = load half, half addrspace(1)* %arg_sign
+  %sign.ext = fpext half %sign to double
+  %out = call double @llvm.copysign.f64(double %mag, double %sign.ext)
+  store double %out, double addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_out_f16_mag_f16_sign_f32:
+; GCN-DAG: buffer_load_ushort v[[MAG:[0-9]+]]
+; GCN-DAG: buffer_load_dword v[[SIGN:[0-9]+]]
+; SI-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
+; SI-DAG: v_cvt_f32_f16_e32 v[[MAG_F32:[0-9]+]], v[[MAG]]
+; SI: v_bfi_b32 v[[OUT_F32:[0-9]+]], s[[CONST]], v[[MAG_F32]], v[[SIGN]]
+; SI: v_cvt_f16_f32_e32 v[[OUT:[0-9]+]], v[[OUT_F32]]
+; VI-DAG: s_movk_i32 s[[CONST:[0-9]+]], 0x7fff
+; VI-DAG: v_lshrrev_b32_e32 v[[SIGN_SHIFT:[0-9]+]], 16, v[[SIGN]]
+; VI: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG]], v[[SIGN_SHIFT]]
+; GCN: buffer_store_short v[[OUT]]
+; GCN: s_endpgm
+define void @test_copysign_out_f16_mag_f16_sign_f32(
+  half addrspace(1)* %arg_out,
+  half addrspace(1)* %arg_mag,
+  float addrspace(1)* %arg_sign) {
+entry:
+  %mag = load half, half addrspace(1)* %arg_mag
+  %sign = load float, float addrspace(1)* %arg_sign
+  %sign.trunc = fptrunc float %sign to half
+  %out = call half @llvm.copysign.f16(half %mag, half %sign.trunc)
+  store half %out, half addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_out_f16_mag_f16_sign_f64:
+; GCN-DAG: buffer_load_ushort v[[MAG:[0-9]+]]
+; GCN-DAG: buffer_load_dwordx2 v{{\[}}[[SIGN_LO:[0-9]+]]:[[SIGN_HI:[0-9]+]]{{\]}}
+; SI-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
+; SI-DAG: v_cvt_f32_f16_e32 v[[MAG_F32:[0-9]+]], v[[MAG]]
+; SI: v_bfi_b32 v[[OUT_F32:[0-9]+]], s[[CONST]], v[[MAG_F32]], v[[SIGN_HI]]
+; SI: v_cvt_f16_f32_e32 v[[OUT:[0-9]+]], v[[OUT_F32]]
+; VI-DAG: s_movk_i32 s[[CONST:[0-9]+]], 0x7fff
+; VI-DAG: v_lshrrev_b32_e32 v[[SIGN_SHIFT:[0-9]+]], 16, v[[SIGN_HI]]
+; VI: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG]], v[[SIGN_SHIFT]]
+; GCN: buffer_store_short v[[OUT]]
+; GCN: s_endpgm
+define void @test_copysign_out_f16_mag_f16_sign_f64(
+  half addrspace(1)* %arg_out,
+  half addrspace(1)* %arg_mag,
+  double addrspace(1)* %arg_sign) {
+entry:
+  %mag = load half, half addrspace(1)* %arg_mag
+  %sign = load double, double addrspace(1)* %arg_sign
+  %sign.trunc = fptrunc double %sign to half
+  %out = call half @llvm.copysign.f16(half %mag, half %sign.trunc)
+  store half %out, half addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_out_f16_mag_f32_sign_f16:
+; GCN-DAG: buffer_load_dword v[[MAG:[0-9]+]]
+; GCN-DAG: buffer_load_ushort v[[SIGN:[0-9]+]]
+; SI-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
+; SI-DAG: v_cvt_f16_f32_e32 v[[MAG_TRUNC:[0-9]+]], v[[MAG]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[SIGN_F32:[0-9]+]], v[[SIGN]]
+; SI-DAG: v_cvt_f32_f16_e32 v[[MAG_F32:[0-9]+]], v[[MAG_TRUNC]]
+; SI: v_bfi_b32 v[[OUT_F32:[0-9]+]], s[[CONST]], v[[MAG_F32]], v[[SIGN_F32]]
+; SI: v_cvt_f16_f32_e32 v[[OUT:[0-9]+]], v[[OUT_F32]]
+; VI-DAG: s_movk_i32 s[[CONST:[0-9]+]], 0x7fff
+; VI-DAG: v_cvt_f16_f32_e32 v[[MAG_TRUNC:[0-9]+]], v[[MAG]]
+; VI: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG_TRUNC]], v[[SIGN]]
+; GCN: buffer_store_short v[[OUT]]
+; GCN: s_endpgm
+define void @test_copysign_out_f16_mag_f32_sign_f16(
+  half addrspace(1)* %arg_out,
+  float addrspace(1)* %arg_mag,
+  half addrspace(1)* %arg_sign) {
+entry:
+  %mag = load float, float addrspace(1)* %arg_mag
+  %mag.trunc = fptrunc float %mag to half
+  %sign = load half, half addrspace(1)* %arg_sign
+  %out = call half @llvm.copysign.f16(half %mag.trunc, half %sign)
+  store half %out, half addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_out_f16_mag_f64_sign_f16:
+; GCN: v_bfi_b32
+; GCN: s_endpgm
+define void @test_copysign_out_f16_mag_f64_sign_f16(
+  half addrspace(1)* %arg_out,
+  double addrspace(1)* %arg_mag,
+  half addrspace(1)* %arg_sign) {
+entry:
+  %mag = load double, double addrspace(1)* %arg_mag
+  %mag.trunc = fptrunc double %mag to half
+  %sign = load half, half addrspace(1)* %arg_sign
+  %out = call half @llvm.copysign.f16(half %mag.trunc, half %sign)
+  store half %out, half addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_v2f16:
+; GCN: v_bfi_b32
+; GCN: v_bfi_b32
+; GCN: s_endpgm
+define void @test_copysign_v2f16(
+  <2 x half> addrspace(1)* %arg_out,
+  <2 x half> %arg_mag,
+  <2 x half> %arg_sign) {
+entry:
+  %out = call <2 x half> @llvm.copysign.v2f16(<2 x half> %arg_mag, <2 x half> %arg_sign)
+  store <2 x half> %out, <2 x half> addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_v3f16:
+; GCN: v_bfi_b32
+; GCN: v_bfi_b32
+; GCN: v_bfi_b32
+; GCN: s_endpgm
+define void @test_copysign_v3f16(
+  <3 x half> addrspace(1)* %arg_out,
+  <3 x half> %arg_mag,
+  <3 x half> %arg_sign) {
+entry:
+  %out = call <3 x half> @llvm.copysign.v3f16(<3 x half> %arg_mag, <3 x half> %arg_sign)
+  store <3 x half> %out, <3 x half> addrspace(1)* %arg_out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_copysign_v4f16:
+; GCN: v_bfi_b32
+; GCN: v_bfi_b32
+; GCN: v_bfi_b32
+; GCN: v_bfi_b32
+; GCN: s_endpgm
+define void @test_copysign_v4f16(
+  <4 x half> addrspace(1)* %arg_out,
+  <4 x half> %arg_mag,
+  <4 x half> %arg_sign) {
+entry:
+  %out = call <4 x half> @llvm.copysign.v4f16(<4 x half> %arg_mag, <4 x half> %arg_sign)
+  store <4 x half> %out, <4 x half> addrspace(1)* %arg_out
+  ret void
+}