AMDGPU: Fix V_FMA_F16 selection on GFX9

[kzhuravl]

Summary

GFX9 should select opsel version.

[rampitec]

LGTM

[arsenm]

Is this the one that zeros the high bits vs. leaves them intact?

[kzhuravl]

In D54545#1299055, @arsenm wrote:

Is this the one that zeros the high bits vs. leaves them intact?

Did you mean?

If op_sel[3] is 0 Result is written to 16 LSBs of destination VGPR and hi 16 bits are preserved.
If op_sel[3] is 1 Result is written to 16 MSBs of destination VGPR and lo 16 bits are preserved.

llvm/trunk/lib/Target/AMDGPU/VOP3Instructions.td

def V_DIV_FIXUP_F16 : VOP3Inst <"v_div_fixup_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, AMDGPUdiv_fixup> {
   let Predicates = [Has16BitInsts, isVIOnly];
 }
 def V_DIV_FIXUP_F16_gfx9 : VOP3Inst <"v_div_fixup_f16_gfx9",
                                       VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, AMDGPUdiv_fixup> {
   let renamedInGFX9 = 1;
   let Predicates = [Has16BitInsts, isGFX9];
 }
 
+def V_FMA_F16 : VOP3Inst <"v_fma_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, fma> {
+  let Predicates = [Has16BitInsts, isVIOnly];
+}
+def V_FMA_F16_gfx9 : VOP3Inst <"v_fma_f16_gfx9", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, fma> {
+  let renamedInGFX9 = 1;
+  let Predicates = [Has16BitInsts, isGFX9];
+}
+
 let SubtargetPredicate = Has16BitInsts, isCommutable = 1 in {
 
 let renamedInGFX9 = 1 in {
 def V_MAD_F16 : VOP3Inst <"v_mad_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, fmad>;
 def V_MAD_U16 : VOP3Inst <"v_mad_u16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CLAMP>>;
 def V_MAD_I16 : VOP3Inst <"v_mad_i16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CLAMP>>;
-def V_FMA_F16 : VOP3Inst <"v_fma_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, fma>;
 def V_INTERP_P2_F16 : VOP3Interp <"v_interp_p2_f16", VOP3_INTERP16<[f16, f32, i32, f32]>>;
 }
 
 let SubtargetPredicate = isGFX9 in {
 def V_MAD_F16_gfx9   : VOP3Inst <"v_mad_f16_gfx9", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
 def V_MAD_U16_gfx9   : VOP3Inst <"v_mad_u16_gfx9", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_OPSEL>>;
 def V_MAD_I16_gfx9   : VOP3Inst <"v_mad_i16_gfx9", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_OPSEL>>;
-def V_FMA_F16_gfx9   : VOP3Inst <"v_fma_f16_gfx9", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
 def V_INTERP_P2_F16_gfx9 : VOP3Interp <"v_interp_p2_f16_gfx9", VOP3_INTERP16<[f16, f32, i32, f32]>>;
 } // End SubtargetPredicate = isGFX9
 
 def V_INTERP_P1LL_F16 : VOP3Interp <"v_interp_p1ll_f16", VOP3_INTERP16<[f32, f32, i32, untyped]>>;
 def V_INTERP_P1LV_F16 : VOP3Interp <"v_interp_p1lv_f16", VOP3_INTERP16<[f32, f32, i32, f16]>>;
 
 } // End SubtargetPredicate = Has16BitInsts, isCommutable = 1
 

llvm/trunk/test/CodeGen/AMDGPU/fdot2.ll

 ; RUN: llc -march=amdgcn -mcpu=gfx900 -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX900
 ; RUN: llc -march=amdgcn -mcpu=gfx906 -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-UNSAFE
 ; RUN: llc -march=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906
 ; RUN: llc -march=amdgcn -mcpu=gfx906 -mattr=-fp64-fp16-denormals,-fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-CONTRACT
 ; RUN: llc -march=amdgcn -mcpu=gfx906 -mattr=+fp64-fp16-denormals,+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-DENORM-CONTRACT
 ; (fadd (fmul S1.x, S2.x), (fadd (fmul (S1.y, S2.y), z))) -> (fdot2 S1, S2, z)
 
 ; Tests to make sure fdot2 is not generated when vector elements of dot-product expressions
 ; are not converted from f16 to f32.
 ; GCN-LABEL: {{^}}dotproduct_f16
-; GFX900: v_fma_legacy_f16
-; GFX900: v_fma_legacy_f16
+; GFX900: v_fma_f16
+; GFX900: v_fma_f16
 
 ; GFX906: v_mul_f16_e32
 ; GFX906: v_mul_f16_e32
 
-; GFX906-UNSAFE:  v_fma_legacy_f16
+; GFX906-UNSAFE:  v_fma_f16
 
 ; GFX906-CONTRACT: v_mac_f16_e32
-; GFX906-DENORM-CONTRACT: v_fma_legacy_f16
+; GFX906-DENORM-CONTRACT: v_fma_f16
 define amdgpu_kernel void @dotproduct_f16(<2 x half> addrspace(1)* %src1,
                                           <2 x half> addrspace(1)* %src2,
                                           half addrspace(1)* nocapture %dst) {
 entry:
   %src1.vec = load <2 x half>, <2 x half> addrspace(1)* %src1
   %src2.vec = load <2 x half>, <2 x half> addrspace(1)* %src2
 
   %src1.el1 = extractelement <2 x half> %src1.vec, i64 0

llvm/trunk/test/CodeGen/AMDGPU/fpext-free.ll

entry:
   ret float %add
 }
 
 ; fold (fadd x, (fpext (fma y, z, (fmul u, v)))
 ;   -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x))
 
 ; GCN-LABEL: {{^}}fadd_fpext_fmuladd_f16_to_f32:
 ; GFX9: v_mul_f16
-; GFX9: v_fma_legacy_f16
+; GFX9: v_fma_f16
 ; GFX9: v_cvt_f32_f16
 ; GFX9: v_add_f32_e32
 define float @fadd_fpext_fmuladd_f16_to_f32(float %x, half %y, half %z, half %u, half %v) #0 {
 entry:
   %mul = fmul contract half %u, %v
   %fma = call half @llvm.fmuladd.f16(half %y, half %z, half %mul)
   %ext.fma = fpext half %fma to float
   %add = fadd float %x, %ext.fma
   ret float %add
 }
 
 ; GCN-LABEL: {{^}}fadd_fpext_fma_f16_to_f32:
 ; GFX9: v_mul_f16
-; GFX9: v_fma_legacy_f16
+; GFX9: v_fma_f16
 ; GFX9: v_cvt_f32_f16
 ; GFX9: v_add_f32_e32
 define float @fadd_fpext_fma_f16_to_f32(float %x, half %y, half %z, half %u, half %v) #0 {
 entry:
   %mul = fmul contract half %u, %v
   %fma = call half @llvm.fma.f16(half %y, half %z, half %mul)
   %ext.fma = fpext half %fma to float
   %add = fadd float %x, %ext.fma
   ret float %add
 }
 
 ; GCN-LABEL: {{^}}fadd_fpext_fma_f16_to_f32_commute:
 ; GFX9: v_mul_f16
-; GFX9: v_fma_legacy_f16
+; GFX9: v_fma_f16
 ; GFX9: v_cvt_f32_f16
 ; GFX9: v_add_f32_e32
 define float @fadd_fpext_fma_f16_to_f32_commute(float %x, half %y, half %z, half %u, half %v) #0 {
 entry:
   %mul = fmul contract half %u, %v
   %fma = call half @llvm.fma.f16(half %y, half %z, half %mul)
   %ext.fma = fpext half %fma to float
   %add = fadd float %ext.fma, %x

 }
 
 ;  fold (fsub (fpext (fmad x, y, (fmul u, v))), z)
 ;    -> (fmad (fpext x), (fpext y),
 ;            (fmad (fpext u), (fpext v), (fneg z)))
 
 ; GCN-LABEL: {{^}}fsub_fpext_muladd_mul_f16_to_f32:
 ; GFX9: v_mul_f16
-; GFX9: v_fma_legacy_f16
+; GFX9: v_fma_f16
 ; GFX9: v_cvt_f32_f16
 ; GFX9: v_sub_f32
 ; GCN: s_setpc_b64
 define float @fsub_fpext_muladd_mul_f16_to_f32(half %x, half %y, float %z, half %u, half %v) #0 {
 entry:
   %mul = fmul half %u, %v
   %fma = call half @llvm.fmuladd.f16(half %x, half %y, half %mul)
   %fma.ext = fpext half %fma to float

 }
 
 ; fold (fsub x, (fpext (fma y, z, (fmul u, v))))
 ;    -> (fma (fneg (fpext y)), (fpext z),
 ;            (fma (fneg (fpext u)), (fpext v), x))
 ; GCN-LABEL: {{^}}fsub_fpext_muladd_mul_f16_to_f32_commute:
 ; GCN: s_waitcnt
 ; GFX9-NEXT: v_mul_f16_e32 v3, v3, v4
-; GFX9-NEXT: v_fma_legacy_f16 v1, v1, v2, v3
+; GFX9-NEXT: v_fma_f16 v1, v1, v2, v3
 ; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
 ; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
 ; GFX9-NEXT: s_setpc_b64
 define float @fsub_fpext_muladd_mul_f16_to_f32_commute(float %x, half %y, half %z, half %u, half %v) #0 {
 entry:
   %mul = fmul half %u, %v
   %fma = call half @llvm.fmuladd.f16(half %y, half %z, half %mul)
   %fma.ext = fpext half %fma to float