Index: llvm/lib/Target/AMDGPU/VOP3PInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/VOP3PInstructions.td +++ llvm/lib/Target/AMDGPU/VOP3PInstructions.td @@ -228,7 +228,7 @@ >; } -let SubtargetPredicate = HasMadMixInsts in { +let SubtargetPredicate = HasMadMixInsts, OtherPredicates = [NoFP32Denormals] in { // These are VOP3a-like opcodes which accept no omod. // Size of src arguments (16/32) is controlled by op_sel. @@ -252,7 +252,7 @@ // Essentially the same as the mad_mix versions -let SubtargetPredicate = HasFmaMixInsts in { +let SubtargetPredicate = HasFmaMixInsts, OtherPredicates = [NoFP32Denormals] in { let isCommutable = 1 in { let isReMaterializable = 1 in Index: llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-fma.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-fma.ll +++ llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-fma.ll @@ -16,15 +16,17 @@ ; GFX10-LABEL: test_f16_f32_add_fma_ext_mul: ; GFX10: ; %bb.0: ; %.entry ; GFX10-NEXT: v_mul_f16_e32 v3, v3, v4 -; GFX10-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1] -; GFX10-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v3 +; GFX10-NEXT: v_fmac_f32_e32 v3, v0, v1 +; GFX10-NEXT: v_add_f32_e32 v0, v3, v2 ; GFX10-NEXT: ; return to shader part epilog ; ; GFX10-CONTRACT-LABEL: test_f16_f32_add_fma_ext_mul: ; GFX10-CONTRACT: ; %bb.0: ; %.entry ; GFX10-CONTRACT-NEXT: v_mul_f16_e32 v3, v3, v4 -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_e32 v3, v3 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v3, v0, v1 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v0, v3, v2 ; GFX10-CONTRACT-NEXT: ; return to shader part epilog ; ; GFX10-DENORM-LABEL: test_f16_f32_add_fma_ext_mul: @@ -92,15 +94,17 @@ ; GFX10-LABEL: test_f16_f32_add_fma_ext_mul_rhs: ; GFX10: ; %bb.0: ; %.entry ; GFX10-NEXT: v_mul_f16_e32 v3, v3, v4 -; GFX10-NEXT: v_fma_mix_f32 v1, v1, v2, v3 op_sel_hi:[0,0,1] -; GFX10-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v3 +; GFX10-NEXT: v_fmac_f32_e32 v3, v1, v2 +; GFX10-NEXT: v_add_f32_e32 v0, v0, v3 ; GFX10-NEXT: ; return to shader part epilog ; ; GFX10-CONTRACT-LABEL: test_f16_f32_add_fma_ext_mul_rhs: ; GFX10-CONTRACT: ; %bb.0: ; %.entry ; GFX10-CONTRACT-NEXT: v_mul_f16_e32 v3, v3, v4 -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v1, v1, v2, v3 op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_e32 v3, v3 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v3, v1, v2 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v0, v0, v3 ; GFX10-CONTRACT-NEXT: ; return to shader part epilog ; ; GFX10-DENORM-LABEL: test_f16_f32_add_fma_ext_mul_rhs: @@ -177,28 +181,36 @@ ; GFX10: ; %bb.0: ; %.entry ; GFX10-NEXT: v_pk_mul_f16 v12, v12, v14 ; GFX10-NEXT: v_pk_mul_f16 v13, v13, v15 -; GFX10-NEXT: v_fma_mix_f32 v0, v0, v4, v12 op_sel_hi:[0,0,1] -; GFX10-NEXT: v_fma_mix_f32 v1, v1, v5, v12 op_sel:[0,0,1] op_sel_hi:[0,0,1] -; GFX10-NEXT: v_fma_mix_f32 v2, v2, v6, v13 op_sel_hi:[0,0,1] -; GFX10-NEXT: v_fma_mix_f32 v3, v3, v7, v13 op_sel:[0,0,1] op_sel_hi:[0,0,1] -; GFX10-NEXT: v_add_f32_e32 v0, v0, v8 -; GFX10-NEXT: v_add_f32_e32 v1, v1, v9 -; GFX10-NEXT: v_add_f32_e32 v2, v2, v10 -; GFX10-NEXT: v_add_f32_e32 v3, v3, v11 +; GFX10-NEXT: v_cvt_f32_f16_e32 v14, v12 +; GFX10-NEXT: v_cvt_f32_f16_sdwa v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-NEXT: v_cvt_f32_f16_e32 v15, v13 +; GFX10-NEXT: v_cvt_f32_f16_sdwa v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-NEXT: v_fmac_f32_e32 v14, v0, v4 +; GFX10-NEXT: v_fmac_f32_e32 v12, v1, v5 +; GFX10-NEXT: v_fmac_f32_e32 v15, v2, v6 +; GFX10-NEXT: v_fmac_f32_e32 v13, v3, v7 +; GFX10-NEXT: v_add_f32_e32 v0, v14, v8 +; GFX10-NEXT: v_add_f32_e32 v1, v12, v9 +; GFX10-NEXT: v_add_f32_e32 v2, v15, v10 +; GFX10-NEXT: v_add_f32_e32 v3, v13, v11 ; GFX10-NEXT: ; return to shader part epilog ; ; GFX10-CONTRACT-LABEL: test_v4f16_v4f32_add_fma_ext_mul: ; GFX10-CONTRACT: ; %bb.0: ; %.entry ; GFX10-CONTRACT-NEXT: v_pk_mul_f16 v12, v12, v14 ; GFX10-CONTRACT-NEXT: v_pk_mul_f16 v13, v13, v15 -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v0, v0, v4, v12 op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v1, v1, v5, v12 op_sel:[0,0,1] op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v2, v2, v6, v13 op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v3, v3, v7, v13 op_sel:[0,0,1] op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v0, v0, v8 -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v1, v1, v9 -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v2, v2, v10 -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v3, v3, v11 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_e32 v14, v12 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_sdwa v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_e32 v15, v13 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_sdwa v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v14, v0, v4 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v12, v1, v5 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v15, v2, v6 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v13, v3, v7 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v0, v14, v8 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v1, v12, v9 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v2, v15, v10 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v3, v13, v11 ; GFX10-CONTRACT-NEXT: ; return to shader part epilog ; ; GFX10-DENORM-LABEL: test_v4f16_v4f32_add_fma_ext_mul: @@ -319,28 +331,36 @@ ; GFX10: ; %bb.0: ; %.entry ; GFX10-NEXT: v_pk_mul_f16 v12, v12, v14 ; GFX10-NEXT: v_pk_mul_f16 v13, v13, v15 -; GFX10-NEXT: v_fma_mix_f32 v4, v4, v8, v12 op_sel_hi:[0,0,1] -; GFX10-NEXT: v_fma_mix_f32 v5, v5, v9, v12 op_sel:[0,0,1] op_sel_hi:[0,0,1] -; GFX10-NEXT: v_fma_mix_f32 v6, v6, v10, v13 op_sel_hi:[0,0,1] -; GFX10-NEXT: v_fma_mix_f32 v7, v7, v11, v13 op_sel:[0,0,1] op_sel_hi:[0,0,1] -; GFX10-NEXT: v_add_f32_e32 v0, v0, v4 -; GFX10-NEXT: v_add_f32_e32 v1, v1, v5 -; GFX10-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX10-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX10-NEXT: v_cvt_f32_f16_e32 v14, v12 +; GFX10-NEXT: v_cvt_f32_f16_sdwa v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-NEXT: v_cvt_f32_f16_e32 v15, v13 +; GFX10-NEXT: v_cvt_f32_f16_sdwa v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-NEXT: v_fmac_f32_e32 v14, v4, v8 +; GFX10-NEXT: v_fmac_f32_e32 v12, v5, v9 +; GFX10-NEXT: v_fmac_f32_e32 v15, v6, v10 +; GFX10-NEXT: v_fmac_f32_e32 v13, v7, v11 +; GFX10-NEXT: v_add_f32_e32 v0, v0, v14 +; GFX10-NEXT: v_add_f32_e32 v1, v1, v12 +; GFX10-NEXT: v_add_f32_e32 v2, v2, v15 +; GFX10-NEXT: v_add_f32_e32 v3, v3, v13 ; GFX10-NEXT: ; return to shader part epilog ; ; GFX10-CONTRACT-LABEL: test_v4f16_v4f32_add_fma_ext_mul_rhs: ; GFX10-CONTRACT: ; %bb.0: ; %.entry ; GFX10-CONTRACT-NEXT: v_pk_mul_f16 v12, v12, v14 ; GFX10-CONTRACT-NEXT: v_pk_mul_f16 v13, v13, v15 -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v4, v4, v8, v12 op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v5, v5, v9, v12 op_sel:[0,0,1] op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v6, v6, v10, v13 op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_fma_mix_f32 v7, v7, v11, v13 op_sel:[0,0,1] op_sel_hi:[0,0,1] -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v0, v0, v4 -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v1, v1, v5 -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX10-CONTRACT-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_e32 v14, v12 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_sdwa v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_e32 v15, v13 +; GFX10-CONTRACT-NEXT: v_cvt_f32_f16_sdwa v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v14, v4, v8 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v12, v5, v9 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v15, v6, v10 +; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v13, v7, v11 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v0, v0, v14 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v1, v1, v12 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v2, v2, v15 +; GFX10-CONTRACT-NEXT: v_add_f32_e32 v3, v3, v13 ; GFX10-CONTRACT-NEXT: ; return to shader part epilog ; ; GFX10-DENORM-LABEL: test_v4f16_v4f32_add_fma_ext_mul_rhs: Index: llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll +++ llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll @@ -1,13 +1,14 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; Denormal mode shouldn't matter for f16, check with and without flushing. + ; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX6,GFX6-IEEE %s ; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX6,GFX6-FLUSH %s ; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX8 %s ; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX8 %s -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX9 %s -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX9 %s +; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-IEEE %s +; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-FLUSH %s ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s @@ -3602,3 +3603,6 @@ declare <2 x half> @llvm.sqrt.v2f16(<2 x half>) !0 = !{float 2.500000e+00} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX9-FLUSH: {{.*}} +; GFX9-IEEE: {{.*}} Index: llvm/test/CodeGen/AMDGPU/fdot2.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/fdot2.ll +++ llvm/test/CodeGen/AMDGPU/fdot2.ll @@ -135,7 +135,6 @@ ; GCN-DL-UNSAFE: v_fma_mix_f32 ; GFX906-CONTRACT: v_fma_mix_f32 -; GFX906-DENORM-CONTRACT: v_fma_mix_f32 define amdgpu_kernel void @dotproduct_v4f16(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) { @@ -172,7 +171,6 @@ ; GCN-DL-UNSAFE: v_fma_mix_f32 ; GFX906-CONTRACT: v_fma_mix_f32 -; GFX906-DENORM-CONTRACT: v_fma_mix_f32 define amdgpu_kernel void @NotAdotproduct(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) { @@ -209,7 +207,6 @@ ; GCN-DL-UNSAFE: v_fma_mix_f32 ; GFX906-CONTRACT: v_fma_mix_f32 -; GFX906-DENORM-CONTRACT: v_fma_mix_f32 define amdgpu_kernel void @Diff_Idx_NotAdotproduct(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) { Index: llvm/test/CodeGen/AMDGPU/fpext-free.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/fpext-free.ll +++ llvm/test/CodeGen/AMDGPU/fpext-free.ll @@ -124,14 +124,25 @@ ; fold (fadd (fma x, y, (fpext (fmul u, v))), z) ; -> (fma x, y, (fma (fpext u), (fpext v), z)) define float @fadd_muladd_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 { -; GFX11-LABEL: fadd_muladd_fpext_fmul_f16_to_f32: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] -; GFX11-NEXT: v_add_f32_e32 v0, v0, v4 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-F32FLUSH-LABEL: fadd_muladd_fpext_fmul_f16_to_f32: +; GFX11-F32FLUSH: ; %bb.0: ; %entry +; GFX11-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32FLUSH-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32FLUSH-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] +; GFX11-F32FLUSH-NEXT: v_add_f32_e32 v0, v0, v4 +; GFX11-F32FLUSH-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-F32DENORM-LABEL: fadd_muladd_fpext_fmul_f16_to_f32: +; GFX11-F32DENORM: ; %bb.0: ; %entry +; GFX11-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2 +; GFX11-F32DENORM-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_add_f32_e32 v0, v2, v4 +; GFX11-F32DENORM-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-F32FLUSH-LABEL: fadd_muladd_fpext_fmul_f16_to_f32: ; GFX9-F32FLUSH: ; %bb.0: ; %entry @@ -160,14 +171,25 @@ ; fold (fadd x, (fma y, z, (fpext (fmul u, v))) ; -> (fma y, z, (fma (fpext u), (fpext v), x)) define float @fadd_muladd_fpext_fmul_f16_to_f32_commute(float %x, float %y, half %u, half %v, float %z) #0 { -; GFX11-LABEL: fadd_muladd_fpext_fmul_f16_to_f32_commute: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] -; GFX11-NEXT: v_add_f32_e32 v0, v4, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-F32FLUSH-LABEL: fadd_muladd_fpext_fmul_f16_to_f32_commute: +; GFX11-F32FLUSH: ; %bb.0: ; %entry +; GFX11-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32FLUSH-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32FLUSH-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] +; GFX11-F32FLUSH-NEXT: v_add_f32_e32 v0, v4, v0 +; GFX11-F32FLUSH-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-F32DENORM-LABEL: fadd_muladd_fpext_fmul_f16_to_f32_commute: +; GFX11-F32DENORM: ; %bb.0: ; %entry +; GFX11-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2 +; GFX11-F32DENORM-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_add_f32_e32 v0, v4, v2 +; GFX11-F32DENORM-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-F32FLUSH-LABEL: fadd_muladd_fpext_fmul_f16_to_f32_commute: ; GFX9-F32FLUSH: ; %bb.0: ; %entry @@ -194,14 +216,25 @@ } define float @fadd_fmad_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 { -; GFX11-LABEL: fadd_fmad_fpext_fmul_f16_to_f32: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] -; GFX11-NEXT: v_add_f32_e32 v0, v0, v4 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-F32FLUSH-LABEL: fadd_fmad_fpext_fmul_f16_to_f32: +; GFX11-F32FLUSH: ; %bb.0: ; %entry +; GFX11-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32FLUSH-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32FLUSH-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] +; GFX11-F32FLUSH-NEXT: v_add_f32_e32 v0, v0, v4 +; GFX11-F32FLUSH-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-F32DENORM-LABEL: fadd_fmad_fpext_fmul_f16_to_f32: +; GFX11-F32DENORM: ; %bb.0: ; %entry +; GFX11-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2 +; GFX11-F32DENORM-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_add_f32_e32 v0, v2, v4 +; GFX11-F32DENORM-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-F32FLUSH-LABEL: fadd_fmad_fpext_fmul_f16_to_f32: ; GFX9-F32FLUSH: ; %bb.0: ; %entry @@ -231,14 +264,25 @@ ; fold (fadd (fma x, y, (fpext (fmul u, v))), z) ; -> (fma x, y, (fma (fpext u), (fpext v), z)) define float @fadd_fma_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 { -; GFX11-LABEL: fadd_fma_fpext_fmul_f16_to_f32: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] -; GFX11-NEXT: v_add_f32_e32 v0, v0, v4 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-F32FLUSH-LABEL: fadd_fma_fpext_fmul_f16_to_f32: +; GFX11-F32FLUSH: ; %bb.0: ; %entry +; GFX11-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32FLUSH-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32FLUSH-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] +; GFX11-F32FLUSH-NEXT: v_add_f32_e32 v0, v0, v4 +; GFX11-F32FLUSH-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-F32DENORM-LABEL: fadd_fma_fpext_fmul_f16_to_f32: +; GFX11-F32DENORM: ; %bb.0: ; %entry +; GFX11-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2 +; GFX11-F32DENORM-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_add_f32_e32 v0, v2, v4 +; GFX11-F32DENORM-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-F32FLUSH-LABEL: fadd_fma_fpext_fmul_f16_to_f32: ; GFX9-F32FLUSH: ; %bb.0: ; %entry @@ -265,14 +309,25 @@ } define float @fadd_fma_fpext_fmul_f16_to_f32_commute(float %x, float %y, half %u, half %v, float %z) #0 { -; GFX11-LABEL: fadd_fma_fpext_fmul_f16_to_f32_commute: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] -; GFX11-NEXT: v_add_f32_e32 v0, v4, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-F32FLUSH-LABEL: fadd_fma_fpext_fmul_f16_to_f32_commute: +; GFX11-F32FLUSH: ; %bb.0: ; %entry +; GFX11-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32FLUSH-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32FLUSH-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1] +; GFX11-F32FLUSH-NEXT: v_add_f32_e32 v0, v4, v0 +; GFX11-F32FLUSH-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-F32DENORM-LABEL: fadd_fma_fpext_fmul_f16_to_f32_commute: +; GFX11-F32DENORM: ; %bb.0: ; %entry +; GFX11-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2 +; GFX11-F32DENORM-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_add_f32_e32 v0, v4, v2 +; GFX11-F32DENORM-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-F32FLUSH-LABEL: fadd_fma_fpext_fmul_f16_to_f32_commute: ; GFX9-F32FLUSH: ; %bb.0: ; %entry @@ -544,14 +599,25 @@ ; fold (fsub (fmad x, y, (fpext (fmul u, v))), z) ; -> (fmad x, y (fmad (fpext u), (fpext v), (fneg z))) define float @fsub_muladd_fpext_mul_f16_to_f32(float %x, float %y, float %z, half %u, half %v) #0 { -; GFX11-LABEL: fsub_muladd_fpext_mul_f16_to_f32: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1] -; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-F32FLUSH-LABEL: fsub_muladd_fpext_mul_f16_to_f32: +; GFX11-F32FLUSH: ; %bb.0: ; %entry +; GFX11-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32FLUSH-NEXT: v_mul_f16_e32 v3, v3, v4 +; GFX11-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32FLUSH-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1] +; GFX11-F32FLUSH-NEXT: v_sub_f32_e32 v0, v0, v2 +; GFX11-F32FLUSH-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-F32DENORM-LABEL: fsub_muladd_fpext_mul_f16_to_f32: +; GFX11-F32DENORM: ; %bb.0: ; %entry +; GFX11-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_cvt_f32_f16_e32 v3, v3 +; GFX11-F32DENORM-NEXT: v_fmac_f32_e32 v3, v0, v1 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_sub_f32_e32 v0, v3, v2 +; GFX11-F32DENORM-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-F32FLUSH-LABEL: fsub_muladd_fpext_mul_f16_to_f32: ; GFX9-F32FLUSH: ; %bb.0: ; %entry @@ -611,14 +677,25 @@ ; fold (fsub x, (fmad y, z, (fpext (fmul u, v)))) ; -> (fmad (fneg y), z, (fmad (fneg (fpext u)), (fpext v), x)) define float @fsub_muladd_fpext_mul_f16_to_f32_commute(float %x, float %y, float %z, half %u, half %v) #0 { -; GFX11-LABEL: fsub_muladd_fpext_mul_f16_to_f32_commute: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_mix_f32 v1, v1, v2, v3 op_sel_hi:[0,0,1] -; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-F32FLUSH-LABEL: fsub_muladd_fpext_mul_f16_to_f32_commute: +; GFX11-F32FLUSH: ; %bb.0: ; %entry +; GFX11-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32FLUSH-NEXT: v_mul_f16_e32 v3, v3, v4 +; GFX11-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32FLUSH-NEXT: v_fma_mix_f32 v1, v1, v2, v3 op_sel_hi:[0,0,1] +; GFX11-F32FLUSH-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX11-F32FLUSH-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-F32DENORM-LABEL: fsub_muladd_fpext_mul_f16_to_f32_commute: +; GFX11-F32DENORM: ; %bb.0: ; %entry +; GFX11-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_cvt_f32_f16_e32 v3, v3 +; GFX11-F32DENORM-NEXT: v_fmac_f32_e32 v3, v1, v2 +; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v3 +; GFX11-F32DENORM-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-F32FLUSH-LABEL: fsub_muladd_fpext_mul_f16_to_f32_commute: ; GFX9-F32FLUSH: ; %bb.0: ; %entry Index: llvm/test/CodeGen/AMDGPU/mad-mix.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/mad-mix.ll +++ llvm/test/CodeGen/AMDGPU/mad-mix.ll @@ -1468,7 +1468,11 @@ ; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals: ; GFX1100: ; %bb.0: ; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] +; GFX1100-NEXT: v_cvt_f32_f16_e32 v3, v0 +; GFX1100-NEXT: v_cvt_f32_f16_e32 v1, v1 +; GFX1100-NEXT: v_cvt_f32_f16_e32 v0, v2 +; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1100-NEXT: v_fmac_f32_e32 v0, v3, v1 ; GFX1100-NEXT: s_setpc_b64 s[30:31] ; ; GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals: @@ -1483,7 +1487,10 @@ ; GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals: ; GFX906: ; %bb.0: ; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] +; GFX906-NEXT: v_cvt_f32_f16_e32 v3, v0 +; GFX906-NEXT: v_cvt_f32_f16_e32 v1, v1 +; GFX906-NEXT: v_cvt_f32_f16_e32 v0, v2 +; GFX906-NEXT: v_fmac_f32_e32 v0, v3, v1 ; GFX906-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals: @@ -1521,7 +1528,10 @@ ; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals: ; GFX1100: ; %bb.0: ; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1100-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] +; GFX1100-NEXT: v_cvt_f32_f16_e32 v0, v0 +; GFX1100-NEXT: v_cvt_f32_f16_e32 v1, v1 +; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1100-NEXT: v_fma_f32 v0, v0, v1, v2 ; GFX1100-NEXT: s_setpc_b64 s[30:31] ; ; GFX900-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals: @@ -1535,7 +1545,9 @@ ; GFX906-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals: ; GFX906: ; %bb.0: ; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX906-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] +; GFX906-NEXT: v_cvt_f32_f16_e32 v0, v0 +; GFX906-NEXT: v_cvt_f32_f16_e32 v1, v1 +; GFX906-NEXT: v_fma_f32 v0, v0, v1, v2 ; GFX906-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_mad_mix_f32_f16lo_f16lo_f32_denormals: Index: llvm/test/CodeGen/AMDGPU/preserve-hi16.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/preserve-hi16.ll +++ llvm/test/CodeGen/AMDGPU/preserve-hi16.ll @@ -754,21 +754,23 @@ ; GFX906-LABEL: zext_fptrunc_fma_f16: ; GFX906: ; %bb.0: ; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX906-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 -; GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX906-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX906-NEXT: v_cvt_f16_f32_e32 v0, v2 ; GFX906-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: zext_fptrunc_fma_f16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 +; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: zext_fptrunc_fma_f16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 +; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v2 ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %fma = call float @llvm.fma.f32(float %x, float %y, float %z)