Index: lib/Target/AMDGPU/AMDGPUISelLowering.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -3323,6 +3323,16 @@ return performLoadCombine(N, DCI); case ISD::STORE: return performStoreCombine(N, DCI); + case AMDGPUISD::RCP: { + if (const auto *CFP = dyn_cast(N->getOperand(0))) { + // XXX - Should this flush denormals? + const APFloat &Val = CFP->getValueAPF(); + APFloat One(Val.getSemantics(), "1.0"); + return DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0)); + } + + break; + } } return SDValue(); } Index: test/CodeGen/AMDGPU/fdiv.f16.ll =================================================================== --- test/CodeGen/AMDGPU/fdiv.f16.ll +++ test/CodeGen/AMDGPU/fdiv.f16.ll @@ -204,6 +204,42 @@ ret void } +; FUNC-LABEL: {{^}}div_arcp_2_x_pat_f16: +; SI: v_mul_f32_e32 v{{[0-9]+}}, 0.5, v{{[0-9]+}} + +; VI: v_mul_f16_e32 [[MUL:v[0-9]+]], 0.5, v{{[0-9]+}} +; VI: buffer_store_short [[MUL]] +define void @div_arcp_2_x_pat_f16(half addrspace(1)* %out) #0 { + %x = load half, half addrspace(1)* undef + %rcp = fdiv arcp half %x, 2.0 + store half %rcp, half addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}div_arcp_k_x_pat_f16: +; SI: v_mul_f32_e32 v{{[0-9]+}}, 0x3dcccccd, v{{[0-9]+}} + +; VI: v_mul_f16_e32 [[MUL:v[0-9]+]], 0x2e66, v{{[0-9]+}} +; VI: buffer_store_short [[MUL]] +define void @div_arcp_k_x_pat_f16(half addrspace(1)* %out) #0 { + %x = load half, half addrspace(1)* undef + %rcp = fdiv arcp half %x, 10.0 + store half %rcp, half addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}div_arcp_neg_k_x_pat_f16: +; SI: v_mul_f32_e32 v{{[0-9]+}}, 0xbdcccccd, v{{[0-9]+}} + +; VI: v_mul_f16_e32 [[MUL:v[0-9]+]], 0xae66, v{{[0-9]+}} +; VI: buffer_store_short [[MUL]] +define void @div_arcp_neg_k_x_pat_f16(half addrspace(1)* %out) #0 { + %x = load half, half addrspace(1)* undef + %rcp = fdiv arcp half %x, -10.0 + store half %rcp, half addrspace(1)* %out, align 4 + ret void +} + declare i32 @llvm.amdgcn.workitem.id.x() #1 declare half @llvm.sqrt.f16(half) #1 declare half @llvm.fabs.f16(half) #1 Index: test/CodeGen/AMDGPU/fdiv.f64.ll =================================================================== --- test/CodeGen/AMDGPU/fdiv.f64.ll +++ test/CodeGen/AMDGPU/fdiv.f64.ll @@ -1,11 +1,11 @@ -; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=COMMON %s -; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=COMMON %s -; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=COMMON %s +; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN %s -; COMMON-LABEL: {{^}}fdiv_f64: -; COMMON-DAG: buffer_load_dwordx2 [[NUM:v\[[0-9]+:[0-9]+\]]], off, {{s\[[0-9]+:[0-9]+\]}}, 0 -; COMMON-DAG: buffer_load_dwordx2 [[DEN:v\[[0-9]+:[0-9]+\]]], off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:8 +; GCN-LABEL: {{^}}fdiv_f64: +; GCN-DAG: buffer_load_dwordx2 [[NUM:v\[[0-9]+:[0-9]+\]]], off, {{s\[[0-9]+:[0-9]+\]}}, 0 +; GCN-DAG: buffer_load_dwordx2 [[DEN:v\[[0-9]+:[0-9]+\]]], off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:8 ; CI-DAG: v_div_scale_f64 [[SCALE0:v\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, [[DEN]], [[DEN]], [[NUM]] ; CI-DAG: v_div_scale_f64 [[SCALE1:v\[[0-9]+:[0-9]+\]]], vcc, [[NUM]], [[DEN]], [[NUM]] @@ -13,23 +13,23 @@ ; SI-DAG: v_div_scale_f64 [[SCALE0:v\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, [[DEN]], [[DEN]], [[NUM]] ; SI-DAG: v_div_scale_f64 [[SCALE1:v\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, [[NUM]], [[DEN]], [[NUM]] -; COMMON-DAG: v_rcp_f64_e32 [[RCP_SCALE0:v\[[0-9]+:[0-9]+\]]], [[SCALE0]] +; GCN-DAG: v_rcp_f64_e32 [[RCP_SCALE0:v\[[0-9]+:[0-9]+\]]], [[SCALE0]] ; SI-DAG: v_cmp_eq_u32_e32 vcc, {{v[0-9]+}}, {{v[0-9]+}} ; SI-DAG: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, {{v[0-9]+}} ; SI-DAG: s_xor_b64 vcc, [[CMP0]], vcc -; COMMON-DAG: v_fma_f64 [[FMA0:v\[[0-9]+:[0-9]+\]]], -[[SCALE0]], [[RCP_SCALE0]], 1.0 -; COMMON-DAG: v_fma_f64 [[FMA1:v\[[0-9]+:[0-9]+\]]], [[RCP_SCALE0]], [[FMA0]], [[RCP_SCALE0]] -; COMMON-DAG: v_fma_f64 [[FMA2:v\[[0-9]+:[0-9]+\]]], -[[SCALE0]], [[FMA1]], 1.0 -; COMMON-DAG: v_fma_f64 [[FMA3:v\[[0-9]+:[0-9]+\]]], [[FMA1]], [[FMA2]], [[FMA1]] -; COMMON-DAG: v_mul_f64 [[MUL:v\[[0-9]+:[0-9]+\]]], [[SCALE1]], [[FMA3]] -; COMMON-DAG: v_fma_f64 [[FMA4:v\[[0-9]+:[0-9]+\]]], -[[SCALE0]], [[MUL]], [[SCALE1]] -; COMMON: v_div_fmas_f64 [[FMAS:v\[[0-9]+:[0-9]+\]]], [[FMA4]], [[FMA3]], [[MUL]] -; COMMON: v_div_fixup_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[FMAS]], [[DEN]], [[NUM]] -; COMMON: buffer_store_dwordx2 [[RESULT]] -; COMMON: s_endpgm -define void @fdiv_f64(double addrspace(1)* %out, double addrspace(1)* %in) nounwind { +; GCN-DAG: v_fma_f64 [[FMA0:v\[[0-9]+:[0-9]+\]]], -[[SCALE0]], [[RCP_SCALE0]], 1.0 +; GCN-DAG: v_fma_f64 [[FMA1:v\[[0-9]+:[0-9]+\]]], [[RCP_SCALE0]], [[FMA0]], [[RCP_SCALE0]] +; GCN-DAG: v_fma_f64 [[FMA2:v\[[0-9]+:[0-9]+\]]], -[[SCALE0]], [[FMA1]], 1.0 +; GCN-DAG: v_fma_f64 [[FMA3:v\[[0-9]+:[0-9]+\]]], [[FMA1]], [[FMA2]], [[FMA1]] +; GCN-DAG: v_mul_f64 [[MUL:v\[[0-9]+:[0-9]+\]]], [[SCALE1]], [[FMA3]] +; GCN-DAG: v_fma_f64 [[FMA4:v\[[0-9]+:[0-9]+\]]], -[[SCALE0]], [[MUL]], [[SCALE1]] +; GCN: v_div_fmas_f64 [[FMAS:v\[[0-9]+:[0-9]+\]]], [[FMA4]], [[FMA3]], [[MUL]] +; GCN: v_div_fixup_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[FMAS]], [[DEN]], [[NUM]] +; GCN: buffer_store_dwordx2 [[RESULT]] +; GCN: s_endpgm +define void @fdiv_f64(double addrspace(1)* %out, double addrspace(1)* %in) #0 { %gep.1 = getelementptr double, double addrspace(1)* %in, i32 1 %num = load volatile double, double addrspace(1)* %in %den = load volatile double, double addrspace(1)* %gep.1 @@ -38,31 +38,31 @@ ret void } -; COMMON-LABEL: {{^}}fdiv_f64_s_v: -define void @fdiv_f64_s_v(double addrspace(1)* %out, double addrspace(1)* %in, double %num) nounwind { +; GCN-LABEL: {{^}}fdiv_f64_s_v: +define void @fdiv_f64_s_v(double addrspace(1)* %out, double addrspace(1)* %in, double %num) #0 { %den = load double, double addrspace(1)* %in %result = fdiv double %num, %den store double %result, double addrspace(1)* %out ret void } -; COMMON-LABEL: {{^}}fdiv_f64_v_s: -define void @fdiv_f64_v_s(double addrspace(1)* %out, double addrspace(1)* %in, double %den) nounwind { +; GCN-LABEL: {{^}}fdiv_f64_v_s: +define void @fdiv_f64_v_s(double addrspace(1)* %out, double addrspace(1)* %in, double %den) #0 { %num = load double, double addrspace(1)* %in %result = fdiv double %num, %den store double %result, double addrspace(1)* %out ret void } -; COMMON-LABEL: {{^}}fdiv_f64_s_s: -define void @fdiv_f64_s_s(double addrspace(1)* %out, double %num, double %den) nounwind { +; GCN-LABEL: {{^}}fdiv_f64_s_s: +define void @fdiv_f64_s_s(double addrspace(1)* %out, double %num, double %den) #0 { %result = fdiv double %num, %den store double %result, double addrspace(1)* %out ret void } -; COMMON-LABEL: {{^}}v_fdiv_v2f64: -define void @v_fdiv_v2f64(<2 x double> addrspace(1)* %out, <2 x double> addrspace(1)* %in) nounwind { +; GCN-LABEL: {{^}}v_fdiv_v2f64: +define void @v_fdiv_v2f64(<2 x double> addrspace(1)* %out, <2 x double> addrspace(1)* %in) #0 { %gep.1 = getelementptr <2 x double>, <2 x double> addrspace(1)* %in, i32 1 %num = load <2 x double>, <2 x double> addrspace(1)* %in %den = load <2 x double>, <2 x double> addrspace(1)* %gep.1 @@ -71,15 +71,15 @@ ret void } -; COMMON-LABEL: {{^}}s_fdiv_v2f64: +; GCN-LABEL: {{^}}s_fdiv_v2f64: define void @s_fdiv_v2f64(<2 x double> addrspace(1)* %out, <2 x double> %num, <2 x double> %den) { %result = fdiv <2 x double> %num, %den store <2 x double> %result, <2 x double> addrspace(1)* %out ret void } -; COMMON-LABEL: {{^}}v_fdiv_v4f64: -define void @v_fdiv_v4f64(<4 x double> addrspace(1)* %out, <4 x double> addrspace(1)* %in) nounwind { +; GCN-LABEL: {{^}}v_fdiv_v4f64: +define void @v_fdiv_v4f64(<4 x double> addrspace(1)* %out, <4 x double> addrspace(1)* %in) #0 { %gep.1 = getelementptr <4 x double>, <4 x double> addrspace(1)* %in, i32 1 %num = load <4 x double>, <4 x double> addrspace(1)* %in %den = load <4 x double>, <4 x double> addrspace(1)* %gep.1 @@ -88,9 +88,46 @@ ret void } -; COMMON-LABEL: {{^}}s_fdiv_v4f64: -define void @s_fdiv_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %num, <4 x double> %den) { +; GCN-LABEL: {{^}}s_fdiv_v4f64: +define void @s_fdiv_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %num, <4 x double> %den) #0 { %result = fdiv <4 x double> %num, %den store <4 x double> %result, <4 x double> addrspace(1)* %out ret void } + +; GCN-LABEL: {{^}}div_fast_2_x_pat_f64: +; GCN: v_mul_f64 [[MUL:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, 0.5 +; GCN: buffer_store_dwordx2 [[MUL]] +define void @div_fast_2_x_pat_f64(double addrspace(1)* %out) #1 { + %x = load double, double addrspace(1)* undef + %rcp = fdiv fast double %x, 2.0 + store double %rcp, double addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}div_fast_k_x_pat_f64: +; GCN-DAG: s_mov_b32 s[[K_LO:[0-9]+]], 0x9999999a +; GCN-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0x3fb99999 +; GCN: v_mul_f64 [[MUL:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}} +; GCN: buffer_store_dwordx2 [[MUL]] +define void @div_fast_k_x_pat_f64(double addrspace(1)* %out) #1 { + %x = load double, double addrspace(1)* undef + %rcp = fdiv fast double %x, 10.0 + store double %rcp, double addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}div_fast_neg_k_x_pat_f64: +; GCN-DAG: s_mov_b32 s[[K_LO:[0-9]+]], 0x9999999a +; GCN-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0xbfb99999 +; GCN: v_mul_f64 [[MUL:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}} +; GCN: buffer_store_dwordx2 [[MUL]] +define void @div_fast_neg_k_x_pat_f64(double addrspace(1)* %out) #1 { + %x = load double, double addrspace(1)* undef + %rcp = fdiv fast double %x, -10.0 + store double %rcp, double addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind } +attributes #1 = { nounwind "unsafe-fp-math"="true" } Index: test/CodeGen/AMDGPU/llvm.amdgcn.rcp.ll =================================================================== --- test/CodeGen/AMDGPU/llvm.amdgcn.rcp.ll +++ test/CodeGen/AMDGPU/llvm.amdgcn.rcp.ll @@ -14,6 +14,24 @@ ret void } +; FUNC-LABEL: {{^}}rcp_2_f32: +; SI-NOT: v_rcp_f32 +; SI: v_mov_b32_e32 v{{[0-9]+}}, 0.5 +define void @rcp_2_f32(float addrspace(1)* %out) #1 { + %rcp = call float @llvm.amdgcn.rcp.f32(float 2.0) + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}rcp_10_f32: +; SI-NOT: v_rcp_f32 +; SI: v_mov_b32_e32 v{{[0-9]+}}, 0x3dcccccd +define void @rcp_10_f32(float addrspace(1)* %out) #1 { + %rcp = call float @llvm.amdgcn.rcp.f32(float 10.0) + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + ; FUNC-LABEL: {{^}}safe_no_fp32_denormals_rcp_f32: ; SI: v_rcp_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}} ; SI-NOT: [[RESULT]] Index: test/CodeGen/AMDGPU/rcp-pattern.ll =================================================================== --- test/CodeGen/AMDGPU/rcp-pattern.ll +++ test/CodeGen/AMDGPU/rcp-pattern.ll @@ -117,6 +117,35 @@ ret void } +; FUNC-LABEL: {{^}}div_arcp_2_x_pat_f32: +; GCN: v_mul_f32_e32 [[MUL:v[0-9]+]], 0.5, v{{[0-9]+}} +; GCN: buffer_store_dword [[MUL]] +define void @div_arcp_2_x_pat_f32(float addrspace(1)* %out) #0 { + %x = load float, float addrspace(1)* undef + %rcp = fdiv arcp float %x, 2.0 + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}div_arcp_k_x_pat_f32: +; GCN: v_mul_f32_e32 [[MUL:v[0-9]+]], 0x3dcccccd, v{{[0-9]+}} +; GCN: buffer_store_dword [[MUL]] +define void @div_arcp_k_x_pat_f32(float addrspace(1)* %out) #0 { + %x = load float, float addrspace(1)* undef + %rcp = fdiv arcp float %x, 10.0 + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}div_arcp_neg_k_x_pat_f32: +; GCN: v_mul_f32_e32 [[MUL:v[0-9]+]], 0xbdcccccd, v{{[0-9]+}} +; GCN: buffer_store_dword [[MUL]] +define void @div_arcp_neg_k_x_pat_f32(float addrspace(1)* %out) #0 { + %x = load float, float addrspace(1)* undef + %rcp = fdiv arcp float %x, -10.0 + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} declare float @llvm.fabs.f32(float) #1 declare float @llvm.sqrt.f32(float) #1