This is an archive of the discontinued LLVM Phabricator instance.

Differential D28332

DAGCombiner: Add hasOneUse checks to fadd/fma combine
ClosedPublic

Authored by arsenm on Jan 4 2017, 3:23 PM.

Details

Reviewers
RKSimon
• tstellarAMD
escha
Summary

Even with aggressive fusion enabled, this requires duplicating
the fmul, or increases an fadd to another fma which is not an
improvement.

Diff Detail

Event Timeline

arsenm updated this revision to Diff 83152.Jan 4 2017, 3:23 PM
arsenm retitled this revision from to DAGCombiner: Add hasOneUse checks to fadd/fma combine.
arsenm updated this object.
arsenm added a reviewer: escha.
arsenm added a subscriber: llvm-commits.
Herald added a reviewer: • tstellarAMD. · View Herald TranscriptJan 4 2017, 3:23 PM
Herald added subscribers: nhaehnle, wdng. · View Herald Transcript
arsenm added a comment.Jan 10 2017, 11:16 AM

ping

RKSimon accepted this revision.Jan 10 2017, 3:45 PM
RKSimon added a reviewer: RKSimon.
RKSimon added a subscriber: RKSimon.

Agreed, LGTM

lib/CodeGen/SelectionDAG/DAGCombiner.cpp
8141

clang-format - reduce lines

8154

clang-format - reduce lines

8388

clang-format - reduce lines

This revision is now accepted and ready to land.Jan 10 2017, 3:45 PM
arsenm closed this revision.Jan 10 2017, 6:13 PM

r291642

Revision Contents

PathSize
lib/
CodeGen/
SelectionDAG/
12 lines
test/
CodeGen/
AMDGPU/
262 lines

Diff 83152

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 8,130 Lines • ▼ Show 20 Lines if (N1.getOpcode() == ISD::FP_EXTEND) {
N10.getOperand(1)), N0); N10.getOperand(1)), N0);
} }
} }
// More folding opportunities when target permits. // More folding opportunities when target permits.
if ((AllowFusion || HasFMAD) && Aggressive) { if ((AllowFusion || HasFMAD) && Aggressive) {
// fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, z)) // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, z))
if (N0.getOpcode() == PreferredFusedOpcode && if (N0.getOpcode() == PreferredFusedOpcode &&
N0.getOperand(2).getOpcode() == ISD::FMUL) { N0.getOperand(2).getOpcode() == ISD::FMUL &&
N0->hasOneUse() &&
N0.getOperand(2)->hasOneUse()) {
RKSimonUnsubmitted
Not Done
ReplyInline Actions

clang-format - reduce lines

RKSimon: clang-format - reduce lines
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
N0.getOperand(0), N0.getOperand(1), N0.getOperand(0), N0.getOperand(1),
DAG.getNode(PreferredFusedOpcode, SL, VT, DAG.getNode(PreferredFusedOpcode, SL, VT,
N0.getOperand(2).getOperand(0), N0.getOperand(2).getOperand(0),
N0.getOperand(2).getOperand(1), N0.getOperand(2).getOperand(1),
N1)); N1));
} }
// fold (fadd x, (fma y, z, (fmul u, v)) -> (fma y, z (fma u, v, x)) // fold (fadd x, (fma y, z, (fmul u, v)) -> (fma y, z (fma u, v, x))
if (N1->getOpcode() == PreferredFusedOpcode && if (N1->getOpcode() == PreferredFusedOpcode &&
N1.getOperand(2).getOpcode() == ISD::FMUL) { N1.getOperand(2).getOpcode() == ISD::FMUL &&
N1->hasOneUse() &&
N1.getOperand(2)->hasOneUse()) {
RKSimonUnsubmitted
Not Done
ReplyInline Actions

clang-format - reduce lines

RKSimon: clang-format - reduce lines
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
N1.getOperand(0), N1.getOperand(1), N1.getOperand(0), N1.getOperand(1),
DAG.getNode(PreferredFusedOpcode, SL, VT, DAG.getNode(PreferredFusedOpcode, SL, VT,
N1.getOperand(2).getOperand(0), N1.getOperand(2).getOperand(0),
N1.getOperand(2).getOperand(1), N1.getOperand(2).getOperand(1),
N0)); N0));
} }
▲ Show 20 Lines • Show All 215 Lines • ▼ Show 20 LinesSDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
} }
// More folding opportunities when target permits. // More folding opportunities when target permits.
if ((AllowFusion || HasFMAD) && Aggressive) { if ((AllowFusion || HasFMAD) && Aggressive) {
// fold (fsub (fma x, y, (fmul u, v)), z) // fold (fsub (fma x, y, (fmul u, v)), z)
// -> (fma x, y (fma u, v, (fneg z))) // -> (fma x, y (fma u, v, (fneg z)))
if (N0.getOpcode() == PreferredFusedOpcode && if (N0.getOpcode() == PreferredFusedOpcode &&
N0.getOperand(2).getOpcode() == ISD::FMUL) { N0.getOperand(2).getOpcode() == ISD::FMUL &&
N0->hasOneUse() &&
N0.getOperand(2)->hasOneUse()) {
RKSimonUnsubmitted
Not Done
ReplyInline Actions

clang-format - reduce lines

RKSimon: clang-format - reduce lines
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
N0.getOperand(0), N0.getOperand(1), N0.getOperand(0), N0.getOperand(1),
DAG.getNode(PreferredFusedOpcode, SL, VT, DAG.getNode(PreferredFusedOpcode, SL, VT,
N0.getOperand(2).getOperand(0), N0.getOperand(2).getOperand(0),
N0.getOperand(2).getOperand(1), N0.getOperand(2).getOperand(1),
DAG.getNode(ISD::FNEG, SL, VT, DAG.getNode(ISD::FNEG, SL, VT,
N1))); N1)));
} }
▲ Show 20 Lines • Show All 7,171 Lines • Show Last 20 Lines

test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll

  • This file was added.
; RUN: llc -march=amdgcn -mattr=+fast-fmaf,-fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s
; RUN: llc -march=amdgcn -mattr=-fast-fmaf,-fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s
; RUN: llc -march=amdgcn -mattr=+fast-fmaf,+fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FASTFMA %s
; RUN: llc -march=amdgcn -mattr=-fast-fmaf,+fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-SLOWFMA %s
; FIXME: This should also fold when fma is actually fast if an FMA
; exists in the original program.
; (fadd (fma x, y, (fmul u, v), z) -> (fma x, y (fma u, v, z))
; GCN-LABEL: {{^}}fast_add_fmuladd_fmul:
; GCN: buffer_load_dword [[X:v[0-9]+]]
; GCN: buffer_load_dword [[Y:v[0-9]+]]
; GCN: buffer_load_dword [[Z:v[0-9]+]]
; GCN: buffer_load_dword [[U:v[0-9]+]]
; GCN: buffer_load_dword [[V:v[0-9]+]]
; GCN-FLUSH: v_mac_f32_e32 [[Z]], [[V]], [[U]]
; GCN-FLUSH-NEXT: v_mac_f32_e32 [[Z]], [[Y]], [[X]]
; GCN-FLUSH-NEXT: buffer_store_dword [[Z]]
; GCN-FASTFMA: v_fma_f32 [[FMA0:v[0-9]+]], [[U]], [[V]], [[Z]]
; GCN-FASTFMA: v_fma_f32 [[FMA1:v[0-9]+]], [[X]], [[Y]], [[FMA0]]
; GCN-FASTFMA: buffer_store_dword [[FMA1]]
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
define void @fast_add_fmuladd_fmul() #0 {
%x = load volatile float, float addrspace(1)* undef
%y = load volatile float, float addrspace(1)* undef
%z = load volatile float, float addrspace(1)* undef
%u = load volatile float, float addrspace(1)* undef
%v = load volatile float, float addrspace(1)* undef
%mul.u.v = fmul fast float %u, %v
%fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
%add = fadd fast float %fma, %z
store volatile float %add, float addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul:
; GCN: buffer_load_dword [[X:v[0-9]+]]
; GCN: buffer_load_dword [[Y:v[0-9]+]]
; GCN: buffer_load_dword [[Z:v[0-9]+]]
; GCN: buffer_load_dword [[U:v[0-9]+]]
; GCN: buffer_load_dword [[V:v[0-9]+]]
; GCN-FLUSH: v_mad_f32 [[TMP:v[0-9]]], [[U]], [[V]], -[[Z]]
; GCN-FLUSH-NEXT: v_mac_f32_e32 [[TMP]], [[Y]], [[X]]
; GCN-FLUSH-NEXT: buffer_store_dword [[Z]]
; GCN-FASTFMA: v_fma_f32 [[FMA0:v[0-9]+]], [[U]], [[V]], -[[Z]]
; GCN-FASTFMA: v_fma_f32 [[FMA1:v[0-9]+]], [[X]], [[Y]], [[FMA0]]
; GCN-FASTFMA: buffer_store_dword [[FMA1]]
define void @fast_sub_fmuladd_fmul() #0 {
%x = load volatile float, float addrspace(1)* undef
%y = load volatile float, float addrspace(1)* undef
%z = load volatile float, float addrspace(1)* undef
%u = load volatile float, float addrspace(1)* undef
%v = load volatile float, float addrspace(1)* undef
%mul.u.v = fmul fast float %u, %v
%fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
%add = fsub fast float %fma, %z
store volatile float %add, float addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}fast_add_fmuladd_fmul_multi_use_mul:
; GCN: buffer_load_dword [[X:v[0-9]+]]
; GCN: buffer_load_dword [[Y:v[0-9]+]]
; GCN: buffer_load_dword [[Z:v[0-9]+]]
; GCN: buffer_load_dword [[U:v[0-9]+]]
; GCN: buffer_load_dword [[V:v[0-9]+]]
; GCN-FLUSH-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[V]], [[U]]
; GCN-FLUSH-DAG: v_mac_f32_e32 [[MUL]], [[Y]], [[X]]
; GCN-FLUSH: v_add_f32_e32 v{{[0-9]+}}, [[Z]], [[U]]
; GCN-FASTFMA: v_mul_f32_e32 [[MUL:v[0-9]+]], [[V]], [[U]]
; GCN-FASTFMA: v_fma_f32 [[FMA1:v[0-9]+]], [[X]], [[Y]], [[MUL]]
; GCN-FASTFMA: v_add_f32_e32 v{{[0-9]+}}, [[Z]], [[FMA1]]
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
define void @fast_add_fmuladd_fmul_multi_use_mul() #0 {
%x = load volatile float, float addrspace(1)* undef
%y = load volatile float, float addrspace(1)* undef
%z = load volatile float, float addrspace(1)* undef
%u = load volatile float, float addrspace(1)* undef
%v = load volatile float, float addrspace(1)* undef
%mul.u.v = fmul fast float %u, %v
store volatile float %mul.u.v, float addrspace(1)* undef
%fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
%add = fadd fast float %fma, %z
store volatile float %add, float addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}fast_add_fmuladd_fmul_multi_use_mul_commute:
; GCN: buffer_load_dword [[X:v[0-9]+]]
; GCN: buffer_load_dword [[Y:v[0-9]+]]
; GCN: buffer_load_dword [[Z:v[0-9]+]]
; GCN: buffer_load_dword [[U:v[0-9]+]]
; GCN: buffer_load_dword [[V:v[0-9]+]]
; GCN-FLUSH-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[V]], [[U]]
; GCN-FLUSH-DAG: v_mac_f32_e32 [[MUL]], [[Y]], [[X]]
; GCN-FLUSH: v_add_f32_e32 v{{[0-9]+}}, [[U]], [[Z]]
; GCN-FASTFMA: v_mul_f32_e32 [[MUL:v[0-9]+]], [[V]], [[U]]
; GCN-FASTFMA: v_fma_f32 [[FMA1:v[0-9]+]], [[X]], [[Y]], [[MUL]]
; GCN-FASTFMA: v_add_f32_e32 v{{[0-9]+}}, [[FMA1]], [[Z]]
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
define void @fast_add_fmuladd_fmul_multi_use_mul_commute() #0 {
%x = load volatile float, float addrspace(1)* undef
%y = load volatile float, float addrspace(1)* undef
%z = load volatile float, float addrspace(1)* undef
%u = load volatile float, float addrspace(1)* undef
%v = load volatile float, float addrspace(1)* undef
%mul.u.v = fmul fast float %u, %v
store volatile float %mul.u.v, float addrspace(1)* undef
%fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
%add = fadd fast float %z, %fma
store volatile float %add, float addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}fast_add_fmuladd_fmul_multi_use_fmuladd:
; GCN: buffer_load_dword [[X:v[0-9]+]]
; GCN: buffer_load_dword [[Y:v[0-9]+]]
; GCN: buffer_load_dword [[Z:v[0-9]+]]
; GCN: buffer_load_dword [[U:v[0-9]+]]
; GCN: buffer_load_dword [[V:v[0-9]+]]
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
define void @fast_add_fmuladd_fmul_multi_use_fmuladd() #0 {
%x = load volatile float, float addrspace(1)* undef
%y = load volatile float, float addrspace(1)* undef
%z = load volatile float, float addrspace(1)* undef
%u = load volatile float, float addrspace(1)* undef
%v = load volatile float, float addrspace(1)* undef
%mul.u.v = fmul fast float %u, %v
%fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
store volatile float %fma, float addrspace(1)* undef
%add = fadd fast float %fma, %z
store volatile float %add, float addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}fast_add_fmuladd_fmul_multi_use_fmuladd_commute:
; GCN: buffer_load_dword [[X:v[0-9]+]]
; GCN: buffer_load_dword [[Y:v[0-9]+]]
; GCN: buffer_load_dword [[Z:v[0-9]+]]
; GCN: buffer_load_dword [[U:v[0-9]+]]
; GCN: buffer_load_dword [[V:v[0-9]+]]
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_mul_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
; GCN-SLOWFMA: v_add_f32_e32
define void @fast_add_fmuladd_fmul_multi_use_fmuladd_commute() #0 {
%x = load volatile float, float addrspace(1)* undef
%y = load volatile float, float addrspace(1)* undef
%z = load volatile float, float addrspace(1)* undef
%u = load volatile float, float addrspace(1)* undef
%v = load volatile float, float addrspace(1)* undef
%mul.u.v = fmul fast float %u, %v
%fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
store volatile float %fma, float addrspace(1)* undef
%add = fadd fast float %z, %fma
store volatile float %add, float addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul_multi_use_mul:
; GCN: buffer_load_dword [[X:v[0-9]+]]
; GCN: buffer_load_dword [[Y:v[0-9]+]]
; GCN: buffer_load_dword [[Z:v[0-9]+]]
; GCN: buffer_load_dword [[U:v[0-9]+]]
; GCN: buffer_load_dword [[V:v[0-9]+]]
; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[V]], [[U]]
; GCN-FLUSH: v_mad_f32 [[MAD:v[0-9]+]], [[Y]], [[X]], [[MUL]]
; GCN-FLUSH: v_subrev_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[MAD]]
; GCN-FASTFMA: v_fma_f32 [[MAD:v[0-9]+]], [[X]], [[Y]], [[MUL]]
; GCN-FASTFMA: v_subrev_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[MAD]]
; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[Y]], [[X]]
; GCN-SLOWFMA: v_add_f32_e32
; GCN-SLOWFMA: v_subrev_f32_e32 [[MAD:v[0-9]+]]
; GCN: buffer_store_dword [[MUL]]
; GCN: buffer_store_dword [[MAD]]
define void @fast_sub_fmuladd_fmul_multi_use_mul() #0 {
%x = load volatile float, float addrspace(1)* undef
%y = load volatile float, float addrspace(1)* undef
%z = load volatile float, float addrspace(1)* undef
%u = load volatile float, float addrspace(1)* undef
%v = load volatile float, float addrspace(1)* undef
%mul.u.v = fmul fast float %u, %v
%fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
%add = fsub fast float %fma, %z
store volatile float %mul.u.v, float addrspace(1)* undef
store volatile float %add, float addrspace(1)* undef
ret void
}
; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul_multi_use_fmuladd:
; GCN: buffer_load_dword [[X:v[0-9]+]]
; GCN: buffer_load_dword [[Y:v[0-9]+]]
; GCN: buffer_load_dword [[Z:v[0-9]+]]
; GCN: buffer_load_dword [[U:v[0-9]+]]
; GCN: buffer_load_dword [[V:v[0-9]+]]
; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[V]], [[U]]
; GCN-FLUSH-NEXT: v_mac_f32_e32 [[MUL]], [[Y]], [[X]]
; GCN-FLUSH-NEXT: v_subrev_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[MUL]]
; GCN-FLUSH-NEXT: buffer_store_dword [[MUL]]
; GCN-FLUSH-NEXT: buffer_store_dword [[SUB]]
; GCN-FASTFMA-NEXT: v_fma_f32 [[FMA:v[0-9]+]], [[X]], [[Y]], [[U]]
; GCN-FASTFMA-NEXT: v_subrev_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[FMA]]
; GCN-FASTFMA-NEXT: buffer_store_dword [[FMA]]
; GCN-FASTFMA-NEXT: buffer_store_dword [[SUB]]
; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[Y]], [[X]]
; GCN-SLOWFMA: v_add_f32_e32
; GCN-SLOWFMA: v_subrev_f32_e32
define void @fast_sub_fmuladd_fmul_multi_use_fmuladd() #0 {
%x = load volatile float, float addrspace(1)* undef
%y = load volatile float, float addrspace(1)* undef
%z = load volatile float, float addrspace(1)* undef
%u = load volatile float, float addrspace(1)* undef
%v = load volatile float, float addrspace(1)* undef
%mul.u.v = fmul fast float %u, %v
%fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)
%add = fsub fast float %fma, %z
store volatile float %fma, float addrspace(1)* undef
store volatile float %add, float addrspace(1)* undef
ret void
}
declare float @llvm.fma.f32(float, float, float) #1
declare float @llvm.fmuladd.f32(float, float, float) #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }