Index: lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -8273,6 +8273,12 @@ } } + // For reciprocal optimization of a division node, check the division + // node's optimization flags as well as the function-wide fast-math setting. + bool AllowRecip = false; + if (const auto *NodeWithFlags = dyn_cast(N)) + AllowRecip = NodeWithFlags->Flags.hasAllowReciprocal(); + // Combine multiple FDIVs with the same divisor into multiple FMULs by the // reciprocal. // E.g., (a / D; b / D;) -> (recip = 1.0 / D; a * recip; b * recip) @@ -8280,7 +8286,7 @@ // may have different costs for FDIV and FMUL, so sometimes the cost of two // FDIVs may be lower than the cost of one FDIV and two FMULs. Another reason // is the critical path is increased from "one FDIV" to "one FDIV + one FMUL". - if (Options.UnsafeFPMath) { + if (Options.UnsafeFPMath || AllowRecip) { // Skip if current node is a reciprocal. if (N0CFP && N0CFP->isExactlyValue(1.0)) return SDValue(); @@ -8292,7 +8298,9 @@ UI != UE; ++UI) { SDNode *User = UI.getUse().getUser(); if (User->getOpcode() == ISD::FDIV && User->getOperand(1) == N1) - Users.push_back(User); + if (const auto *NodeWithFlags = dyn_cast(User)) + if (Options.UnsafeFPMath || NodeWithFlags->Flags.hasAllowReciprocal()) + Users.push_back(User); } if (TLI.combineRepeatedFPDivisors(Users.size())) { Index: test/CodeGen/X86/fdiv-combine.ll =================================================================== --- test/CodeGen/X86/fdiv-combine.ll +++ test/CodeGen/X86/fdiv-combine.ll @@ -1,9 +1,11 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s -; Anything more than one division using a single divisor operand +; More than one 'arcp' division using a single divisor operand ; should be converted into a reciprocal and multiplication. -define float @div1_arcp(float %x, float %y, float %z) #0 { +; Don't do anything for just one division. +; +define float @div1_arcp(float %x, float %y, float %z) { ; CHECK-LABEL: div1_arcp: ; CHECK: # BB#0: ; CHECK-NEXT: divss %xmm1, %xmm0 @@ -12,8 +14,10 @@ ret float %div1 } -define float @div2_arcp(float %x, float %y, float %z) #0 { -; CHECK-LABEL: div2_arcp: +; All math instructions are 'arcp', so optimize. +; +define float @div2_arcp_all(float %x, float %y, float %z) { +; CHECK-LABEL: div2_arcp_all: ; CHECK: # BB#0: ; CHECK-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero ; CHECK-NEXT: divss %xmm2, %xmm3 @@ -27,5 +31,50 @@ ret float %div2 } -; FIXME: If the backend understands 'arcp', then this attribute is unnecessary. -attributes #0 = { "unsafe-fp-math"="true" } +; The first division is not 'arcp', so do not optimize. +; +define float @div2_arcp_partial1(float %x, float %y, float %z) { +; CHECK-LABEL: div2_arcp_partial1: +; CHECK: # BB#0: +; CHECK-NEXT: divss %xmm2, %xmm0 +; CHECK-NEXT: mulss %xmm1, %xmm0 +; CHECK-NEXT: divss %xmm2, %xmm0 +; CHECK-NEXT: retq + %div1 = fdiv float %x, %z + %mul = fmul arcp float %div1, %y + %div2 = fdiv arcp float %mul, %z + ret float %div2 +} + +; The second division is not 'arcp', so do not optimize. +; +define float @div2_arcp_partial2(float %x, float %y, float %z) { +; CHECK-LABEL: div2_arcp_partial2: +; CHECK: # BB#0: +; CHECK-NEXT: divss %xmm2, %xmm0 +; CHECK-NEXT: mulss %xmm1, %xmm0 +; CHECK-NEXT: divss %xmm2, %xmm0 +; CHECK-NEXT: retq + %div1 = fdiv arcp float %x, %z + %mul = fmul arcp float %div1, %y + %div2 = fdiv float %mul, %z + ret float %div2 +} + +; The multiply is not 'arcp', but that does not prevent optimizing the divisions. +; +define float @div2_arcp_partial3(float %x, float %y, float %z) { +; CHECK-LABEL: div2_arcp_partial3: +; CHECK: # BB#0: +; CHECK-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero +; CHECK-NEXT: divss %xmm2, %xmm3 +; CHECK-NEXT: mulss %xmm3, %xmm0 +; CHECK-NEXT: mulss %xmm1, %xmm0 +; CHECK-NEXT: mulss %xmm3, %xmm0 +; CHECK-NEXT: retq + %div1 = fdiv arcp float %x, %z + %mul = fmul float %div1, %y + %div2 = fdiv arcp float %mul, %z + ret float %div2 +} +