Index: llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -11916,7 +11916,15 @@ // possibly be enough uses of the divisor to make the transform worthwhile. SDValue N1 = N->getOperand(1); unsigned MinUses = TLI.combineRepeatedFPDivisors(); - if (!MinUses || N1->use_size() < MinUses) + + // For vectors, scale the number of uses by the splat factor. If we can + // convert the division into a scalar op, that will likely be much faster. + unsigned NumElts = 1; + EVT VT = N->getValueType(0); + if (VT.isVector() && DAG.isSplatValue(N1)) + NumElts = VT.getVectorNumElements(); + + if (!MinUses || (N1->use_size() * NumElts) < MinUses) return SDValue(); // Find all FDIV users of the same divisor. @@ -11933,10 +11941,9 @@ // Now that we have the actual number of divisor uses, make sure it meets // the minimum threshold specified by the target. - if (Users.size() < MinUses) + if ((Users.size() * NumElts) < MinUses) return SDValue(); - EVT VT = N->getValueType(0); SDLoc DL(N); SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); @@ -11976,6 +11983,9 @@ if (N0CFP && N1CFP) return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1, Flags); + if (SDValue CombineRepeatedDivisors = combineRepeatedFPDivisors(N)) + return CombineRepeatedDivisors; + if (SDValue NewSel = foldBinOpIntoSelect(N)) return NewSel; @@ -12068,9 +12078,6 @@ } } - if (SDValue CombineRepeatedDivisors = combineRepeatedFPDivisors(N)) - return CombineRepeatedDivisors; - return SDValue(); } Index: llvm/test/CodeGen/X86/fdiv-combine-vec.ll =================================================================== --- llvm/test/CodeGen/X86/fdiv-combine-vec.ll +++ llvm/test/CodeGen/X86/fdiv-combine-vec.ll @@ -5,14 +5,18 @@ define <2 x double> @splat_fdiv_v2f64(<2 x double> %x, double %y) { ; SSE-LABEL: splat_fdiv_v2f64: ; SSE: # %bb.0: -; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0,0] -; SSE-NEXT: divpd %xmm1, %xmm0 +; SSE-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero +; SSE-NEXT: divsd %xmm1, %xmm2 +; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0,0] +; SSE-NEXT: mulpd %xmm2, %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: splat_fdiv_v2f64: ; AVX: # %bb.0: +; AVX-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero +; AVX-NEXT: vdivsd %xmm1, %xmm2, %xmm1 ; AVX-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] -; AVX-NEXT: vdivpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmulpd %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq %vy = insertelement <2 x double> undef, double %y, i32 0 %splaty = shufflevector <2 x double> %vy, <2 x double> undef, <2 x i32> zeroinitializer @@ -32,9 +36,11 @@ ; ; AVX-LABEL: splat_fdiv_v4f64: ; AVX: # %bb.0: +; AVX-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero +; AVX-NEXT: vdivsd %xmm1, %xmm2, %xmm1 ; AVX-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm1, %ymm1 -; AVX-NEXT: vdivpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vmulpd %ymm1, %ymm0, %ymm0 ; AVX-NEXT: retq %vy = insertelement <4 x double> undef, double %y, i32 0 %splaty = shufflevector <4 x double> %vy, <4 x double> undef, <4 x i32> zeroinitializer @@ -45,25 +51,17 @@ define <4 x float> @splat_fdiv_v4f32(<4 x float> %x, float %y) { ; SSE-LABEL: splat_fdiv_v4f32: ; SSE: # %bb.0: -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,0,0] -; SSE-NEXT: rcpps %xmm1, %xmm2 -; SSE-NEXT: mulps %xmm2, %xmm1 -; SSE-NEXT: movaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] -; SSE-NEXT: subps %xmm1, %xmm3 -; SSE-NEXT: mulps %xmm2, %xmm3 -; SSE-NEXT: addps %xmm2, %xmm3 -; SSE-NEXT: mulps %xmm3, %xmm0 +; SSE-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero +; SSE-NEXT: divss %xmm1, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0,0,0] +; SSE-NEXT: mulps %xmm2, %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: splat_fdiv_v4f32: ; AVX: # %bb.0: +; AVX-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero +; AVX-NEXT: vdivss %xmm1, %xmm2, %xmm1 ; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[0,0,0,0] -; AVX-NEXT: vrcpps %xmm1, %xmm2 -; AVX-NEXT: vmulps %xmm2, %xmm1, %xmm1 -; AVX-NEXT: vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] -; AVX-NEXT: vsubps %xmm1, %xmm3, %xmm1 -; AVX-NEXT: vmulps %xmm1, %xmm2, %xmm1 -; AVX-NEXT: vaddps %xmm1, %xmm2, %xmm1 ; AVX-NEXT: vmulps %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq %vy = insertelement <4 x float> undef, float %y, i32 0 @@ -75,27 +73,19 @@ define <8 x float> @splat_fdiv_v8f32(<8 x float> %x, float %y) { ; SSE-LABEL: splat_fdiv_v8f32: ; SSE: # %bb.0: -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0,0,0] -; SSE-NEXT: rcpps %xmm2, %xmm3 -; SSE-NEXT: mulps %xmm3, %xmm2 -; SSE-NEXT: movaps {{.*#+}} xmm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] -; SSE-NEXT: subps %xmm2, %xmm4 -; SSE-NEXT: mulps %xmm3, %xmm4 -; SSE-NEXT: addps %xmm3, %xmm4 -; SSE-NEXT: mulps %xmm4, %xmm0 -; SSE-NEXT: mulps %xmm4, %xmm1 +; SSE-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero +; SSE-NEXT: divss %xmm2, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,0,0,0] +; SSE-NEXT: mulps %xmm3, %xmm0 +; SSE-NEXT: mulps %xmm3, %xmm1 ; SSE-NEXT: retq ; ; AVX-LABEL: splat_fdiv_v8f32: ; AVX: # %bb.0: +; AVX-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero +; AVX-NEXT: vdivss %xmm1, %xmm2, %xmm1 ; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[0,0,0,0] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm1, %ymm1 -; AVX-NEXT: vrcpps %ymm1, %ymm2 -; AVX-NEXT: vmulps %ymm2, %ymm1, %ymm1 -; AVX-NEXT: vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] -; AVX-NEXT: vsubps %ymm1, %ymm3, %ymm1 -; AVX-NEXT: vmulps %ymm1, %ymm2, %ymm1 -; AVX-NEXT: vaddps %ymm1, %ymm2, %ymm1 ; AVX-NEXT: vmulps %ymm1, %ymm0, %ymm0 ; AVX-NEXT: retq %vy = insertelement <8 x float> undef, float %y, i32 0