Index: clang/test/CodeGen/complex-math.c =================================================================== --- clang/test/CodeGen/complex-math.c +++ clang/test/CodeGen/complex-math.c @@ -148,10 +148,11 @@ // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast float [[CC]], [[DD]] // // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast float [[D]], %a - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fdiv fast float [[AC]], [[CCpDD]] - // AARCH64-FASTMATH: [[DIV:%.*]] = fdiv fast float [[AD]], [[CCpDD]] - // AARCH64-FASTMATH: fsub fast float -0.000000e+00, [[DIV]] + // AARCH64-FASTMATH: [[NEGA:%.*]] = fsub fast float -0.000000e+00, %a + // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast float [[D]], [[NEGA]] + // + // AARCH64-FASTMATH: fdiv fast float [[AC]], [[CCpDD]] + // AARCH64-FASTMATH: fdiv fast float [[AD]], [[CCpDD]] // AARCH64-FASTMATH: ret return a / b; } @@ -325,10 +326,11 @@ // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast double [[CC]], [[DD]] // // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast double [[D]], %a - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fdiv fast double [[AC]], [[CCpDD]] - // AARCH64-FASTMATH: [[DIV:%.*]] = fdiv fast double [[AD]], [[CCpDD]] - // AARCH64-FASTMATH: fsub fast double -0.000000e+00, [[DIV]] + // AARCH64-FASTMATH: [[NEGA:%.*]] = fsub fast double -0.000000e+00, %a + // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast double [[D]], [[NEGA]] + // + // AARCH64-FASTMATH: fdiv fast double [[AC]], [[CCpDD]] + // AARCH64-FASTMATH: fdiv fast double [[AD]], [[CCpDD]] // AARCH64-FASTMATH: ret return a / b; } @@ -520,10 +522,11 @@ // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast fp128 [[CC]], [[DD]] // // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast fp128 [[D]], %a - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fdiv fast fp128 [[AC]], [[CCpDD]] - // AARCH64-FASTMATH: [[DIV:%.*]] = fdiv fast fp128 [[AD]], [[CCpDD]] - // AARCH64-FASTMATH: fsub fast fp128 0xL00000000000000008000000000000000, [[DIV]] + // AARCH64-FASTMATH: [[NEGA:%.*]] = fsub fast fp128 0xL00000000000000008000000000000000, %a + // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast fp128 [[D]], [[NEGA]] + // + // AARCH64-FASTMATH: fdiv fast fp128 [[AC]], [[CCpDD]] + // AARCH64-FASTMATH: fdiv fast fp128 [[AD]], [[CCpDD]] // AARCH64-FASTMATH: ret return a / b; } Index: llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp =================================================================== --- llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -1900,6 +1900,22 @@ return nullptr; } +static Instruction *hoistFNegAboveFMulFDiv(Instruction &I, + InstCombiner::BuilderTy &Builder) { + Value *FNeg; + if (!match(&I, m_FNeg(m_Value(FNeg)))) + return nullptr; + + Value *X, *Y; + if (match(FNeg, m_OneUse(m_FMul(m_Value(X), m_Value(Y))))) + return BinaryOperator::CreateFMulFMF(Builder.CreateFNegFMF(X, &I), Y, &I); + + if (match(FNeg, m_OneUse(m_FDiv(m_Value(X), m_Value(Y))))) + return BinaryOperator::CreateFDivFMF(Builder.CreateFNegFMF(X, &I), Y, &I); + + return nullptr; +} + Instruction *InstCombiner::visitFNeg(UnaryOperator &I) { Value *Op = I.getOperand(0); @@ -1917,6 +1933,9 @@ match(Op, m_OneUse(m_FSub(m_Value(X), m_Value(Y))))) return BinaryOperator::CreateFSubFMF(Y, X, &I); + if (Instruction *R = hoistFNegAboveFMulFDiv(I, Builder)) + return R; + return nullptr; } @@ -1938,6 +1957,9 @@ if (Instruction *X = foldFNegIntoConstant(I)) return X; + if (Instruction *R = hoistFNegAboveFMulFDiv(I, Builder)) + return R; + Value *X, *Y; Constant *C; Index: llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp =================================================================== --- llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -373,16 +373,6 @@ if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_Constant(C))) return BinaryOperator::CreateFMulFMF(X, ConstantExpr::getFNeg(C), &I); - // Sink negation: -X * Y --> -(X * Y) - // But don't transform constant expressions because there's an inverse fold. - if (match(Op0, m_OneUse(m_FNeg(m_Value(X)))) && !isa(Op0)) - return BinaryOperator::CreateFNegFMF(Builder.CreateFMulFMF(X, Op1, &I), &I); - - // Sink negation: Y * -X --> -(X * Y) - // But don't transform constant expressions because there's an inverse fold. - if (match(Op1, m_OneUse(m_FNeg(m_Value(X)))) && !isa(Op1)) - return BinaryOperator::CreateFNegFMF(Builder.CreateFMulFMF(X, Op0, &I), &I); - // fabs(X) * fabs(X) -> X * X if (Op0 == Op1 && match(Op0, m_Intrinsic(m_Value(X)))) return BinaryOperator::CreateFMulFMF(X, X, &I); @@ -1234,16 +1224,6 @@ return &I; } - // Sink negation: -X / Y --> -(X / Y) - // But don't transform constant expressions because there's an inverse fold. - if (match(Op0, m_OneUse(m_FNeg(m_Value(X)))) && !isa(Op0)) - return BinaryOperator::CreateFNegFMF(Builder.CreateFDivFMF(X, Op1, &I), &I); - - // Sink negation: Y / -X --> -(Y / X) - // But don't transform constant expressions because there's an inverse fold. - if (match(Op1, m_OneUse(m_FNeg(m_Value(X)))) && !isa(Op1)) - return BinaryOperator::CreateFNegFMF(Builder.CreateFDivFMF(Op0, X, &I), &I); - // X / (X * Y) --> 1.0 / Y // Reassociate to (X / X -> 1.0) is legal when NaNs are not allowed. // We can ignore the possibility that X is infinity because INF/INF is NaN. Index: llvm/test/Transforms/InstCombine/fadd.ll =================================================================== --- llvm/test/Transforms/InstCombine/fadd.ll +++ llvm/test/Transforms/InstCombine/fadd.ll @@ -165,10 +165,10 @@ define float @fdiv_fneg1_extra_use(float %x, float %y, float %pz) { ; CHECK-LABEL: @fdiv_fneg1_extra_use( ; CHECK-NEXT: [[Z:%.*]] = frem float 4.200000e+01, [[PZ:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[NEG]], [[Y:%.*]] ; CHECK-NEXT: call void @use(float [[DIV]]) -; CHECK-NEXT: [[R:%.*]] = fsub float [[Z]], [[TMP1]] +; CHECK-NEXT: [[R:%.*]] = fadd float [[Z]], [[DIV]] ; CHECK-NEXT: ret float [[R]] ; %z = frem float 42.0, %pz ; thwart complexity-based canonicalization @@ -185,10 +185,10 @@ ; CHECK-LABEL: @fdiv_fneg2_extra_use( ; CHECK-NEXT: [[Y:%.*]] = frem float -4.200000e+01, [[PY:%.*]] ; CHECK-NEXT: [[Z:%.*]] = frem float 4.200000e+01, [[PZ:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = fdiv float [[Y]], [[X:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[Y]], [[NEG]] ; CHECK-NEXT: call void @use(float [[DIV]]) -; CHECK-NEXT: [[R:%.*]] = fsub float [[Z]], [[TMP1]] +; CHECK-NEXT: [[R:%.*]] = fadd float [[Z]], [[DIV]] ; CHECK-NEXT: ret float [[R]] ; %y = frem float -42.0, %py ; thwart complexity-based canonicalization @@ -205,10 +205,10 @@ define <2 x float> @fmul_fneg1_extra_use(<2 x float> %x, <2 x float> %y, <2 x float> %pz) { ; CHECK-LABEL: @fmul_fneg1_extra_use( ; CHECK-NEXT: [[Z:%.*]] = frem <2 x float> , [[PZ:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub <2 x float> , [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub <2 x float> , [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x float> [[NEG]], [[Y:%.*]] ; CHECK-NEXT: call void @use_vec(<2 x float> [[MUL]]) -; CHECK-NEXT: [[R:%.*]] = fsub <2 x float> [[Z]], [[TMP1]] +; CHECK-NEXT: [[R:%.*]] = fadd <2 x float> [[Z]], [[MUL]] ; CHECK-NEXT: ret <2 x float> [[R]] ; %z = frem <2 x float> , %pz ; thwart complexity-based canonicalization @@ -225,10 +225,10 @@ ; CHECK-LABEL: @fmul_fneg2_extra_use( ; CHECK-NEXT: [[Y:%.*]] = frem float -4.200000e+01, [[PY:%.*]] ; CHECK-NEXT: [[Z:%.*]] = frem float 4.200000e+01, [[PZ:%.*]] -; CHECK-NEXT: [[TMP1:%.*]] = fmul float [[Y]], [[X:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul float [[Y]], [[NEG]] ; CHECK-NEXT: call void @use(float [[MUL]]) -; CHECK-NEXT: [[R:%.*]] = fsub float [[Z]], [[TMP1]] +; CHECK-NEXT: [[R:%.*]] = fadd float [[Z]], [[MUL]] ; CHECK-NEXT: ret float [[R]] ; %y = frem float -42.0, %py ; thwart complexity-based canonicalization Index: llvm/test/Transforms/InstCombine/fdiv.ll =================================================================== --- llvm/test/Transforms/InstCombine/fdiv.ll +++ llvm/test/Transforms/InstCombine/fdiv.ll @@ -501,8 +501,8 @@ define double @fdiv_fneg1(double %x, double %y) { ; CHECK-LABEL: @fdiv_fneg1( -; CHECK-NEXT: [[TMP1:%.*]] = fdiv double [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = fsub double -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub double -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = fdiv double [[NEG]], [[Y:%.*]] ; CHECK-NEXT: ret double [[DIV]] ; %neg = fsub double -0.0, %x @@ -512,8 +512,8 @@ define <2 x float> @fdiv_fneg2(<2 x float> %x, <2 x float> %y) { ; CHECK-LABEL: @fdiv_fneg2( -; CHECK-NEXT: [[TMP1:%.*]] = fdiv <2 x float> [[Y:%.*]], [[X:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = fsub <2 x float> , [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub <2 x float> , [[X:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = fdiv <2 x float> [[Y:%.*]], [[NEG]] ; CHECK-NEXT: ret <2 x float> [[DIV]] ; %neg = fsub <2 x float> , %x Index: llvm/test/Transforms/InstCombine/fmul.ll =================================================================== --- llvm/test/Transforms/InstCombine/fmul.ll +++ llvm/test/Transforms/InstCombine/fmul.ll @@ -280,8 +280,8 @@ ; (-0.0 - X) * Y => -0.0 - (X * Y) define float @neg_sink(float %x, float %y) { ; CHECK-LABEL: @neg_sink( -; CHECK-NEXT: [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[SUB:%.*]] = fsub float -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul float [[SUB]], [[Y:%.*]] ; CHECK-NEXT: ret float [[MUL]] ; %sub = fsub float -0.0, %x @@ -291,8 +291,8 @@ define float @unary_neg_sink(float %x, float %y) { ; CHECK-LABEL: @unary_neg_sink( -; CHECK-NEXT: [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fneg float [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul float [[NEG]], [[Y:%.*]] ; CHECK-NEXT: ret float [[MUL]] ; %neg = fneg float %x @@ -302,8 +302,8 @@ define <2 x float> @neg_sink_vec(<2 x float> %x, <2 x float> %y) { ; CHECK-LABEL: @neg_sink_vec( -; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub <2 x float> , [[TMP1]] +; CHECK-NEXT: [[SUB:%.*]] = fsub <2 x float> , [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x float> [[SUB]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[MUL]] ; %sub = fsub <2 x float> , %x @@ -314,8 +314,8 @@ ; FIXME: Should generate a unary FNeg. define <2 x float> @unary_neg_sink_vec(<2 x float> %x, <2 x float> %y) { ; CHECK-LABEL: @unary_neg_sink_vec( -; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub <2 x float> , [[TMP1]] +; CHECK-NEXT: [[SUB:%.*]] = fneg <2 x float> [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x float> [[SUB]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[MUL]] ; %sub = fneg <2 x float> %x @@ -325,8 +325,8 @@ define <2 x float> @neg_sink_vec_undef(<2 x float> %x, <2 x float> %y) { ; CHECK-LABEL: @neg_sink_vec_undef( -; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub <2 x float> , [[TMP1]] +; CHECK-NEXT: [[SUB:%.*]] = fsub <2 x float> , [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x float> [[SUB]], [[Y:%.*]] ; CHECK-NEXT: ret <2 x float> [[MUL]] ; %sub = fsub <2 x float> , %x @@ -337,8 +337,8 @@ ; (0.0 - X) * Y => 0.0 - (X * Y) define float @neg_sink_nsz(float %x, float %y) { ; CHECK-LABEL: @neg_sink_nsz( -; CHECK-NEXT: [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[SUB1:%.*]] = fsub nsz float -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul float [[SUB1]], [[Y:%.*]] ; CHECK-NEXT: ret float [[MUL]] ; %sub1 = fsub nsz float 0.0, %x Index: llvm/test/Transforms/InstCombine/fsub.ll =================================================================== --- llvm/test/Transforms/InstCombine/fsub.ll +++ llvm/test/Transforms/InstCombine/fsub.ll @@ -454,10 +454,10 @@ define float @fsub_fdiv_fneg1_extra_use(float %x, float %y, float %z) { ; CHECK-LABEL: @fsub_fdiv_fneg1_extra_use( -; CHECK-NEXT: [[TMP1:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[NEG]], [[Y:%.*]] ; CHECK-NEXT: call void @use(float [[DIV]]) -; CHECK-NEXT: [[R:%.*]] = fadd float [[TMP1]], [[Z:%.*]] +; CHECK-NEXT: [[R:%.*]] = fsub float [[Z:%.*]], [[DIV]] ; CHECK-NEXT: ret float [[R]] ; %neg = fsub float -0.000000e+00, %x @@ -469,10 +469,10 @@ define float @fsub_fdiv_fneg2_extra_use(float %x, float %y, float %z) { ; CHECK-LABEL: @fsub_fdiv_fneg2_extra_use( -; CHECK-NEXT: [[TMP1:%.*]] = fdiv float [[Y:%.*]], [[X:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[Y:%.*]], [[NEG]] ; CHECK-NEXT: call void @use(float [[DIV]]) -; CHECK-NEXT: [[R:%.*]] = fadd float [[TMP1]], [[Z:%.*]] +; CHECK-NEXT: [[R:%.*]] = fsub float [[Z:%.*]], [[DIV]] ; CHECK-NEXT: ret float [[R]] ; %neg = fsub float -0.000000e+00, %x @@ -486,10 +486,10 @@ define <2 x float> @fsub_fmul_fneg1_extra_use(<2 x float> %x, <2 x float> %y, <2 x float> %z) { ; CHECK-LABEL: @fsub_fmul_fneg1_extra_use( -; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub <2 x float> , [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub <2 x float> , [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x float> [[NEG]], [[Y:%.*]] ; CHECK-NEXT: call void @use_vec(<2 x float> [[MUL]]) -; CHECK-NEXT: [[R:%.*]] = fadd <2 x float> [[TMP1]], [[Z:%.*]] +; CHECK-NEXT: [[R:%.*]] = fsub <2 x float> [[Z:%.*]], [[MUL]] ; CHECK-NEXT: ret <2 x float> [[R]] ; %neg = fsub <2 x float> , %x @@ -501,10 +501,10 @@ define float @fsub_fmul_fneg2_extra_use(float %x, float %y, float %z) { ; CHECK-LABEL: @fsub_fmul_fneg2_extra_use( -; CHECK-NEXT: [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]] +; CHECK-NEXT: [[NEG:%.*]] = fsub float -0.000000e+00, [[X:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = fmul float [[NEG]], [[Y:%.*]] ; CHECK-NEXT: call void @use(float [[MUL]]) -; CHECK-NEXT: [[R:%.*]] = fadd float [[TMP1]], [[Z:%.*]] +; CHECK-NEXT: [[R:%.*]] = fsub float [[Z:%.*]], [[MUL]] ; CHECK-NEXT: ret float [[R]] ; %neg = fsub float -0.000000e+00, %x Index: llvm/test/Transforms/Reassociate/fast-basictest.ll =================================================================== --- llvm/test/Transforms/Reassociate/fast-basictest.ll +++ llvm/test/Transforms/Reassociate/fast-basictest.ll @@ -614,12 +614,25 @@ ret float %f } -; It is not safe to reassociate unary fneg without nnan. +; fneg of fneg is an identity operation, so no FMF are needed to remove those instructions. + +define float @test18_unary_fneg_no_FMF(float %a, float %b, float %z) { +; CHECK-LABEL: @test18_unary_fneg_no_FMF( +; CHECK-NEXT: [[TMP1:%.*]] = fmul float [[Z:%.*]], 4.000000e+01 +; CHECK-NEXT: [[F:%.*]] = fmul float [[TMP1]], [[A:%.*]] +; CHECK-NEXT: ret float [[F]] +; + %d = fmul float %z, 4.000000e+01 + %c = fneg float %d + %e = fmul float %a, %c + %f = fneg float %e + ret float %f +} + define float @test18_reassoc_unary_fneg(float %a, float %b, float %z) { ; CHECK-LABEL: @test18_reassoc_unary_fneg( -; CHECK-NEXT: [[C:%.*]] = fmul reassoc float [[Z:%.*]], -4.000000e+01 -; CHECK-NEXT: [[E:%.*]] = fmul reassoc float [[C]], [[A:%.*]] -; CHECK-NEXT: [[F:%.*]] = fneg reassoc float [[E]] +; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc float [[Z:%.*]], 4.000000e+01 +; CHECK-NEXT: [[F:%.*]] = fmul reassoc float [[TMP1]], [[A:%.*]] ; CHECK-NEXT: ret float [[F]] ; %d = fmul reassoc float %z, 4.000000e+01