Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -36784,16 +36784,44 @@ /// Returns the negated value if the node \p N flips sign of FP value. /// -/// FP-negation node may have different forms: FNEG(x) or FXOR (x, 0x80000000). +/// FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000) +/// or FSUB(0, x) /// AVX512F does not have FXOR, so FNEG is lowered as /// (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))). /// In this case we go though all bitcasts. -static SDValue isFNEG(SDNode *N) { +/// This also recognizes splat of a negated value and returns the splat of that +/// value. +static SDValue isFNEG(SelectionDAG &DAG, SDNode *N) { if (N->getOpcode() == ISD::FNEG) return N->getOperand(0); SDValue Op = peekThroughBitcasts(SDValue(N, 0)); - if (Op.getOpcode() != X86ISD::FXOR && Op.getOpcode() != ISD::XOR) + auto VT = Op->getValueType(0); + if (auto SVOp = dyn_cast(Op.getNode())) { + // For a VECTOR_SHUFFLE(VEC1, VEC2), if the VEC2 is undef, then the negate + // of this is VECTOR_SHUFFLE(-VEC1, UNDEF). The mask can be anything here. + if (!SVOp->getOperand(1).isUndef()) + return SDValue(); + if (SDValue NegOp0 = isFNEG(DAG, SVOp->getOperand(0).getNode())) + return DAG.getVectorShuffle(VT, SDLoc(SVOp), NegOp0, DAG.getUNDEF(VT), + SVOp->getMask()); + return SDValue(); + } + auto Opc = Op.getOpcode(); + if (Opc == ISD::INSERT_VECTOR_ELT) { + // Negate of INSERT_VECTOR_ELT(UNDEF, V, INDEX) is INSERT_VECTOR_ELT(UNDEF, + // -V, INDEX). + SDValue InsVector = Op.getOperand(0); + SDValue InsVal = Op.getOperand(1); + if (!InsVector.isUndef()) + return SDValue(); + if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode())) + return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector, + NegInsVal, Op.getOperand(2)); + return SDValue(); + } + + if (Opc != X86ISD::FXOR && Opc != ISD::XOR && Opc != ISD::FSUB) return SDValue(); SDValue Op1 = peekThroughBitcasts(Op.getOperand(1)); @@ -36813,34 +36841,46 @@ // - BUILD_VECTOR node // - load from a constant pool. // We check all variants here. - if (Op1.getOpcode() == X86ISD::VBROADCAST) { - if (auto *C = getTargetConstantFromNode(Op1.getOperand(0))) - if (isSignMask(cast(C))) - return Op0; + auto IsNeg = [=](const ConstantFP *Val) { + return (isSignMask(Val) && Opc != ISD::FSUB) || + (Val->isZero() && Opc == ISD::FSUB); + }; - } else if (BuildVectorSDNode *BV = dyn_cast(Op1)) { - if (ConstantFPSDNode *CN = BV->getConstantFPSplatNode()) - if (isSignMask(CN->getConstantFPValue())) - return Op0; + auto Negate = [=](SDValue Op0, SDValue Op1) { + if (Op1.getOpcode() == X86ISD::VBROADCAST) { + if (auto *C = cast_or_null( + getTargetConstantFromNode(Op1.getOperand(0)))) + if (IsNeg(C)) + return Op0; - } else if (auto *C = getTargetConstantFromNode(Op1)) { - if (C->getType()->isVectorTy()) { - if (auto *SplatV = C->getSplatValue()) - if (isSignMask(cast(SplatV))) + } else if (auto *BV = dyn_cast(Op1)) { + if (auto *CN = BV->getConstantFPSplatNode()) + if (IsNeg(CN->getConstantFPValue())) return Op0; - } else if (auto *FPConst = dyn_cast(C)) - if (isSignMask(FPConst)) - return Op0; - } - return SDValue(); + + } else if (const Constant *C = getTargetConstantFromNode(Op1)) { + if (C->getType()->isVectorTy()) { + if (auto *SplatV = cast_or_null(C->getSplatValue())) + if (IsNeg(SplatV)) + return Op0; + } else if (auto *FPConst = dyn_cast(C)) + if (IsNeg(FPConst)) + return Op0; + } + return SDValue(); + }; + if (Opc == ISD::FSUB) + std::swap(Op0, Op1); + return Negate(Op0, Op1); } /// Do target-specific dag combines on floating point negations. static SDValue combineFneg(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) { EVT OrigVT = N->getValueType(0); - SDValue Arg = isFNEG(N); - assert(Arg.getNode() && "N is expected to be an FNEG node"); + SDValue Arg = isFNEG(DAG, N); + if (!Arg) + return SDValue(); EVT VT = Arg.getValueType(); EVT SVT = VT.getScalarType(); @@ -36955,9 +36995,7 @@ if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget)) return FPLogic; - if (isFNEG(N)) - return combineFneg(N, DAG, Subtarget); - return SDValue(); + return combineFneg(N, DAG, Subtarget); } static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG, @@ -37090,9 +37128,8 @@ if (isNullFPScalarOrVectorConst(N->getOperand(1))) return N->getOperand(0); - if (isFNEG(N)) - if (SDValue NewVal = combineFneg(N, DAG, Subtarget)) - return NewVal; + if (SDValue NewVal = combineFneg(N, DAG, Subtarget)) + return NewVal; return lowerX86FPLogicOp(N, DAG, Subtarget); } @@ -37779,7 +37816,7 @@ SDValue C = N->getOperand(2); auto invertIfNegative = [&DAG](SDValue &V) { - if (SDValue NegVal = isFNEG(V.getNode())) { + if (SDValue NegVal = isFNEG(DAG, V.getNode())) { V = DAG.getBitcast(V.getValueType(), NegVal); return true; } @@ -37788,7 +37825,7 @@ if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT && isa(V.getOperand(1)) && cast(V.getOperand(1))->getZExtValue() == 0) { - if (SDValue NegVal = isFNEG(V.getOperand(0).getNode())) { + if (SDValue NegVal = isFNEG(DAG, V.getOperand(0).getNode())) { NegVal = DAG.getBitcast(V.getOperand(0).getValueType(), NegVal); V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(), NegVal, V.getOperand(1)); @@ -37821,7 +37858,7 @@ SDLoc dl(N); EVT VT = N->getValueType(0); - SDValue NegVal = isFNEG(N->getOperand(2).getNode()); + SDValue NegVal = isFNEG(DAG, N->getOperand(2).getNode()); if (!NegVal) return SDValue(); Index: test/CodeGen/X86/avx2-fma-fneg-combine.ll =================================================================== --- test/CodeGen/X86/avx2-fma-fneg-combine.ll +++ test/CodeGen/X86/avx2-fma-fneg-combine.ll @@ -118,20 +118,14 @@ define <8 x float> @test7(float %a, <8 x float> %b, <8 x float> %c) { ; X32-LABEL: test7: ; X32: # %bb.0: # %entry -; X32-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero -; X32-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero -; X32-NEXT: vsubps %ymm2, %ymm3, %ymm2 -; X32-NEXT: vbroadcastss %xmm2, %ymm2 -; X32-NEXT: vfmadd213ps {{.*#+}} ymm0 = (ymm2 * ymm0) + ymm1 +; X32-NEXT: vbroadcastss {{[0-9]+}}(%esp), %ymm2 +; X32-NEXT: vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm1 ; X32-NEXT: retl ; ; X64-LABEL: test7: ; X64: # %bb.0: # %entry -; X64-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; X64-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero -; X64-NEXT: vsubps %ymm0, %ymm3, %ymm0 ; X64-NEXT: vbroadcastss %xmm0, %ymm0 -; X64-NEXT: vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 +; X64-NEXT: vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 ; X64-NEXT: retq entry: %0 = insertelement <8 x float> undef, float %a, i32 0 @@ -145,19 +139,14 @@ define <8 x float> @test8(float %a, <8 x float> %b, <8 x float> %c) { ; X32-LABEL: test8: ; X32: # %bb.0: # %entry -; X32-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero -; X32-NEXT: vbroadcastss {{.*#+}} xmm3 = [-0,-0,-0,-0] -; X32-NEXT: vxorps %xmm3, %xmm2, %xmm2 -; X32-NEXT: vbroadcastss %xmm2, %ymm2 -; X32-NEXT: vfmadd213ps {{.*#+}} ymm0 = (ymm2 * ymm0) + ymm1 +; X32-NEXT: vbroadcastss {{[0-9]+}}(%esp), %ymm2 +; X32-NEXT: vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm1 ; X32-NEXT: retl ; ; X64-LABEL: test8: ; X64: # %bb.0: # %entry -; X64-NEXT: vbroadcastss {{.*#+}} xmm3 = [-0,-0,-0,-0] -; X64-NEXT: vxorps %xmm3, %xmm0, %xmm0 ; X64-NEXT: vbroadcastss %xmm0, %ymm0 -; X64-NEXT: vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 +; X64-NEXT: vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 ; X64-NEXT: retq entry: %0 = fsub float -0.0, %a