Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -36696,16 +36696,45 @@ /// Returns the negated value if the node \p N flips sign of FP value. /// -/// FP-negation node may have different forms: FNEG(x) or FXOR (x, 0x80000000). +/// FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000) +/// or FSUB(0, x) /// AVX512F does not have FXOR, so FNEG is lowered as /// (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))). /// In this case we go though all bitcasts. -static SDValue isFNEG(SDNode *N) { +/// This also recognizes splat of a negated value and returns the splat of that +/// value. +static SDValue isFNEG(SelectionDAG &DAG, SDNode *N) { if (N->getOpcode() == ISD::FNEG) return N->getOperand(0); SDValue Op = peekThroughBitcasts(SDValue(N, 0)); - if (Op.getOpcode() != X86ISD::FXOR && Op.getOpcode() != ISD::XOR) + auto VT = Op->getValueType(0); + ShuffleVectorSDNode *SVOp = dyn_cast(Op.getNode()); + if (SVOp) { + // For a VECTOR_SHUFFLE(VEC1, VEC2), if the VEC2 is undef, then the negate + // of this is VECTOR_SHUFFLE(-VEC1, UNDEF). The mask can be anything here. + if (!SVOp->getOperand(1).isUndef()) + return SDValue(); + if (SDValue NegOp0 = isFNEG(DAG, SVOp->getOperand(0).getNode())) + return DAG.getVectorShuffle(VT, SDLoc(SVOp), NegOp0, DAG.getUNDEF(VT), + SVOp->getMask()); + return SDValue(); + } + auto Opc = Op.getOpcode(); + if (Opc == ISD::INSERT_VECTOR_ELT) { + // Negate of INSERT_VECTOR_ELT(UNDEF, V, INDEX) is INSERT_VECTOR_ELT(UNDEF, + // -V, INDEX). + SDValue InsVector = Op.getOperand(0); + SDValue InsVal = Op.getOperand(1); + if (!InsVector.isUndef()) + return SDValue(); + if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode())) + return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector, + NegInsVal, Op.getOperand(2)); + return SDValue(); + } + + if (Opc != X86ISD::FXOR && Opc != ISD::XOR && Opc != ISD::FSUB) return SDValue(); SDValue Op1 = peekThroughBitcasts(Op.getOperand(1)); @@ -36725,33 +36754,45 @@ // - BUILD_VECTOR node // - load from a constant pool. // We check all variants here. - if (Op1.getOpcode() == X86ISD::VBROADCAST) { - if (auto *C = getTargetConstantFromNode(Op1.getOperand(0))) - if (isSignMask(cast(C))) - return Op0; + auto IsNeg = [=](const ConstantFP *Val) { + return (isSignMask(Val) && Opc != ISD::FSUB) || + (Val->isZero() && Opc == ISD::FSUB); + }; - } else if (BuildVectorSDNode *BV = dyn_cast(Op1)) { - if (ConstantFPSDNode *CN = BV->getConstantFPSplatNode()) - if (isSignMask(CN->getConstantFPValue())) - return Op0; + auto Negate = [=](SDValue Op0, SDValue Op1) { + if (Op1.getOpcode() == X86ISD::VBROADCAST) { + if (auto *C = cast_or_null( + getTargetConstantFromNode(Op1.getOperand(0)))) + if (IsNeg(C)) + return Op0; - } else if (auto *C = getTargetConstantFromNode(Op1)) { - if (C->getType()->isVectorTy()) { - if (auto *SplatV = C->getSplatValue()) - if (isSignMask(cast(SplatV))) + } else if (BuildVectorSDNode *BV = dyn_cast(Op1)) { + if (ConstantFPSDNode *CN = BV->getConstantFPSplatNode()) + if (IsNeg(CN->getConstantFPValue())) return Op0; - } else if (auto *FPConst = dyn_cast(C)) - if (isSignMask(FPConst)) - return Op0; - } - return SDValue(); + + } else if (auto *C = getTargetConstantFromNode(Op1)) { + if (C->getType()->isVectorTy()) { + if (auto *SplatV = cast_or_null(C->getSplatValue())) + if (IsNeg(SplatV)) + return Op0; + } else if (auto *FPConst = dyn_cast(C)) + if (IsNeg(FPConst)) + return Op0; + } + return SDValue(); + }; + if (Opc == ISD::FSUB) + return Negate(Op1, Op0); + else + return Negate(Op0, Op1); } /// Do target-specific dag combines on floating point negations. static SDValue combineFneg(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) { EVT OrigVT = N->getValueType(0); - SDValue Arg = isFNEG(N); + SDValue Arg = isFNEG(DAG, N); assert(Arg.getNode() && "N is expected to be an FNEG node"); EVT VT = Arg.getValueType(); @@ -36867,7 +36908,7 @@ if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget)) return FPLogic; - if (isFNEG(N)) + if (isFNEG(DAG, N)) return combineFneg(N, DAG, Subtarget); return SDValue(); } @@ -37002,7 +37043,7 @@ if (isNullFPScalarOrVectorConst(N->getOperand(1))) return N->getOperand(0); - if (isFNEG(N)) + if (isFNEG(DAG, N)) if (SDValue NewVal = combineFneg(N, DAG, Subtarget)) return NewVal; @@ -37659,8 +37700,8 @@ SDValue B = N->getOperand(1); SDValue C = N->getOperand(2); - auto invertIfNegative = [](SDValue &V) { - if (SDValue NegVal = isFNEG(V.getNode())) { + auto invertIfNegative = [&DAG](SDValue &V) { + if (SDValue NegVal = isFNEG(DAG, V.getNode())) { V = NegVal; return true; } @@ -37757,7 +37798,7 @@ SDLoc dl(N); EVT VT = N->getValueType(0); - SDValue NegVal = isFNEG(N->getOperand(2).getNode()); + SDValue NegVal = isFNEG(DAG, N->getOperand(2).getNode()); if (!NegVal) return SDValue(); Index: test/CodeGen/X86/avx2-fma-fneg-combine.ll =================================================================== --- test/CodeGen/X86/avx2-fma-fneg-combine.ll +++ test/CodeGen/X86/avx2-fma-fneg-combine.ll @@ -118,20 +118,14 @@ define <8 x float> @test7(float %a, <8 x float> %b, <8 x float> %c) { ; X32-LABEL: test7: ; X32: # %bb.0: # %entry -; X32-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero -; X32-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero -; X32-NEXT: vsubps %ymm2, %ymm3, %ymm2 -; X32-NEXT: vbroadcastss %xmm2, %ymm2 -; X32-NEXT: vfmadd213ps {{.*#+}} ymm0 = (ymm2 * ymm0) + ymm1 +; X32-NEXT: vbroadcastss {{[0-9]+}}(%esp), %ymm2 +; X32-NEXT: vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm1 ; X32-NEXT: retl ; ; X64-LABEL: test7: ; X64: # %bb.0: # %entry -; X64-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; X64-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero -; X64-NEXT: vsubps %ymm0, %ymm3, %ymm0 ; X64-NEXT: vbroadcastss %xmm0, %ymm0 -; X64-NEXT: vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 +; X64-NEXT: vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 ; X64-NEXT: retq entry: %0 = insertelement <8 x float> undef, float %a, i32 0 @@ -142,4 +136,24 @@ } +define <8 x float> @test8(float %a, <8 x float> %b, <8 x float> %c) { +; X32-LABEL: test8: +; X32: # %bb.0: # %entry +; X32-NEXT: vbroadcastss {{[0-9]+}}(%esp), %ymm2 +; X32-NEXT: vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm1 +; X32-NEXT: retl +; +; X64-LABEL: test8: +; X64: # %bb.0: # %entry +; X64-NEXT: vbroadcastss %xmm0, %ymm0 +; X64-NEXT: vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 +; X64-NEXT: retq +entry: + %0 = fsub float -0.0, %a + %1 = insertelement <8 x float> undef, float %0, i32 0 + %2 = shufflevector <8 x float> %1, <8 x float> undef, <8 x i32> zeroinitializer + %3 = tail call <8 x float> @llvm.fma.v8f32(<8 x float> %2, <8 x float> %b, <8 x float> %c) + ret <8 x float> %3 +} + declare <8 x float> @llvm.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)