diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -7036,7 +7036,8 @@ static SDValue IsNOT(SDValue V, SelectionDAG &DAG) { V = peekThroughBitcasts(V); if (V.getOpcode() == ISD::XOR && - ISD::isBuildVectorAllOnes(V.getOperand(1).getNode())) + (ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()) || + isAllOnesConstant(V.getOperand(1)))) return V.getOperand(0); if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR && (isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) { @@ -48146,16 +48147,31 @@ // Basic X = NOT(Y) detection. if (SDValue Not = IsNOT(V, DAG)) return Not; + SDValue BC = peekThroughOneUseBitcasts(V); // Fold BROADCAST(NOT(Y)) -> BROADCAST(Y). - if (V.getOpcode() == X86ISD::VBROADCAST) { - SDValue Src = V.getOperand(0); + if (BC.getOpcode() == X86ISD::VBROADCAST) { + SDValue Src = BC.getOperand(0); EVT SrcVT = Src.getValueType(); - if (!SrcVT.isVector()) + if (!SrcVT.isVector() && !(Src.hasOneUse() && BC.hasOneUse())) return SDValue(); if (SDValue Not = IsNOT(Src, DAG)) - return DAG.getNode(X86ISD::VBROADCAST, SDLoc(V), VT, + return DAG.getNode(X86ISD::VBROADCAST, SDLoc(BC), BC.getValueType(), DAG.getBitcast(SrcVT, Not)); } + // Fold PSHUFD(SCALAR_TO_VECTOR(NOT(Y))) -> PSHUFD(SCALAR_TO_VECTOR(Y)) + if (BC.getOpcode() == X86ISD::PSHUFD && BC.hasOneUse()) { + SDValue Src = BC.getOperand(0); + SDValue S2V = peekThroughOneUseBitcasts(Src); + if (S2V.getOpcode() != ISD::SCALAR_TO_VECTOR || !S2V.hasOneUse()) + return SDValue(); + if (SDValue Not = IsNOT(S2V.getOperand(0), DAG)) { + SDValue NotS2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(S2V), + S2V.getValueType(), Not); + SDValue NotSrc = DAG.getBitcast(Src.getValueType(), NotS2V); + return DAG.getNode(X86ISD::PSHUFD, SDLoc(BC), BC.getValueType(), NotSrc, + BC.getOperand(1)); + } + } return SDValue(); };