Index: lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -1595,6 +1595,13 @@ return Const != nullptr && Const->isOne(); } +/// If \p N is a ContantSDNode with isOpaque() == false return it casted to a +/// ContantSDNode pointer else nullptr. +static ConstantSDNode *asNonOpaqueConstant(SDValue N) { + ConstantSDNode *Const = dyn_cast(N); + return Const != nullptr && !Const->isOpaque() ? Const : nullptr; +} + SDValue DAGCombiner::visitADD(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); @@ -1618,8 +1625,8 @@ if (N1.getOpcode() == ISD::UNDEF) return N1; // fold (add c1, c2) -> c1+c2 - ConstantSDNode *N0C = dyn_cast(N0); - ConstantSDNode *N1C = dyn_cast(N1); + ConstantSDNode *N0C = asNonOpaqueConstant(N0); + ConstantSDNode *N1C = asNonOpaqueConstant(N1); if (N0C && N1C) return DAG.FoldConstantArithmetic(ISD::ADD, SDLoc(N), VT, N0C, N1C); // canonicalize constant to RHS @@ -1638,7 +1645,7 @@ (uint64_t)N1C->getSExtValue()); // fold ((c1-A)+c2) -> (c1+c2)-A if (N1C && N0.getOpcode() == ISD::SUB) - if (ConstantSDNode *N0C = dyn_cast(N0.getOperand(0))) { + if (ConstantSDNode *N0C = asNonOpaqueConstant(N0.getOperand(0))) { SDLoc DL(N); return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(N1C->getAPIntValue()+ @@ -1853,8 +1860,8 @@ if (N0 == N1) return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes); // fold (sub c1, c2) -> c1-c2 - ConstantSDNode *N0C = dyn_cast(N0.getNode()); - ConstantSDNode *N1C = dyn_cast(N1.getNode()); + ConstantSDNode *N0C = asNonOpaqueConstant(N0); + ConstantSDNode *N1C = asNonOpaqueConstant(N1); if (N0C && N1C) return DAG.FoldConstantArithmetic(ISD::SUB, SDLoc(N), VT, N0C, N1C); // fold (sub x, c) -> (add x, -c) @@ -1996,6 +2003,8 @@ bool N0IsConst = false; bool N1IsConst = false; + bool N1IsOpaqueConst = false; + bool N0IsOpaqueConst = false; APInt ConstValue0, ConstValue1; // fold vector ops if (VT.isVector()) { @@ -2006,15 +2015,19 @@ N1IsConst = isConstantSplatVector(N1.getNode(), ConstValue1); } else { N0IsConst = isa(N0); - if (N0IsConst) + if (N0IsConst) { ConstValue0 = cast(N0)->getAPIntValue(); + N0IsOpaqueConst = cast(N0)->isOpaque(); + } N1IsConst = isa(N1); - if (N1IsConst) + if (N1IsConst) { ConstValue1 = cast(N1)->getAPIntValue(); + N1IsOpaqueConst = cast(N1)->isOpaque(); + } } // fold (mul c1, c2) -> c1*c2 - if (N0IsConst && N1IsConst) + if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst) return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, N0.getNode(), N1.getNode()); @@ -2039,14 +2052,16 @@ DAG.getConstant(0, DL, VT), N0); } // fold (mul x, (1 << c)) -> x << c - if (N1IsConst && ConstValue1.isPowerOf2() && IsFullSplat) { + if (N1IsConst && !N1IsOpaqueConst && ConstValue1.isPowerOf2() && + IsFullSplat) { SDLoc DL(N); return DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ConstValue1.logBase2(), DL, getShiftAmountTy(N0.getValueType()))); } // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c - if (N1IsConst && (-ConstValue1).isPowerOf2() && IsFullSplat) { + if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2() && + IsFullSplat) { unsigned Log2Val = (-ConstValue1).logBase2(); SDLoc DL(N); // FIXME: If the input is something that is easily negated (e.g. a @@ -2124,7 +2139,7 @@ // fold (sdiv c1, c2) -> c1/c2 ConstantSDNode *N0C = isConstOrConstSplat(N0); ConstantSDNode *N1C = isConstOrConstSplat(N1); - if (N0C && N1C && !N1C->isNullValue()) + if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque()) return DAG.FoldConstantArithmetic(ISD::SDIV, SDLoc(N), VT, N0C, N1C); // fold (sdiv X, 1) -> X if (N1C && N1C->isOne()) @@ -2144,8 +2159,9 @@ } // fold (sdiv X, pow2) -> simple ops after legalize - if (N1C && !N1C->isNullValue() && (N1C->getAPIntValue().isPowerOf2() || - (-N1C->getAPIntValue()).isPowerOf2())) { + if (N1C && !N1C->isNullValue() && !N1C->isOpaque() && + (N1C->getAPIntValue().isPowerOf2() || + (-N1C->getAPIntValue()).isPowerOf2())) { // If dividing by powers of two is cheap, then don't perform the following // fold. if (TLI.isPow2SDivCheap()) @@ -2217,10 +2233,12 @@ // fold (udiv c1, c2) -> c1/c2 ConstantSDNode *N0C = isConstOrConstSplat(N0); ConstantSDNode *N1C = isConstOrConstSplat(N1); - if (N0C && N1C && !N1C->isNullValue()) - return DAG.FoldConstantArithmetic(ISD::UDIV, SDLoc(N), VT, N0C, N1C); + if (N0C && N1C) + if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, SDLoc(N), VT, + N0C, N1C)) + return Folded; // fold (udiv x, (1 << c)) -> x >>u c - if (N1C && N1C->getAPIntValue().isPowerOf2()) { + if (N1C && !N1C->isOpaque() && N1C->getAPIntValue().isPowerOf2()) { SDLoc DL(N); return DAG.getNode(ISD::SRL, DL, VT, N0, DAG.getConstant(N1C->getAPIntValue().logBase2(), DL, @@ -2228,7 +2246,7 @@ } // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2 if (N1.getOpcode() == ISD::SHL) { - if (ConstantSDNode *SHC = dyn_cast(N1.getOperand(0))) { + if (ConstantSDNode *SHC = asNonOpaqueConstant(N1.getOperand(0))) { if (SHC->getAPIntValue().isPowerOf2()) { EVT ADDVT = N1.getOperand(1).getValueType(); SDLoc DL(N); @@ -2266,8 +2284,10 @@ // fold (srem c1, c2) -> c1%c2 ConstantSDNode *N0C = isConstOrConstSplat(N0); ConstantSDNode *N1C = isConstOrConstSplat(N1); - if (N0C && N1C && !N1C->isNullValue()) - return DAG.FoldConstantArithmetic(ISD::SREM, SDLoc(N), VT, N0C, N1C); + if (N0C && N1C) + if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::SREM, SDLoc(N), VT, + N0C, N1C)) + return Folded; // If we know the sign bits of both operands are zero, strength reduce to a // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15 if (!VT.isVector()) { @@ -2308,17 +2328,20 @@ // fold (urem c1, c2) -> c1%c2 ConstantSDNode *N0C = isConstOrConstSplat(N0); ConstantSDNode *N1C = isConstOrConstSplat(N1); - if (N0C && N1C && !N1C->isNullValue()) - return DAG.FoldConstantArithmetic(ISD::UREM, SDLoc(N), VT, N0C, N1C); + if (N0C && N1C) + if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UREM, SDLoc(N), VT, + N0C, N1C)) + return Folded; // fold (urem x, pow2) -> (and x, pow2-1) - if (N1C && !N1C->isNullValue() && N1C->getAPIntValue().isPowerOf2()) { + if (N1C && !N1C->isNullValue() && !N1C->isOpaque() && + N1C->getAPIntValue().isPowerOf2()) { SDLoc DL(N); return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getConstant(N1C->getAPIntValue() - 1, DL, VT)); } // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1)) if (N1.getOpcode() == ISD::SHL) { - if (ConstantSDNode *SHC = dyn_cast(N1.getOperand(0))) { + if (ConstantSDNode *SHC = asNonOpaqueConstant(N1.getOperand(0))) { if (SHC->getAPIntValue().isPowerOf2()) { SDLoc DL(N); SDValue Add = @@ -2872,9 +2895,9 @@ } // fold (and c1, c2) -> c1&c2 - ConstantSDNode *N0C = dyn_cast(N0); + ConstantSDNode *N0C = asNonOpaqueConstant(N0); ConstantSDNode *N1C = dyn_cast(N1); - if (N0C && N1C) + if (N0C && N1C && !N1C->isOpaque()) return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C); // canonicalize constant to RHS if (isConstantIntBuildVectorOrConstantInt(N0) && @@ -3468,26 +3491,27 @@ } // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible. - if (N0.getOpcode() == ISD::AND && - N1.getOpcode() == ISD::AND && - N0.getOperand(1).getOpcode() == ISD::Constant && - N1.getOperand(1).getOpcode() == ISD::Constant && + if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND && // Don't increase # computations. (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) { // We can only do this xform if we know that bits from X that are set in C2 // but not in C1 are already zero. Likewise for Y. - const APInt &LHSMask = - cast(N0.getOperand(1))->getAPIntValue(); - const APInt &RHSMask = - cast(N1.getOperand(1))->getAPIntValue(); - - if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) && - DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) { - SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, - N0.getOperand(0), N1.getOperand(0)); - SDLoc DL(LocReference); - return DAG.getNode(ISD::AND, DL, VT, X, - DAG.getConstant(LHSMask | RHSMask, DL, VT)); + if (const ConstantSDNode *N0O1C = asNonOpaqueConstant(N0.getOperand(1))) { + if (const ConstantSDNode *N1O1C = asNonOpaqueConstant(N1.getOperand(1))) { + // We can only do this xform if we know that bits from X that are set in + // C2 but not in C1 are already zero. Likewise for Y. + const APInt &LHSMask = N0O1C->getAPIntValue(); + const APInt &RHSMask = N1O1C->getAPIntValue(); + + if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) && + DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) { + SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, + N0.getOperand(0), N1.getOperand(0)); + SDLoc DL(LocReference); + return DAG.getNode(ISD::AND, DL, VT, X, + DAG.getConstant(LHSMask | RHSMask, DL, VT)); + } + } } } @@ -3593,9 +3617,9 @@ } // fold (or c1, c2) -> c1|c2 - ConstantSDNode *N0C = dyn_cast(N0); + ConstantSDNode *N0C = asNonOpaqueConstant(N0); ConstantSDNode *N1C = dyn_cast(N1); - if (N0C && N1C) + if (N0C && N1C && !N1C->isOpaque()) return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C); // canonicalize constant to RHS if (isConstantIntBuildVectorOrConstantInt(N0) && @@ -3937,8 +3961,8 @@ if (N1.getOpcode() == ISD::UNDEF) return N1; // fold (xor c1, c2) -> c1^c2 - ConstantSDNode *N0C = dyn_cast(N0); - ConstantSDNode *N1C = dyn_cast(N1); + ConstantSDNode *N0C = asNonOpaqueConstant(N0); + ConstantSDNode *N1C = asNonOpaqueConstant(N1); if (N0C && N1C) return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C); // canonicalize constant to RHS @@ -4019,15 +4043,13 @@ } // fold (xor (xor x, c1), c2) -> (xor x, (xor c1, c2)) if (N1C && N0.getOpcode() == ISD::XOR) { - ConstantSDNode *N00C = dyn_cast(N0.getOperand(0)); - ConstantSDNode *N01C = dyn_cast(N0.getOperand(1)); - if (N00C) { + if (const ConstantSDNode *N00C = asNonOpaqueConstant(N0.getOperand(0))) { SDLoc DL(N); return DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), DAG.getConstant(N1C->getAPIntValue() ^ N00C->getAPIntValue(), DL, VT)); } - if (N01C) { + if (const ConstantSDNode *N01C = asNonOpaqueConstant(N0.getOperand(1))) { SDLoc DL(N); return DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), DAG.getConstant(N1C->getAPIntValue() ^ @@ -4080,10 +4102,6 @@ /// Handle transforms common to the three shifts, when the shift amount is a /// constant. SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) { - // We can't and shouldn't fold opaque constants. - if (Amt->isOpaque()) - return SDValue(); - SDNode *LHS = N->getOperand(0).getNode(); if (!LHS->hasOneUse()) return SDValue(); @@ -4110,8 +4128,8 @@ } // We require the RHS of the binop to be a constant and not opaque as well. - ConstantSDNode *BinOpCst = dyn_cast(LHS->getOperand(1)); - if (!BinOpCst || BinOpCst->isOpaque()) return SDValue(); + ConstantSDNode *BinOpCst = asNonOpaqueConstant(LHS->getOperand(1)); + if (!BinOpCst) return SDValue(); // FIXME: disable this unless the input to the binop is a shift by a constant. // If it is not a shift, it pessimizes some common cases like: @@ -4164,15 +4182,17 @@ SDValue N01 = N->getOperand(0).getOperand(1); if (ConstantSDNode *N01C = isConstOrConstSplat(N01)) { - EVT TruncVT = N->getValueType(0); - SDValue N00 = N->getOperand(0).getOperand(0); - APInt TruncC = N01C->getAPIntValue(); - TruncC = TruncC.trunc(TruncVT.getScalarSizeInBits()); - SDLoc DL(N); + if (!N01C->isOpaque()) { + EVT TruncVT = N->getValueType(0); + SDValue N00 = N->getOperand(0).getOperand(0); + APInt TruncC = N01C->getAPIntValue(); + TruncC = TruncC.trunc(TruncVT.getScalarSizeInBits()); + SDLoc DL(N); - return DAG.getNode(ISD::AND, DL, TruncVT, - DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00), - DAG.getConstant(TruncC, DL, TruncVT)); + return DAG.getNode(ISD::AND, DL, TruncVT, + DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00), + DAG.getConstant(TruncC, DL, TruncVT)); + } } } @@ -4226,8 +4246,8 @@ } // fold (shl c1, c2) -> c1<(N0); - if (N0C && N1C) + ConstantSDNode *N0C = asNonOpaqueConstant(N0); + if (N0C && N1C && !N1C->isOpaque()) return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C); // fold (shl 0, x) -> 0 if (isNullConstant(N0)) @@ -4372,7 +4392,7 @@ return DAG.getNode(ISD::ADD, SDLoc(N), VT, Shl0, Shl1); } - if (N1C) { + if (N1C && !N1C->isOpaque()) { SDValue NewSHL = visitShiftByConstant(N, N1C); if (NewSHL.getNode()) return NewSHL; @@ -4397,8 +4417,8 @@ } // fold (sra c1, c2) -> (sra c1, c2) - ConstantSDNode *N0C = dyn_cast(N0); - if (N0C && N1C) + ConstantSDNode *N0C = asNonOpaqueConstant(N0); + if (N0C && N1C && !N1C->isOpaque()) return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C); // fold (sra 0, x) -> 0 if (isNullConstant(N0)) @@ -4521,7 +4541,7 @@ if (DAG.SignBitIsZero(N0)) return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); - if (N1C) { + if (N1C && !N1C->isOpaque()) { SDValue NewSRA = visitShiftByConstant(N, N1C); if (NewSRA.getNode()) return NewSRA; @@ -4546,8 +4566,8 @@ } // fold (srl c1, c2) -> c1 >>u c2 - ConstantSDNode *N0C = dyn_cast(N0); - if (N0C && N1C) + ConstantSDNode *N0C = asNonOpaqueConstant(N0); + if (N0C && N1C && !N1C->isOpaque()) return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C); // fold (srl 0, x) -> 0 if (isNullConstant(N0)) @@ -4692,7 +4712,7 @@ if (N1C && SimplifyDemandedBits(SDValue(N, 0))) return SDValue(N, 0); - if (N1C) { + if (N1C && !N1C->isOpaque()) { SDValue NewSRL = visitShiftByConstant(N, N1C); if (NewSRL.getNode()) return NewSRL; @@ -6465,7 +6485,7 @@ // Only look at single-use SRLs. if (!V.getNode()->hasOneUse()) break; - if (ConstantSDNode *RHSC = dyn_cast(V.getOperand(1))) { + if (ConstantSDNode *RHSC = asNonOpaqueConstant(V.getOperand(1))) { // See if we can recursively simplify the LHS. unsigned Amt = RHSC->getZExtValue(); Index: lib/CodeGen/SelectionDAG/TargetLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -1086,9 +1086,19 @@ // If we know the value of all of the demanded bits, return this as a // constant. - if ((NewMask & (KnownZero|KnownOne)) == NewMask) + if ((NewMask & (KnownZero|KnownOne)) == NewMask) { + // Avoid folding to a constant if any OpaqueConstant is involved. + const SDNode *N = Op.getNode(); + for (SDNodeIterator I = SDNodeIterator::begin(N), + E = SDNodeIterator::end(N); I != E; ++I) { + SDNode *Op = *I; + if (ConstantSDNode *C = dyn_cast(Op)) + if (C->isOpaque()) + return false; + } return TLO.CombineTo(Op, TLO.DAG.getConstant(KnownOne, dl, Op.getValueType())); + } return false; }