diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -2495,59 +2495,129 @@ return SDValue(ShiftNode, 0); } -/// Does this tree qualify as an attempt to move a bitfield into position, -/// essentially "(and (shl VAL, N), Mask)". -static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, - bool BiggerPattern, - SDValue &Src, int &ShiftAmount, - int &MaskWidth) { +// For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)" +static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, + bool BiggerPattern, + const uint64_t NonZeroBits, + SDValue &Src, int &DstLSB, + int &Width) { + assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed"); + EVT VT = Op.getValueType(); - unsigned BitWidth = VT.getSizeInBits(); - (void)BitWidth; - assert(BitWidth == 32 || BitWidth == 64); + assert((VT == MVT::i32 || VT == MVT::i64) && + "Caller guarantees VT is one of i32 or i64"); - KnownBits Known = CurDAG->computeKnownBits(Op); + SDNode *OpNode = Op.getNode(); - // Non-zero in the sense that they're not provably zero, which is the key - // point if we want to use this value - uint64_t NonZeroBits = (~Known.Zero).getZExtValue(); - - // Discard a constant AND mask if present. It's safe because the node will - // already have been factored into the computeKnownBits calculation above. uint64_t AndImm; - if (isOpcWithIntImmediate(Op.getNode(), ISD::AND, AndImm)) { - assert((~APInt(BitWidth, AndImm) & ~Known.Zero) == 0); - Op = Op.getOperand(0); - } + if (!isOpcWithIntImmediate(OpNode, ISD::AND, AndImm)) + return false; + + // If (~AndImm & NonZeroBits) is not zero at POS, we know that + // 1) (AndImm & (1 << POS) == 0) + // 2) the result of AND is not zero at POS bit (according to NonZeroBits) + // + // 1) and 2) don't agree so something must be wrong (e.g., in + // 'SelectionDAG::computeKnownBits') + assert((~AndImm & NonZeroBits) == 0 && + "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)"); + + SDValue AndOp0Val = OpNode->getOperand(0); + const SDNode *AndOp0 = AndOp0Val.getNode(); - // Don't match if the SHL has more than one use, since then we'll end up + uint64_t ShlImm; + if (!isOpcWithIntImmediate(AndOp0, ISD::SHL, ShlImm)) + return false; + + // Bail out if the SHL has more than one use, since then we'll end up // generating SHL+UBFIZ instead of just keeping SHL+AND. - if (!BiggerPattern && !Op.hasOneUse()) + if (!BiggerPattern && !AndOp0->hasOneUse()) + return false; + + DstLSB = countTrailingZeros(NonZeroBits); + Width = countTrailingOnes(NonZeroBits >> DstLSB); + + if (ShlImm != DstLSB && !BiggerPattern) return false; + Src = getLeftShift(CurDAG, AndOp0->getOperand(0), ShlImm - DstLSB); + return true; +} + +static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, + bool BiggerPattern, + const uint64_t NonZeroBits, + SDValue &Src, int &DstLSB, + int &Width) { + assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed"); + + SDNode *N = Op.getNode(); uint64_t ShlImm; - if (!isOpcWithIntImmediate(Op.getNode(), ISD::SHL, ShlImm)) + if (!isOpcWithIntImmediate(N, ISD::SHL, ShlImm)) return false; - Op = Op.getOperand(0); - if (!isShiftedMask_64(NonZeroBits)) + if (!BiggerPattern && !Op.hasOneUse()) return false; - ShiftAmount = countTrailingZeros(NonZeroBits); - MaskWidth = countTrailingOnes(NonZeroBits >> ShiftAmount); + EVT VT = N->getValueType(0); + assert((VT == MVT::i32 || VT == MVT::i64) && + "Caller guarantees that type is i32 or i64"); + + SDValue ShlOp0 = N->getOperand(0); - // BFI encompasses sufficiently many nodes that it's worth inserting an extra - // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL - // amount. BiggerPattern is true when this pattern is being matched for BFI, - // BiggerPattern is false when this pattern is being matched for UBFIZ, in - // which case it is not profitable to insert an extra shift. - if (ShlImm - ShiftAmount != 0 && !BiggerPattern) + DstLSB = countTrailingZeros(NonZeroBits); + Width = countTrailingOnes(NonZeroBits >> DstLSB); + Src = ShlOp0; + + if (DstLSB != ShlImm && !BiggerPattern) return false; - Src = getLeftShift(CurDAG, Op, ShlImm - ShiftAmount); + Src = getLeftShift(CurDAG, Src, ShlImm - DstLSB); return true; } +// We are looking for the following pattern which basically inserts Width +// continuous bits starting from the LSB of source value to the destination +// value, starting from DstLsb; all other bits of the destination value is set +// to zero. +// +// This is basically +// UBFIZ Dst, Src, DstLsb, Width, +// and equivalent to +// UBFM Dst, Src, (RegWidth - DstLsb) % RegWidth, Width - 1 +// +// To visualize, this is from _____xxxxx to ___xxxxx__ +static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, + bool BiggerPattern, SDValue &Src, + int &DstLSB, int &Width) { + EVT VT = Op.getValueType(); + unsigned BitWidth = VT.getSizeInBits(); + (void)BitWidth; + assert(BitWidth == 32 || BitWidth == 64); + + KnownBits Known = CurDAG->computeKnownBits(Op); + + // Non-zero in the sense that they're not provably zero, which is the key + // point if we want to use this value + const uint64_t NonZeroBits = (~Known.Zero).getZExtValue(); + if (!isShiftedMask_64(NonZeroBits)) + return false; + + const SDNode *N = Op.getNode(); + switch (N->getOpcode()) { + default: + break; + case ISD::AND: + return isBitfieldPositioningOpFromAnd(CurDAG, Op, BiggerPattern, + NonZeroBits, Src, DstLSB, Width); + case ISD::SHL: + return isBitfieldPositioningOpFromShl(CurDAG, Op, BiggerPattern, + NonZeroBits, Src, DstLSB, Width); + } + + return false; +} + static bool isShiftedMask(uint64_t Mask, EVT VT) { assert(VT == MVT::i32 || VT == MVT::i64); if (VT == MVT::i32)