Page MenuHomePhabricator
Differential D139509 Diff 480789 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

Changeset View

Standalone View

View Options

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 7,374 Lines • ▼ Show 20 Lines
// Check that (every element of) Z is undef or not an exact multiple of BW. // Check that (every element of) Z is undef or not an exact multiple of BW.
static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) { static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
return ISD::matchUnaryPredicate( return ISD::matchUnaryPredicate(
Z, Z,
[=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; }, [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
true); true);
} }
static SDValue expandVPFunnelShift(SDNode *Node, SelectionDAG &DAG) {
EVT VT = Node->getValueType(0);
SDValue ShX, ShY;
SDValue ShAmt, InvShAmt;
SDValue X = Node->getOperand(0);
SDValue Y = Node->getOperand(1);
SDValue Z = Node->getOperand(2);
SDValue Mask = Node->getOperand(3);
SDValue VL = Node->getOperand(4);
unsigned BW = VT.getScalarSizeInBits();
bool IsFSHL = Node->getOpcode() == ISD::VP_FSHL;
SDLoc DL(SDValue(Node, 0));
EVT ShVT = Z.getValueType();
if (isNonZeroModBitWidthOrUndef(Z, BW)) {
// fshl: X << C | Y >> (BW - C)
// fshr: X << (BW - C) | Y >> C
// where C = Z % BW is not zero
SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL);
InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitWidthC, ShAmt, Mask, VL);
ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt, Mask,
VL);
ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt, Mask,
VL);
} else {
// fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
// fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
SDValue BitMask = DAG.getConstant(BW - 1, DL, ShVT);
if (isPowerOf2_32(BW)) {
// Z % BW -> Z & (BW - 1)
ShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, Z, BitMask, Mask, VL);
// (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
SDValue NotZ = DAG.getNode(ISD::VP_XOR, DL, ShVT, Z,
DAG.getAllOnesConstant(DL, ShVT), Mask, VL);
InvShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, NotZ, BitMask, Mask, VL);
} else {
SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL);
InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitMask, ShAmt, Mask, VL);
}
SDValue One = DAG.getConstant(1, DL, ShVT);
if (IsFSHL) {
ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, ShAmt, Mask, VL);
SDValue ShY1 = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, One, Mask, VL);
ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, ShY1, InvShAmt, Mask, VL);
} else {
SDValue ShX1 = DAG.getNode(ISD::VP_SHL, DL, VT, X, One, Mask, VL);
ShX = DAG.getNode(ISD::VP_SHL, DL, VT, ShX1, InvShAmt, Mask, VL);
ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, ShAmt, Mask, VL);
}
}
return DAG.getNode(ISD::VP_OR, DL, VT, ShX, ShY, Mask, VL);
}
SDValue TargetLowering::expandFunnelShift(SDNode *Node, SDValue TargetLowering::expandFunnelShift(SDNode *Node,
SelectionDAG &DAG) const { SelectionDAG &DAG) const {
if (Node->isVPOpcode())
return expandVPFunnelShift(Node, DAG);
EVT VT = Node->getValueType(0); EVT VT = Node->getValueType(0);
if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) || if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
!isOperationLegalOrCustom(ISD::SRL, VT) || !isOperationLegalOrCustom(ISD::SRL, VT) ||
!isOperationLegalOrCustom(ISD::SUB, VT) || !isOperationLegalOrCustom(ISD::SUB, VT) ||
!isOperationLegalOrCustomOrPromote(ISD::OR, VT))) !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
return SDValue(); return SDValue();
▲ Show 20 Lines • Show All 2,843 Lines • Show Last 20 Lines