Differential D79096 Diff 261671 llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp

Changeset View

Standalone View

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp

Show First 20 Lines • Show All 696 Lines • ▼ Show 20 Lines	void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);		Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
Results.push_back(Promoted);		Results.push_back(Promoted);
if (IsStrict)		if (IsStrict)
Results.push_back(Chain);		Results.push_back(Chain);
}		}

std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {		std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
LoadSDNode *LD = cast<LoadSDNode>(N);		LoadSDNode *LD = cast<LoadSDNode>(N);
		return TLI.scalarizeVectorLoad(LD, DAG);
EVT SrcVT = LD->getMemoryVT();
EVT SrcEltVT = SrcVT.getScalarType();
unsigned NumElem = SrcVT.getVectorNumElements();

SDValue NewChain;
SDValue Value;
if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.isByteSized()) {
SDLoc dl(N);

SmallVector<SDValue, 8> Vals;
SmallVector<SDValue, 8> LoadChains;

EVT DstEltVT = LD->getValueType(0).getScalarType();
SDValue Chain = LD->getChain();
SDValue BasePTR = LD->getBasePtr();
ISD::LoadExtType ExtType = LD->getExtensionType();

// When elements in a vector is not byte-addressable, we cannot directly
// load each element by advancing pointer, which could only address bytes.
// Instead, we load all significant words, mask bits off, and concatenate
// them to form each element. Finally, they are extended to destination
// scalar type to build the destination vector.
EVT WideVT = TLI.getPointerTy(DAG.getDataLayout());

assert(WideVT.isRound() &&
"Could not handle the sophisticated case when the widest integer is"
" not power of 2.");
assert(WideVT.bitsGE(SrcEltVT) &&
"Type is not legalized?");

unsigned WideBytes = WideVT.getStoreSize();
unsigned Offset = 0;
unsigned RemainingBytes = SrcVT.getStoreSize();
SmallVector<SDValue, 8> LoadVals;
while (RemainingBytes > 0) {
SDValue ScalarLoad;
unsigned LoadBytes = WideBytes;

if (RemainingBytes >= LoadBytes) {
ScalarLoad = DAG.getLoad(
WideVT, dl, Chain, BasePTR,
LD->getPointerInfo().getWithOffset(Offset), LD->getOriginalAlign(),
LD->getMemOperand()->getFlags(), LD->getAAInfo());
} else {
EVT LoadVT = WideVT;
while (RemainingBytes < LoadBytes) {
LoadBytes >>= 1; // Reduce the load size by half.
LoadVT = EVT::getIntegerVT(*DAG.getContext(), LoadBytes << 3);
}
ScalarLoad =
DAG.getExtLoad(ISD::EXTLOAD, dl, WideVT, Chain, BasePTR,
LD->getPointerInfo().getWithOffset(Offset), LoadVT,
LD->getOriginalAlign(),
LD->getMemOperand()->getFlags(), LD->getAAInfo());
}

RemainingBytes -= LoadBytes;
Offset += LoadBytes;

BasePTR = DAG.getObjectPtrOffset(dl, BasePTR, LoadBytes);

LoadVals.push_back(ScalarLoad.getValue(0));
LoadChains.push_back(ScalarLoad.getValue(1));
}

unsigned BitOffset = 0;
unsigned WideIdx = 0;
unsigned WideBits = WideVT.getSizeInBits();

// Extract bits, pack and extend/trunc them into destination type.
unsigned SrcEltBits = SrcEltVT.getSizeInBits();
SDValue SrcEltBitMask = DAG.getConstant(
APInt::getLowBitsSet(WideBits, SrcEltBits), dl, WideVT);

for (unsigned Idx = 0; Idx != NumElem; ++Idx) {
assert(BitOffset < WideBits && "Unexpected offset!");

SDValue ShAmt = DAG.getConstant(
BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
SDValue Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt);

BitOffset += SrcEltBits;
if (BitOffset >= WideBits) {
WideIdx++;
BitOffset -= WideBits;
if (BitOffset > 0) {
ShAmt = DAG.getConstant(
SrcEltBits - BitOffset, dl,
TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
SDValue Hi =
DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt);
Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi);
}
}

Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask);

switch (ExtType) {
default: llvm_unreachable("Unknown extended-load op!");
case ISD::EXTLOAD:
Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT);
break;
case ISD::ZEXTLOAD:
Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT);
break;
case ISD::SEXTLOAD:
ShAmt =
DAG.getConstant(WideBits - SrcEltBits, dl,
TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt);
Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt);
Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT);
break;
}
Vals.push_back(Lo);
}

NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
Value = DAG.getBuildVector(N->getValueType(0), dl, Vals);
} else {
std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
}

return std::make_pair(Value, NewChain);
}		}

SDValue VectorLegalizer::ExpandStore(SDNode *N) {		SDValue VectorLegalizer::ExpandStore(SDNode *N) {
StoreSDNode *ST = cast<StoreSDNode>(N);		StoreSDNode *ST = cast<StoreSDNode>(N);
SDValue TF = TLI.scalarizeVectorStore(ST, DAG);		SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
return TF;		return TF;
}		}

▲ Show 20 Lines • Show All 732 Lines • Show Last 20 Lines