Index: include/llvm/Target/TargetLowering.h =================================================================== --- include/llvm/Target/TargetLowering.h +++ include/llvm/Target/TargetLowering.h @@ -2851,6 +2851,11 @@ /// \returns MERGE_VALUEs of the scalar loads with their chains. SDValue scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const; + // Turn a store of a vector type into stores of the individual elements. + /// \param ST Store with a vector value type + /// \returns MERGE_VALUs of the individual store chains. + SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const; + //===--------------------------------------------------------------------===// // Instruction Emitting Hooks // Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -326,6 +326,12 @@ ST->getMemoryVT().isVector()) { EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); if (TLI.isTypeLegal(intVT)) { + if (!TLI.isOperationLegalOrCustom(ISD::STORE, intVT)) { + // Scalarize the store and let the individual components be handled. + SDValue Result = TLI.scalarizeVectorStore(ST, DAG); + DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); + return; + } // Expand to a bitconvert of the value to the integer type of the // same size, then a (misaligned) int store. // FIXME: Does not handle truncating floating point stores! Index: lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -636,54 +636,40 @@ } SDValue VectorLegalizer::ExpandStore(SDValue Op) { - SDLoc dl(Op); StoreSDNode *ST = cast(Op.getNode()); - SDValue Chain = ST->getChain(); - SDValue BasePTR = ST->getBasePtr(); - SDValue Value = ST->getValue(); - EVT StVT = ST->getMemoryVT(); - - unsigned Alignment = ST->getAlignment(); - bool isVolatile = ST->isVolatile(); - bool isNonTemporal = ST->isNonTemporal(); - AAMDNodes AAInfo = ST->getAAInfo(); - unsigned NumElem = StVT.getVectorNumElements(); - // The type of the data we want to save - EVT RegVT = Value.getValueType(); - EVT RegSclVT = RegVT.getScalarType(); - // The type of data as saved in memory. + EVT StVT = ST->getMemoryVT(); EVT MemSclVT = StVT.getScalarType(); - - // Cast floats into integers unsigned ScalarSize = MemSclVT.getSizeInBits(); // Round odd types to the next pow of two. - if (!isPowerOf2_32(ScalarSize)) - ScalarSize = NextPowerOf2(ScalarSize); - - // Store Stride in bytes - unsigned Stride = ScalarSize/8; - // Extract each of the elements from the original vector - // and save them into memory individually. - SmallVector Stores; - for (unsigned Idx = 0; Idx < NumElem; Idx++) { - SDValue Ex = DAG.getNode( - ISD::EXTRACT_VECTOR_ELT, dl, RegSclVT, Value, - DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); - - // This scalar TruncStore may be illegal, but we legalize it later. - SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR, - ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT, - isVolatile, isNonTemporal, MinAlign(Alignment, Idx*Stride), - AAInfo); - - BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, - DAG.getConstant(Stride, dl, BasePTR.getValueType())); - - Stores.push_back(Store); + if (!isPowerOf2_32(ScalarSize)) { + // FIXME: This is completely broken and inconsistent with ExpandLoad + // handling. + + // For sub-byte element sizes, this ends up with 0 stride between elements, + // so the same element just gets re-written to the same location. There seem + // to be tests explicitly testing for this broken behavior though. tests + // for this broken behavior. + + LLVMContext &Ctx = *DAG.getContext(); + + EVT NewMemVT + = EVT::getVectorVT(Ctx, + MemSclVT.getIntegerVT(Ctx, NextPowerOf2(ScalarSize)), + StVT.getVectorNumElements()); + + SDValue NewVectorStore + = DAG.getTruncStore(ST->getChain(), SDLoc(Op), ST->getValue(), + ST->getBasePtr(), + ST->getPointerInfo(), NewMemVT, + ST->isVolatile(), ST->isNonTemporal(), + ST->getAlignment(), + ST->getAAInfo()); + ST = cast(NewVectorStore.getNode()); } - SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); + + SDValue TF = TLI.scalarizeVectorStore(ST, DAG); AddLegalizedOperand(Op, TF); return TF; } Index: lib/CodeGen/SelectionDAG/TargetLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -3149,6 +3149,60 @@ return DAG.getMergeValues({ Value, NewChain }, SL); } +// FIXME: This relies on each element having a byte size, otherwise the stride +// is 0 and just overwrites the same location. ExpandStore currently expects +// this broken behavior. +SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST, + SelectionDAG &DAG) const { + SDLoc SL(ST); + + SDValue Chain = ST->getChain(); + SDValue BasePtr = ST->getBasePtr(); + SDValue Value = ST->getValue(); + EVT StVT = ST->getMemoryVT(); + + unsigned Alignment = ST->getAlignment(); + bool isVolatile = ST->isVolatile(); + bool isNonTemporal = ST->isNonTemporal(); + AAMDNodes AAInfo = ST->getAAInfo(); + + // The type of the data we want to save + EVT RegVT = Value.getValueType(); + EVT RegSclVT = RegVT.getScalarType(); + + // The type of data as saved in memory. + EVT MemSclVT = StVT.getScalarType(); + + EVT PtrVT = BasePtr.getValueType(); + + // Store Stride in bytes + unsigned Stride = MemSclVT.getSizeInBits() / 8; + EVT IdxVT = getVectorIdxTy(DAG.getDataLayout()); + unsigned NumElem = StVT.getVectorNumElements(); + + // Extract each of the elements from the original vector and save them into + // memory individually. + SmallVector Stores; + for (unsigned Idx = 0; Idx < NumElem; ++Idx) { + SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value, + DAG.getConstant(Idx, SL, IdxVT)); + + SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr, + DAG.getConstant(Idx * Stride, SL, PtrVT)); + + // This scalar TruncStore may be illegal, but we legalize it later. + SDValue Store = DAG.getTruncStore( + Chain, SL, Elt, Ptr, + ST->getPointerInfo().getWithOffset(Idx * Stride), MemSclVT, + isVolatile, isNonTemporal, MinAlign(Alignment, Idx * Stride), + AAInfo); + + Stores.push_back(Store); + } + + return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores); +} + //===----------------------------------------------------------------------===// // Implementation of Emulated TLS Model //===----------------------------------------------------------------------===//