Index: lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -509,10 +509,11 @@ bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN, EVT LoadResultTy, EVT &ExtVT); - /// Helper function to calculate whether the given Load can have its + /// Helper function to calculate whether the given Load/Store can have its /// width reduced to ExtVT. - bool isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType, - EVT &ExtVT, unsigned ShAmt = 0); + bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType, + EVT &MemVT, unsigned ShAmt = 0, + bool StrictInside = false); /// Used by BackwardsPropagateMask to find suitable loads. bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl &Loads, @@ -3960,64 +3961,78 @@ return true; } -bool DAGCombiner::isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType, - EVT &ExtVT, unsigned ShAmt) { - // Don't transform one with multiple uses, this would require adding a new - // load. - if (!SDValue(LoadN, 0).hasOneUse()) +bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST, + ISD::LoadExtType ExtType, EVT &MemVT, + unsigned ShAmt, bool StrictInside) { + if (!LDST) return false; - - if (LegalOperations && - !TLI.isLoadExtLegal(ExtType, LoadN->getValueType(0), ExtVT)) + // Only allow byte offsets. + if (ShAmt % 8) return false; // Do not generate loads of non-round integer types since these can // be expensive (and would be wrong if the type is not byte sized). - if (!ExtVT.isRound()) + if (!MemVT.isRound()) return false; // Don't change the width of a volatile load. - if (LoadN->isVolatile()) + if (LDST->isVolatile()) return false; // Verify that we are actually reducing a load width here. - if (LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits()) + if (LDST->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits()) return false; - // For the transform to be legal, the load must produce only two values - // (the value loaded and the chain). Don't transform a pre-increment - // load, for example, which produces an extra value. Otherwise the - // transformation is not equivalent, and the downstream logic to replace - // uses gets things wrong. - if (LoadN->getNumValues() > 2) + // Ensure that this isn't going to produce an unsupported unaligned access. + if (ShAmt && + !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT, + LDST->getAddressSpace(), ShAmt / 8)) return false; - // Only allow byte offsets. - if (ShAmt % 8) + // It's not possible to generate a constant of extended or untyped type. + EVT PtrType = LDST->getBasePtr().getValueType(); + if (PtrType == MVT::Untyped || PtrType.isExtended()) return false; - // Ensure that this isn't going to produce an unsupported unaligned access. - if (ShAmt && !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), - ExtVT, LoadN->getAddressSpace(), - ShAmt / 8)) - return false; + if (isa(LDST)) { + LoadSDNode *Load = cast(LDST); + // Don't transform one with multiple uses, this would require adding a new + // load. + if (!SDValue(Load, 0).hasOneUse()) + return false; + if (LegalOperations && + !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT)) + return false; - // If the load that we're shrinking is an extload and we're not just - // discarding the extension we can't simply shrink the load. Bail. - // TODO: It would be possible to merge the extensions in some cases. - if (LoadN->getExtensionType() != ISD::NON_EXTLOAD && - LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits() + ShAmt) - return false; + // For the transform to be legal, the load must produce only two values + // (the value loaded and the chain). Don't transform a pre-increment + // load, for example, which produces an extra value. Otherwise the + // transformation is not equivalent, and the downstream logic to replace + // uses gets things wrong. + if (Load->getNumValues() > 2) + return false; - if (!TLI.shouldReduceLoadWidth(LoadN, ExtType, ExtVT)) - return false; + // If the load that we're shrinking is an extload and we're not just + // discarding the extension we can't simply shrink the load. Bail. + // TODO: It would be possible to merge the extensions in some cases. + if ((StrictInside || Load->getExtensionType() != ISD::NON_EXTLOAD) && + Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt) + return false; - // It's not possible to generate a constant of extended or untyped type. - EVT PtrType = LoadN->getOperand(1).getValueType(); - if (PtrType == MVT::Untyped || PtrType.isExtended()) - return false; + if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT)) + return false; + } else { + assert(isa(LDST) && "It is not a Load nor a Store SDNode"); + StoreSDNode *Store = cast(LDST); + // Can't write outside the original store + if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt) + return false; + if (LegalOperations && + !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT)) + return false; + } return true; } @@ -4050,7 +4065,7 @@ auto *Load = cast(Op); EVT ExtVT; if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) && - isLegalNarrowLoad(Load, ISD::ZEXTLOAD, ExtVT)) { + isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) { // ZEXTLOAD is already small enough. if (Load->getExtensionType() == ISD::ZEXTLOAD && @@ -8787,7 +8802,7 @@ return SDValue(); LoadSDNode *LN0 = cast(N0); - if (!isLegalNarrowLoad(LN0, ExtType, ExtVT, ShAmt)) + if (!isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt)) return SDValue(); // For big endian targets, we need to adjust the offset to the pointer to