diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -14869,16 +14869,51 @@ if (OptLevel == CodeGenOpt::None) return false; + // Perform an early exit check. Do not bother looking at stored values that + // are not constants, loads, or extracted vector elements. + SDValue StoredVal = peekThroughBitcasts(St->getValue()); + bool IsLoadSrc = isa(StoredVal); + bool IsConstantSrc = isa(StoredVal) || + isa(StoredVal); + bool IsExtractVecSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT || + StoredVal.getOpcode() == ISD::EXTRACT_SUBVECTOR); + + if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecSrc) + return false; + + LLVMContext &Context = *DAG.getContext(); EVT MemVT = St->getMemoryVT(); - int64_t ElementSizeBytes = MemVT.getStoreSize(); unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1; + auto IsZero = [](const SDValue &Val) { + if (ConstantSDNode *C = dyn_cast(Val)) + return C->isNullValue(); + else if (ConstantFPSDNode *C = dyn_cast(Val)) + return C->getConstantFPValue()->isNullValue(); + return false; + }; - if (MemVT.getSizeInBits() * 2 > MaximumLegalStoreInBits) + // Construct VT for two merged stores and check if it is legal. + unsigned Size = MemVT.getSizeInBits() * 2; + auto GetMergedEVT = [&](EVT &Base) { + if (Base.isVector() || + IsZero(St->getValue()) || (IsConstantSrc &&TLI.storeOfVectorConstantIsCheap(Base, 2, St->getAddressSpace()))) + return EVT::getVectorVT(Context, Base.getScalarType(), NumMemElts * 2); + return EVT::getIntegerVT(Context, Size); + }; + EVT MergedTy = GetMergedEVT(MemVT); + if (!TLI.isTypeLegal(MergedTy)) { + if (TLI.getTypeAction(Context, MergedTy) == TargetLowering::TypePromoteInteger) + MergedTy =TLI.getTypeToTransformTo(Context, StoredVal.getValueType()); + else + return false; + } + if (!TLI.canMergeStoresTo(St->getAddressSpace(), MergedTy, DAG)) return false; bool NoVectors = DAG.getMachineFunction().getFunction().hasFnAttribute( Attribute::NoImplicitFloat); + int64_t ElementSizeBytes = MemVT.getStoreSize(); // This function cannot currently deal with non-byte-sized memory sizes. if (ElementSizeBytes * 8 != MemVT.getSizeInBits()) return false; @@ -14886,18 +14921,6 @@ if (!MemVT.isSimple()) return false; - // Perform an early exit check. Do not bother looking at stored values that - // are not constants, loads, or extracted vector elements. - SDValue StoredVal = peekThroughBitcasts(St->getValue()); - bool IsLoadSrc = isa(StoredVal); - bool IsConstantSrc = isa(StoredVal) || - isa(StoredVal); - bool IsExtractVecSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT || - StoredVal.getOpcode() == ISD::EXTRACT_SUBVECTOR); - - if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecSrc) - return false; - SmallVector StoreNodes; SDNode *RootNode; // Find potential store merge candidates by searching through chain sub-DAG @@ -14956,7 +14979,6 @@ } // The node with the lowest store address. - LLVMContext &Context = *DAG.getContext(); const DataLayout &DL = DAG.getDataLayout(); // Store the constants into memory as one consecutive store. @@ -14973,16 +14995,13 @@ for (unsigned i = 0; i < NumConsecutiveStores; ++i) { StoreSDNode *ST = cast(StoreNodes[i].MemNode); SDValue StoredVal = ST->getValue(); - bool IsElementZero = false; - if (ConstantSDNode *C = dyn_cast(StoredVal)) - IsElementZero = C->isNullValue(); - else if (ConstantFPSDNode *C = dyn_cast(StoredVal)) - IsElementZero = C->getConstantFPValue()->isNullValue(); + bool IsElementZero = IsZero(StoredVal); if (IsElementZero) { if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores) FirstZeroAfterNonZero = i; } - NonZero |= !IsElementZero; + + NonZero |= !IsElementZero; // Find a legal type for the constant store. unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;