diff --git a/llvm/include/llvm/CodeGen/DAGCombine.h b/llvm/include/llvm/CodeGen/DAGCombine.h --- a/llvm/include/llvm/CodeGen/DAGCombine.h +++ b/llvm/include/llvm/CodeGen/DAGCombine.h @@ -10,6 +10,8 @@ #ifndef LLVM_CODEGEN_DAGCOMBINE_H #define LLVM_CODEGEN_DAGCOMBINE_H +#include "llvm/CodeGen/SelectionDAGNodes.h" + namespace llvm { enum CombineLevel { @@ -19,6 +21,44 @@ AfterLegalizeDAG }; +/// Represents known origin of an individual byte in combine pattern. The +/// value of the byte is either constant zero, or comes from memory / +/// some other productive instruction (e.g. arithmetic instructions). +/// Bit manipulation instructions like shifts are not ByteProviders, rather +/// are used to extract Bytes. +struct ByteProvider { + // For constant zero providers Src is set to nullptr. For actual providers + // Src represents the node which originally produced the relevant bits. + SDNode *Src = nullptr; + // DestOffset is the byte we are trying to provider for + unsigned DestOffset = 0; + // SrcOffset is byteo ffset of the Src to provides the byte + unsigned SrcOffset = 0; + + ByteProvider() = default; + + static ByteProvider getSrc(SDNode *Load, unsigned ByteOffset, + unsigned VectorOffset) { + return ByteProvider(Load, ByteOffset, VectorOffset); + } + + static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0, 0); } + bool isConstantZero() const { return !Src; } + + bool hasSrc() const { return Src; } + + bool hasSameSrc(const ByteProvider &Other) const { return Other.Src == Src; } + + bool operator==(const ByteProvider &Other) const { + return Other.Src == Src && Other.DestOffset == DestOffset && + Other.SrcOffset == SrcOffset; + } + +private: + ByteProvider(SDNode *Src, unsigned DestOffset, unsigned SrcOffset) + : Src(Src), DestOffset(DestOffset), SrcOffset(SrcOffset) {} +}; + } // end llvm namespace #endif diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -7779,42 +7779,6 @@ return SDValue(); } -namespace { - -/// Represents known origin of an individual byte in load combine pattern. The -/// value of the byte is either constant zero or comes from memory. -struct ByteProvider { - // For constant zero providers Load is set to nullptr. For memory providers - // Load represents the node which loads the byte from memory. - // ByteOffset is the offset of the byte in the value produced by the load. - LoadSDNode *Load = nullptr; - unsigned ByteOffset = 0; - unsigned VectorOffset = 0; - - ByteProvider() = default; - - static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset, - unsigned VectorOffset) { - return ByteProvider(Load, ByteOffset, VectorOffset); - } - - static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0, 0); } - - bool isConstantZero() const { return !Load; } - bool isMemory() const { return Load; } - - bool operator==(const ByteProvider &Other) const { - return Other.Load == Load && Other.ByteOffset == ByteOffset && - Other.VectorOffset == VectorOffset; - } - -private: - ByteProvider(LoadSDNode *Load, unsigned ByteOffset, unsigned VectorOffset) - : Load(Load), ByteOffset(ByteOffset), VectorOffset(VectorOffset) {} -}; - -} // end anonymous namespace - /// Recursively traverses the expression calculating the origin of the requested /// byte of the given value. Returns None if the provider can't be calculated. /// @@ -7983,7 +7947,7 @@ : None; unsigned BPVectorIndex = VectorIndex.value_or(0U); - return ByteProvider::getMemory(L, Index, BPVectorIndex); + return ByteProvider::getSrc(L, Index, BPVectorIndex); } } @@ -8273,16 +8237,18 @@ unsigned ByteWidth = VT.getSizeInBits() / 8; bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian(); - auto MemoryByteOffset = [&] (ByteProvider P) { - assert(P.isMemory() && "Must be a memory byte provider"); - unsigned LoadBitWidth = P.Load->getMemoryVT().getScalarSizeInBits(); + auto MemoryByteOffset = [&](ByteProvider P) { + assert(P.hasSrc() && "Must be a memory byte provider"); + LoadSDNode *Load = cast(P.Src); + assert(Load); + + unsigned LoadBitWidth = Load->getMemoryVT().getScalarSizeInBits(); assert(LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"); unsigned LoadByteWidth = LoadBitWidth / 8; - return IsBigEndianTarget - ? bigEndianByteAt(LoadByteWidth, P.ByteOffset) - : littleEndianByteAt(LoadByteWidth, P.ByteOffset); + return IsBigEndianTarget ? bigEndianByteAt(LoadByteWidth, P.DestOffset) + : littleEndianByteAt(LoadByteWidth, P.DestOffset); }; Optional Base; @@ -8309,9 +8275,10 @@ return SDValue(); continue; } - assert(P->isMemory() && "provenance should either be memory or zero"); + assert(P->hasSrc() && "provenance should either be memory or zero"); - LoadSDNode *L = P->Load; + LoadSDNode *L = cast(P->Src); + assert(L); // All loads must share the same chain SDValue LChain = L->getChain(); @@ -8335,7 +8302,7 @@ unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits(); if (LoadWidthInBit % 8 != 0) return SDValue(); - unsigned ByteOffsetFromVector = P->VectorOffset * LoadWidthInBit / 8; + unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8; Ptr.addToOffset(ByteOffsetFromVector); } @@ -8392,7 +8359,8 @@ // So the combined value can be loaded from the first load address. if (MemoryByteOffset(*FirstByteProvider) != 0) return SDValue(); - LoadSDNode *FirstLoad = FirstByteProvider->Load; + LoadSDNode *FirstLoad = cast(FirstByteProvider->Src); + assert(FirstLoad); // The node we are looking at matches with the pattern, check if we can // replace it with a single (possibly zero-extended) load and bswap + shift if