Index: llvm/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -400,15 +400,125 @@ /// The operation that this node performs. int16_t NodeType; - /// This tracks whether this node has one or more dbg_value - /// nodes corresponding to it. - uint16_t HasDebugValue : 1; - protected: - /// This member is defined by this class, but is not used for - /// anything. Subclasses can use it to hold whatever state they find useful. - /// This field is initialized to zero by the ctor. - uint16_t SubclassData : 15; + class SDNodeBitfields { + friend class SDNode; + friend class MemIntrinsicSDNode; + + bool hasDebugValue() const { return _HasDebugValue; } + void setHasDebugValue(bool V) { _HasDebugValue = V; } + bool isMemIntrinsic() const { return _IsMemIntrinsic; } + void setIsMemIntrinsic(bool V) { _IsMemIntrinsic = V; } + + // Prefixed with "_" so you're not tempted to use these directly. Use the + // accessors above. + uint16_t _HasDebugValue : 1; + uint16_t _IsMemIntrinsic : 1; + }; + enum { NumSDNodeBits = 2 }; + + class ConstantSDNodeBitfields { + friend class ConstantSDNode; + + uint16_t : NumSDNodeBits; + + bool isOpaque() const { return _IsOpaque; } + void setIsOpaque(bool V) { _IsOpaque = V; } + + uint16_t _IsOpaque : 1; + }; + + class MemSDNodeBitfields { + friend class MemSDNode; + friend class MemIntrinsicSDNode; + friend class AtomicSDNode; + + uint16_t : NumSDNodeBits; + + bool isVolatile() const { return _IsVolatile; } + void setIsVolatile(bool V) { _IsVolatile = V; } + bool isNonTemporal() const { return _IsNonTemporal; } + void setIsNonTemporal(bool V) { _IsNonTemporal = V; } + bool isInvariant() const { return _IsInvariant; } + void setIsInvariant(bool V) { _IsInvariant = V; } + + SynchronizationScope synchScope() const { + return static_cast(_SynchScope); + } + void setSynchScope(SynchronizationScope V) { + _SynchScope = static_cast(V); + assert(_SynchScope == V && "Value truncated"); + } + + AtomicOrdering ordering() const { + return static_cast(_Ordering); + } + void setOrdering(AtomicOrdering V) { + _Ordering = static_cast(V); + assert(_Ordering == static_cast(V) && "Value truncated"); + } + + uint16_t _IsVolatile : 1; + uint16_t _IsNonTemporal : 1; + uint16_t _IsInvariant : 1; + uint16_t _SynchScope : 1; // enum SynchronizationScope + uint16_t _Ordering : 4; // enum AtomicOrdering + }; + enum { NumMemSDNodeBits = NumSDNodeBits + 8 }; + + class LSBaseSDNodeBitfields { + friend class LSBaseSDNode; + uint16_t : NumMemSDNodeBits; + + ISD::MemIndexedMode addressingMode() const { + return static_cast(_AddressingMode); + } + void setAddressingMode(ISD::MemIndexedMode V) { + _AddressingMode = V; + assert(_AddressingMode == V && "Value truncated"); + } + + uint16_t _AddressingMode : 3; // enum ISD::MemIndexedMode + }; + enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; + + class LoadSDNodeBitfields { + friend class LoadSDNode; + friend class MaskedLoadSDNode; + + uint16_t : NumLSBaseSDNodeBits; + + ISD::LoadExtType extTy() const { + return static_cast(_ExtTy); + } + void setExtTy(ISD::LoadExtType V) { + _ExtTy = static_cast(V); + assert(_ExtTy == V && "Value truncated"); + } + + uint16_t _ExtTy : 2; // enum ISD::LoadExtType + }; + + class StoreSDNodeBitfields { + friend class StoreSDNode; + friend class MaskedStoreSDNode; + + uint16_t : NumLSBaseSDNodeBits; + + bool isTruncating() const { return _IsTruncating; } + void setIsTruncating(bool V) { _IsTruncating = V; } + + uint16_t _IsTruncating : 1; + }; + + union { + SDNodeBitfields SDNodeBits; + ConstantSDNodeBitfields ConstantSDNodeBits; + MemSDNodeBitfields MemSDNodeBits; + LSBaseSDNodeBitfields LSBaseSDNodeBits; + LoadSDNodeBitfields LoadSDNodeBits; + StoreSDNodeBitfields StoreSDNodeBits; + }; private: /// Unique id per SDNode in the DAG. @@ -481,7 +591,8 @@ /// proper classof relationship. bool isMemIntrinsic() const { return (NodeType == ISD::INTRINSIC_W_CHAIN || - NodeType == ISD::INTRINSIC_VOID) && ((SubclassData >> 13) & 1); + NodeType == ISD::INTRINSIC_VOID) && + SDNodeBits.isMemIntrinsic(); } /// Test if this node has a post-isel opcode, directly @@ -496,11 +607,8 @@ return ~NodeType; } - /// Get this bit. - bool getHasDebugValue() const { return HasDebugValue; } - - /// Set this bit. - void setHasDebugValue(bool b) { HasDebugValue = b; } + bool getHasDebugValue() const { return SDNodeBits.hasDebugValue(); } + void setHasDebugValue(bool b) { SDNodeBits.setHasDebugValue(b); } /// Return true if there are no uses of this node. bool use_empty() const { return UseList == nullptr; } @@ -815,10 +923,10 @@ /// SDNodes are created without any operands, and never own the operand /// storage. To add operands, see SelectionDAG::createOperands. SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) - : NodeType(Opc), HasDebugValue(false), SubclassData(0), NodeId(-1), - OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr), - NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order), + : NodeType(Opc), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), + UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) { + memset(&SDNodeBits, 0, sizeof(SDNodeBits)); assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); assert(NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"); @@ -1036,20 +1144,22 @@ /// encoding of the volatile flag, as well as bits used by subclasses. This /// function should only be used to compute a FoldingSetNodeID value. unsigned getRawSubclassData() const { - return SubclassData; + uint16_t Data; + memcpy(&Data, &SDNodeBits, sizeof(SDNodeBits)); + static_assert(sizeof(SDNodeBits) == sizeof(uint16_t), + "SDNodeBits field too small?"); + return Data; } // We access subclass data here so that we can check consistency // with MachineMemOperand information. - bool isVolatile() const { return (SubclassData >> 5) & 1; } - bool isNonTemporal() const { return (SubclassData >> 6) & 1; } - bool isInvariant() const { return (SubclassData >> 7) & 1; } + bool isVolatile() const { return MemSDNodeBits.isVolatile(); } + bool isNonTemporal() const { return MemSDNodeBits.isNonTemporal(); } + bool isInvariant() const { return MemSDNodeBits.isInvariant(); } - AtomicOrdering getOrdering() const { - return AtomicOrdering((SubclassData >> 8) & 15); - } + AtomicOrdering getOrdering() const { return MemSDNodeBits.ordering(); } SynchronizationScope getSynchScope() const { - return SynchronizationScope((SubclassData >> 12) & 1); + return MemSDNodeBits.synchScope(); } // Returns the offset from the location of the access. @@ -1130,23 +1240,9 @@ void InitAtomic(AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) { - // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp. - assert((AtomicOrdering)((unsigned)SuccessOrdering & 15) == - SuccessOrdering && - "Ordering may not require more than 4 bits!"); - assert((AtomicOrdering)((unsigned)FailureOrdering & 15) == - FailureOrdering && - "Ordering may not require more than 4 bits!"); - assert((SynchScope & 1) == SynchScope && - "SynchScope may not require more than 1 bit!"); - SubclassData |= (unsigned)SuccessOrdering << 8; - SubclassData |= SynchScope << 12; + MemSDNodeBits.setOrdering(SuccessOrdering); + MemSDNodeBits.setSynchScope(SynchScope); this->FailureOrdering = FailureOrdering; - assert(getSuccessOrdering() == SuccessOrdering && - "Ordering encoding error!"); - assert(getFailureOrdering() == FailureOrdering && - "Ordering encoding error!"); - assert(getSynchScope() == SynchScope && "Synch-scope encoding error!"); } public: @@ -1205,7 +1301,7 @@ MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { - SubclassData |= 1u << 13; + SDNodeBits.setIsMemIntrinsic(true); } // Methods to support isa and dyn_cast @@ -1285,7 +1381,7 @@ : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DL, getSDVTList(VT)), Value(val) { - SubclassData |= (uint16_t)isOpaque; + ConstantSDNodeBits.setIsOpaque(isOpaque); } public: @@ -1298,7 +1394,7 @@ bool isNullValue() const { return Value->isNullValue(); } bool isAllOnesValue() const { return Value->isAllOnesValue(); } - bool isOpaque() const { return SubclassData & 1; } + bool isOpaque() const { return ConstantSDNodeBits.isOpaque(); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::Constant || @@ -1780,8 +1876,7 @@ SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { - SubclassData |= AM << 2; - assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); + LSBaseSDNodeBits.setAddressingMode(AM); } const SDValue &getOffset() const { @@ -1791,7 +1886,7 @@ /// Return the addressing mode for this load or store: /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. ISD::MemIndexedMode getAddressingMode() const { - return ISD::MemIndexedMode((SubclassData >> 2) & 7); + return LSBaseSDNodeBits.addressingMode(); } /// Return true if this is a pre/post inc/dec load/store. @@ -1813,18 +1908,13 @@ ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { - SubclassData |= (unsigned short)ETy; - assert(getExtensionType() == ETy && "LoadExtType encoding error!"); - assert(readMem() && "Load MachineMemOperand is not a load!"); - assert(!writeMem() && "Load MachineMemOperand is a store!"); + LoadSDNodeBits.setExtTy(ETy); } public: /// Return whether this is a plain node, /// or one of the varieties of value-extending loads. - ISD::LoadExtType getExtensionType() const { - return ISD::LoadExtType(SubclassData & 3); - } + ISD::LoadExtType getExtensionType() const { return LoadSDNodeBits.extTy(); } const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getOffset() const { return getOperand(2); } @@ -1841,17 +1931,14 @@ ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, MachineMemOperand *MMO) : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { - SubclassData |= (unsigned short)isTrunc; - assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); - assert(!readMem() && "Store MachineMemOperand is a load!"); - assert(writeMem() && "Store MachineMemOperand is not a store!"); + StoreSDNodeBits.setIsTruncating(isTrunc); } public: /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. - bool isTruncatingStore() const { return SubclassData & 1; } + bool isTruncatingStore() const { return StoreSDNodeBits.isTruncating(); } const SDValue &getValue() const { return getOperand(1); } const SDValue &getBasePtr() const { return getOperand(2); } @@ -1891,12 +1978,11 @@ MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) { - SubclassData |= (unsigned short)ETy; + LoadSDNodeBits.setExtTy(ETy); } - ISD::LoadExtType getExtensionType() const { - return ISD::LoadExtType(SubclassData & 3); - } + ISD::LoadExtType getExtensionType() const { return LoadSDNodeBits.extTy(); } + const SDValue &getSrc0() const { return getOperand(3); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD; @@ -1911,12 +1997,12 @@ MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, bool isTrunc, EVT MemVT, MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) { - SubclassData |= (unsigned short)isTrunc; + StoreSDNodeBits.setIsTruncating(isTrunc); } /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. - bool isTruncatingStore() const { return SubclassData & 1; } + bool isTruncatingStore() const { return StoreSDNodeBits.isTruncating(); } const SDValue &getValue() const { return getOperand(3); } Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -521,24 +521,6 @@ AddNodeIDCustom(ID, N); } -/// encodeMemSDNodeFlags - Generic routine for computing a value for use in -/// the CSE map that carries volatility, temporalness, indexing mode, and -/// extension/truncation information. -/// -static inline unsigned -encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, - bool isNonTemporal, bool isInvariant) { - assert((ConvType & 3) == ConvType && - "ConvType may not require more than 2 bits!"); - assert((AM & 7) == AM && - "AM may not require more than 3 bits!"); - return ConvType | - (AM << 2) | - (isVolatile << 5) | - (isNonTemporal << 6) | - (isInvariant << 7); -} - //===----------------------------------------------------------------------===// // SelectionDAG Class //===----------------------------------------------------------------------===// @@ -5101,9 +5083,12 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); ID.AddInteger(MemVT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + // The compiler can reduce this expression to a constant iff we pass an empty + // DebugLoc. Thankfully, the debug location doesn't have any bearing on the + // subclass data. + ID.AddInteger( + LoadSDNode(dl.getIROrder(), DebugLoc(), VTs, AM, ExtType, MemVT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5200,8 +5185,9 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger(StoreSDNode(dl.getIROrder(), DebugLoc(), VTs, ISD::UNINDEXED, + false, VT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5265,8 +5251,9 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops); ID.AddInteger(SVT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger(StoreSDNode(dl.getIROrder(), DebugLoc(), VTs, ISD::UNINDEXED, + true, SVT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5318,10 +5305,9 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED, - MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger( + MaskedLoadSDNode(dl.getIROrder(), DebugLoc(), VTs, ExtTy, MemVT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5349,8 +5335,9 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger( + MaskedStoreSDNode(dl.getIROrder(), DebugLoc(), VTs, isTrunc, MemVT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5374,10 +5361,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ISD::NON_EXTLOAD, ISD::UNINDEXED, - MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger(MaskedGatherSDNode(dl.getIROrder(), DebugLoc(), VTs, VT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5411,9 +5396,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger(MaskedScatterSDNode(dl.getIROrder(), DebugLoc(), VTs, VT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -6711,11 +6695,10 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, MachineMemOperand *mmo) : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { - SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant()); - assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); - assert(isNonTemporal() == MMO->isNonTemporal() && - "Non-temporal encoding error!"); + MemSDNodeBits.setIsVolatile(MMO->isVolatile()); + MemSDNodeBits.setIsNonTemporal(MMO->isNonTemporal()); + MemSDNodeBits.setIsInvariant(MMO->isInvariant()); + // We check here that the size of the memory operand fits within the size of // the MMO. This is because the MMO might indicate only a possible address // range instead of specifying the affected memory addresses precisely.