Index: llvm/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -400,15 +400,81 @@ /// The operation that this node performs. int16_t NodeType; - /// This tracks whether this node has one or more dbg_value - /// nodes corresponding to it. - uint16_t HasDebugValue : 1; - protected: - /// This member is defined by this class, but is not used for - /// anything. Subclasses can use it to hold whatever state they find useful. - /// This field is initialized to zero by the ctor. - uint16_t SubclassData : 15; + class SDNodeBitfields { + friend class SDNode; + friend class MemIntrinsicSDNode; + + uint16_t HasDebugValue : 1; + uint16_t IsMemIntrinsic : 1; + }; + enum { NumSDNodeBits = 2 }; + + class ConstantSDNodeBitfields { + friend class ConstantSDNode; + + uint16_t : NumSDNodeBits; + + uint16_t IsOpaque : 1; + }; + + class MemSDNodeBitfields { + friend class MemSDNode; + friend class MemIntrinsicSDNode; + friend class AtomicSDNode; + + uint16_t : NumSDNodeBits; + + uint16_t IsVolatile : 1; + uint16_t IsNonTemporal : 1; + uint16_t IsInvariant : 1; + uint16_t SynchScope : 1; // enum SynchronizationScope + uint16_t Ordering : 4; // enum AtomicOrdering + }; + enum { NumMemSDNodeBits = NumSDNodeBits + 8 }; + + class LSBaseSDNodeBitfields { + friend class LSBaseSDNode; + uint16_t : NumMemSDNodeBits; + + uint16_t AddressingMode : 3; // enum ISD::MemIndexedMode + }; + enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; + + class LoadSDNodeBitfields { + friend class LoadSDNode; + friend class MaskedLoadSDNode; + + uint16_t : NumLSBaseSDNodeBits; + + ISD::LoadExtType extTy() const { + return static_cast(ExtTy_); + } + void setExtTy(ISD::LoadExtType V) { + ExtTy_ = static_cast(V); + assert(ExtTy_ == V && "Value truncated"); + } + + uint16_t ExtTy_ : 2; // enum ISD::LoadExtType + }; + + class StoreSDNodeBitfields { + friend class StoreSDNode; + friend class MaskedStoreSDNode; + + uint16_t : NumLSBaseSDNodeBits; + + uint16_t IsTruncating : 1; + }; + + union { + SDNodeBitfields SDNodeBits; + ConstantSDNodeBitfields ConstantSDNodeBits; + MemSDNodeBitfields MemSDNodeBits; + LSBaseSDNodeBitfields LSBaseSDNodeBits; + LoadSDNodeBitfields LoadSDNodeBits; + StoreSDNodeBitfields StoreSDNodeBits; + }; private: /// Unique id per SDNode in the DAG. @@ -481,7 +547,8 @@ /// proper classof relationship. bool isMemIntrinsic() const { return (NodeType == ISD::INTRINSIC_W_CHAIN || - NodeType == ISD::INTRINSIC_VOID) && ((SubclassData >> 13) & 1); + NodeType == ISD::INTRINSIC_VOID) && + SDNodeBits.IsMemIntrinsic; } /// Test if this node has a post-isel opcode, directly @@ -496,11 +563,8 @@ return ~NodeType; } - /// Get this bit. - bool getHasDebugValue() const { return HasDebugValue; } - - /// Set this bit. - void setHasDebugValue(bool b) { HasDebugValue = b; } + bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } + void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } /// Return true if there are no uses of this node. bool use_empty() const { return UseList == nullptr; } @@ -815,10 +879,10 @@ /// SDNodes are created without any operands, and never own the operand /// storage. To add operands, see SelectionDAG::createOperands. SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) - : NodeType(Opc), HasDebugValue(false), SubclassData(0), NodeId(-1), - OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr), - NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order), + : NodeType(Opc), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), + UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) { + memset(&SDNodeBits, 0, sizeof(SDNodeBits)); assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); assert(NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"); @@ -1036,20 +1100,24 @@ /// encoding of the volatile flag, as well as bits used by subclasses. This /// function should only be used to compute a FoldingSetNodeID value. unsigned getRawSubclassData() const { - return SubclassData; + uint16_t Data; + memcpy(&Data, &SDNodeBits, sizeof(SDNodeBits)); + static_assert(sizeof(SDNodeBits) == sizeof(uint16_t), + "SDNodeBits field too small?"); + return Data; } // We access subclass data here so that we can check consistency // with MachineMemOperand information. - bool isVolatile() const { return (SubclassData >> 5) & 1; } - bool isNonTemporal() const { return (SubclassData >> 6) & 1; } - bool isInvariant() const { return (SubclassData >> 7) & 1; } + bool isVolatile() const { return MemSDNodeBits.IsVolatile; } + bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } + bool isInvariant() const { return MemSDNodeBits.IsInvariant; } AtomicOrdering getOrdering() const { - return AtomicOrdering((SubclassData >> 8) & 15); + return static_cast(MemSDNodeBits.Ordering); } SynchronizationScope getSynchScope() const { - return SynchronizationScope((SubclassData >> 12) & 1); + return static_cast(MemSDNodeBits.SynchScope); } // Returns the offset from the location of the access. @@ -1130,23 +1198,12 @@ void InitAtomic(AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) { - // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp. - assert((AtomicOrdering)((unsigned)SuccessOrdering & 15) == - SuccessOrdering && - "Ordering may not require more than 4 bits!"); - assert((AtomicOrdering)((unsigned)FailureOrdering & 15) == - FailureOrdering && - "Ordering may not require more than 4 bits!"); - assert((SynchScope & 1) == SynchScope && - "SynchScope may not require more than 1 bit!"); - SubclassData |= (unsigned)SuccessOrdering << 8; - SubclassData |= SynchScope << 12; + MemSDNodeBits.Ordering = static_cast(SuccessOrdering); + assert(MemSDNodeBits.Ordering == static_cast(SuccessOrdering) && + "Value truncated"); + MemSDNodeBits.SynchScope = static_cast(SynchScope); + assert(MemSDNodeBits.SynchScope == SynchScope && "Value truncated"); this->FailureOrdering = FailureOrdering; - assert(getSuccessOrdering() == SuccessOrdering && - "Ordering encoding error!"); - assert(getFailureOrdering() == FailureOrdering && - "Ordering encoding error!"); - assert(getSynchScope() == SynchScope && "Synch-scope encoding error!"); } public: @@ -1205,7 +1262,7 @@ MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { - SubclassData |= 1u << 13; + SDNodeBits.IsMemIntrinsic = true; } // Methods to support isa and dyn_cast @@ -1285,7 +1342,7 @@ : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DL, getSDVTList(VT)), Value(val) { - SubclassData |= (uint16_t)isOpaque; + ConstantSDNodeBits.IsOpaque = isOpaque; } public: @@ -1298,7 +1355,7 @@ bool isNullValue() const { return Value->isNullValue(); } bool isAllOnesValue() const { return Value->isAllOnesValue(); } - bool isOpaque() const { return SubclassData & 1; } + bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::Constant || @@ -1780,8 +1837,8 @@ SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { - SubclassData |= AM << 2; - assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); + LSBaseSDNodeBits.AddressingMode = AM; + assert(LSBaseSDNodeBits.AddressingMode == AM && "Value truncated"); } const SDValue &getOffset() const { @@ -1791,7 +1848,7 @@ /// Return the addressing mode for this load or store: /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. ISD::MemIndexedMode getAddressingMode() const { - return ISD::MemIndexedMode((SubclassData >> 2) & 7); + return static_cast(LSBaseSDNodeBits.AddressingMode); } /// Return true if this is a pre/post inc/dec load/store. @@ -1813,18 +1870,13 @@ ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { - SubclassData |= (unsigned short)ETy; - assert(getExtensionType() == ETy && "LoadExtType encoding error!"); - assert(readMem() && "Load MachineMemOperand is not a load!"); - assert(!writeMem() && "Load MachineMemOperand is a store!"); + LoadSDNodeBits.setExtTy(ETy); } public: /// Return whether this is a plain node, /// or one of the varieties of value-extending loads. - ISD::LoadExtType getExtensionType() const { - return ISD::LoadExtType(SubclassData & 3); - } + ISD::LoadExtType getExtensionType() const { return LoadSDNodeBits.extTy(); } const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getOffset() const { return getOperand(2); } @@ -1841,17 +1893,14 @@ ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, MachineMemOperand *MMO) : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { - SubclassData |= (unsigned short)isTrunc; - assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); - assert(!readMem() && "Store MachineMemOperand is a load!"); - assert(writeMem() && "Store MachineMemOperand is not a store!"); + StoreSDNodeBits.IsTruncating = isTrunc; } public: /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. - bool isTruncatingStore() const { return SubclassData & 1; } + bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } const SDValue &getValue() const { return getOperand(1); } const SDValue &getBasePtr() const { return getOperand(2); } @@ -1891,12 +1940,11 @@ MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) { - SubclassData |= (unsigned short)ETy; + LoadSDNodeBits.setExtTy(ETy); } - ISD::LoadExtType getExtensionType() const { - return ISD::LoadExtType(SubclassData & 3); - } + ISD::LoadExtType getExtensionType() const { return LoadSDNodeBits.extTy(); } + const SDValue &getSrc0() const { return getOperand(3); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD; @@ -1911,12 +1959,12 @@ MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, bool isTrunc, EVT MemVT, MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) { - SubclassData |= (unsigned short)isTrunc; + StoreSDNodeBits.IsTruncating = isTrunc; } /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. - bool isTruncatingStore() const { return SubclassData & 1; } + bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } const SDValue &getValue() const { return getOperand(3); } Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -521,24 +521,6 @@ AddNodeIDCustom(ID, N); } -/// encodeMemSDNodeFlags - Generic routine for computing a value for use in -/// the CSE map that carries volatility, temporalness, indexing mode, and -/// extension/truncation information. -/// -static inline unsigned -encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, - bool isNonTemporal, bool isInvariant) { - assert((ConvType & 3) == ConvType && - "ConvType may not require more than 2 bits!"); - assert((AM & 7) == AM && - "AM may not require more than 3 bits!"); - return ConvType | - (AM << 2) | - (isVolatile << 5) | - (isNonTemporal << 6) | - (isInvariant << 7); -} - //===----------------------------------------------------------------------===// // SelectionDAG Class //===----------------------------------------------------------------------===// @@ -5101,9 +5083,12 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); ID.AddInteger(MemVT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + // The compiler can reduce this expression to a constant iff we pass an empty + // DebugLoc. Thankfully, the debug location doesn't have any bearing on the + // subclass data. + ID.AddInteger( + LoadSDNode(dl.getIROrder(), DebugLoc(), VTs, AM, ExtType, MemVT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5200,8 +5185,9 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger(StoreSDNode(dl.getIROrder(), DebugLoc(), VTs, ISD::UNINDEXED, + false, VT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5265,8 +5251,9 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops); ID.AddInteger(SVT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger(StoreSDNode(dl.getIROrder(), DebugLoc(), VTs, ISD::UNINDEXED, + true, SVT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5318,10 +5305,9 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED, - MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger( + MaskedLoadSDNode(dl.getIROrder(), DebugLoc(), VTs, ExtTy, MemVT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5349,8 +5335,9 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger( + MaskedStoreSDNode(dl.getIROrder(), DebugLoc(), VTs, isTrunc, MemVT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5374,10 +5361,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ISD::NON_EXTLOAD, ISD::UNINDEXED, - MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger(MaskedGatherSDNode(dl.getIROrder(), DebugLoc(), VTs, VT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5411,9 +5396,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger(MaskedScatterSDNode(dl.getIROrder(), DebugLoc(), VTs, VT, MMO) + .getRawSubclassData()); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -6711,11 +6695,10 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, MachineMemOperand *mmo) : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { - SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant()); - assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); - assert(isNonTemporal() == MMO->isNonTemporal() && - "Non-temporal encoding error!"); + MemSDNodeBits.IsVolatile = MMO->isVolatile(); + MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); + MemSDNodeBits.IsInvariant = MMO->isInvariant(); + // We check here that the size of the memory operand fits within the size of // the MMO. This is because the MMO might indicate only a possible address // range instead of specifying the affected memory addresses precisely.