Index: llvm/trunk/include/llvm/CodeGen/SelectionDAG.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/SelectionDAG.h +++ llvm/trunk/include/llvm/CodeGen/SelectionDAG.h @@ -272,6 +272,22 @@ SDNodeT(std::forward(Args)...); } + /// Build a synthetic SDNodeT with the given args and extract its subclass + /// data as an integer (e.g. for use in a folding set). + /// + /// The args to this function are the same as the args to SDNodeT's + /// constructor, except the second arg (assumed to be a const DebugLoc&) is + /// omitted. + template + static uint16_t getSyntheticNodeSubclassData(unsigned IROrder, + ArgTypes &&... Args) { + // The compiler can reduce this expression to a constant iff we pass an + // empty DebugLoc. Thankfully, the debug location doesn't have any bearing + // on the subclass data. + return SDNodeT(IROrder, DebugLoc(), std::forward(Args)...) + .getRawSubclassData(); + } + void createOperands(SDNode *Node, ArrayRef Vals) { assert(!Node->OperandList && "Node already has operands"); SDUse *Ops = OperandRecycler.allocate( Index: llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h @@ -400,15 +400,77 @@ /// The operation that this node performs. int16_t NodeType; - /// This tracks whether this node has one or more dbg_value - /// nodes corresponding to it. - uint16_t HasDebugValue : 1; - protected: - /// This member is defined by this class, but is not used for - /// anything. Subclasses can use it to hold whatever state they find useful. - /// This field is initialized to zero by the ctor. - uint16_t SubclassData : 15; + // We define a set of mini-helper classes to help us interpret the bits in our + // SubclassData. These are designed to fit within a uint16_t so they pack + // with NodeType. + + class SDNodeBitfields { + friend class SDNode; + friend class MemIntrinsicSDNode; + + uint16_t HasDebugValue : 1; + uint16_t IsMemIntrinsic : 1; + }; + enum { NumSDNodeBits = 2 }; + + class ConstantSDNodeBitfields { + friend class ConstantSDNode; + + uint16_t : NumSDNodeBits; + + uint16_t IsOpaque : 1; + }; + + class MemSDNodeBitfields { + friend class MemSDNode; + friend class MemIntrinsicSDNode; + friend class AtomicSDNode; + + uint16_t : NumSDNodeBits; + + uint16_t IsVolatile : 1; + uint16_t IsNonTemporal : 1; + uint16_t IsInvariant : 1; + uint16_t SynchScope : 1; // enum SynchronizationScope + uint16_t Ordering : 4; // enum AtomicOrdering + }; + enum { NumMemSDNodeBits = NumSDNodeBits + 8 }; + + class LSBaseSDNodeBitfields { + friend class LSBaseSDNode; + uint16_t : NumMemSDNodeBits; + + uint16_t AddressingMode : 3; // enum ISD::MemIndexedMode + }; + enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; + + class LoadSDNodeBitfields { + friend class LoadSDNode; + friend class MaskedLoadSDNode; + + uint16_t : NumLSBaseSDNodeBits; + + uint16_t ExtTy : 2; // enum ISD::LoadExtType + }; + + class StoreSDNodeBitfields { + friend class StoreSDNode; + friend class MaskedStoreSDNode; + + uint16_t : NumLSBaseSDNodeBits; + + uint16_t IsTruncating : 1; + }; + + union { + SDNodeBitfields SDNodeBits; + ConstantSDNodeBitfields ConstantSDNodeBits; + MemSDNodeBitfields MemSDNodeBits; + LSBaseSDNodeBitfields LSBaseSDNodeBits; + LoadSDNodeBitfields LoadSDNodeBits; + StoreSDNodeBitfields StoreSDNodeBits; + }; private: /// Unique id per SDNode in the DAG. @@ -481,7 +543,8 @@ /// proper classof relationship. bool isMemIntrinsic() const { return (NodeType == ISD::INTRINSIC_W_CHAIN || - NodeType == ISD::INTRINSIC_VOID) && ((SubclassData >> 13) & 1); + NodeType == ISD::INTRINSIC_VOID) && + SDNodeBits.IsMemIntrinsic; } /// Test if this node has a post-isel opcode, directly @@ -496,11 +559,8 @@ return ~NodeType; } - /// Get this bit. - bool getHasDebugValue() const { return HasDebugValue; } - - /// Set this bit. - void setHasDebugValue(bool b) { HasDebugValue = b; } + bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } + void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } /// Return true if there are no uses of this node. bool use_empty() const { return UseList == nullptr; } @@ -815,10 +875,10 @@ /// SDNodes are created without any operands, and never own the operand /// storage. To add operands, see SelectionDAG::createOperands. SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) - : NodeType(Opc), HasDebugValue(false), SubclassData(0), NodeId(-1), - OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr), - NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order), + : NodeType(Opc), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), + UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) { + memset(&SDNodeBits, 0, sizeof(SDNodeBits)); assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); assert(NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"); @@ -1036,20 +1096,24 @@ /// encoding of the volatile flag, as well as bits used by subclasses. This /// function should only be used to compute a FoldingSetNodeID value. unsigned getRawSubclassData() const { - return SubclassData; + uint16_t Data; + memcpy(&Data, &SDNodeBits, sizeof(SDNodeBits)); + static_assert(sizeof(SDNodeBits) == sizeof(uint16_t), + "SDNodeBits field too small?"); + return Data; } // We access subclass data here so that we can check consistency // with MachineMemOperand information. - bool isVolatile() const { return (SubclassData >> 5) & 1; } - bool isNonTemporal() const { return (SubclassData >> 6) & 1; } - bool isInvariant() const { return (SubclassData >> 7) & 1; } + bool isVolatile() const { return MemSDNodeBits.IsVolatile; } + bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } + bool isInvariant() const { return MemSDNodeBits.IsInvariant; } AtomicOrdering getOrdering() const { - return AtomicOrdering((SubclassData >> 8) & 15); + return static_cast(MemSDNodeBits.Ordering); } SynchronizationScope getSynchScope() const { - return SynchronizationScope((SubclassData >> 12) & 1); + return static_cast(MemSDNodeBits.SynchScope); } // Returns the offset from the location of the access. @@ -1130,23 +1194,11 @@ void InitAtomic(AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) { - // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp. - assert((AtomicOrdering)((unsigned)SuccessOrdering & 15) == - SuccessOrdering && - "Ordering may not require more than 4 bits!"); - assert((AtomicOrdering)((unsigned)FailureOrdering & 15) == - FailureOrdering && - "Ordering may not require more than 4 bits!"); - assert((SynchScope & 1) == SynchScope && - "SynchScope may not require more than 1 bit!"); - SubclassData |= (unsigned)SuccessOrdering << 8; - SubclassData |= SynchScope << 12; + MemSDNodeBits.Ordering = static_cast(SuccessOrdering); + assert(getOrdering() == SuccessOrdering && "Value truncated"); + MemSDNodeBits.SynchScope = static_cast(SynchScope); + assert(getSynchScope() == SynchScope && "Value truncated"); this->FailureOrdering = FailureOrdering; - assert(getSuccessOrdering() == SuccessOrdering && - "Ordering encoding error!"); - assert(getFailureOrdering() == FailureOrdering && - "Ordering encoding error!"); - assert(getSynchScope() == SynchScope && "Synch-scope encoding error!"); } public: @@ -1205,7 +1257,7 @@ MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { - SubclassData |= 1u << 13; + SDNodeBits.IsMemIntrinsic = true; } // Methods to support isa and dyn_cast @@ -1285,7 +1337,7 @@ : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DL, getSDVTList(VT)), Value(val) { - SubclassData |= (uint16_t)isOpaque; + ConstantSDNodeBits.IsOpaque = isOpaque; } public: @@ -1298,7 +1350,7 @@ bool isNullValue() const { return Value->isNullValue(); } bool isAllOnesValue() const { return Value->isAllOnesValue(); } - bool isOpaque() const { return SubclassData & 1; } + bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::Constant || @@ -1780,8 +1832,8 @@ SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { - SubclassData |= AM << 2; - assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); + LSBaseSDNodeBits.AddressingMode = AM; + assert(getAddressingMode() == AM && "Value truncated"); } const SDValue &getOffset() const { @@ -1791,7 +1843,7 @@ /// Return the addressing mode for this load or store: /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. ISD::MemIndexedMode getAddressingMode() const { - return ISD::MemIndexedMode((SubclassData >> 2) & 7); + return static_cast(LSBaseSDNodeBits.AddressingMode); } /// Return true if this is a pre/post inc/dec load/store. @@ -1813,8 +1865,7 @@ ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { - SubclassData |= (unsigned short)ETy; - assert(getExtensionType() == ETy && "LoadExtType encoding error!"); + LoadSDNodeBits.ExtTy = ETy; assert(readMem() && "Load MachineMemOperand is not a load!"); assert(!writeMem() && "Load MachineMemOperand is a store!"); } @@ -1823,7 +1874,7 @@ /// Return whether this is a plain node, /// or one of the varieties of value-extending loads. ISD::LoadExtType getExtensionType() const { - return ISD::LoadExtType(SubclassData & 3); + return static_cast(LoadSDNodeBits.ExtTy); } const SDValue &getBasePtr() const { return getOperand(1); } @@ -1841,8 +1892,7 @@ ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, MachineMemOperand *MMO) : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { - SubclassData |= (unsigned short)isTrunc; - assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); + StoreSDNodeBits.IsTruncating = isTrunc; assert(!readMem() && "Store MachineMemOperand is a load!"); assert(writeMem() && "Store MachineMemOperand is not a store!"); } @@ -1851,7 +1901,7 @@ /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. - bool isTruncatingStore() const { return SubclassData & 1; } + bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } const SDValue &getValue() const { return getOperand(1); } const SDValue &getBasePtr() const { return getOperand(2); } @@ -1891,12 +1941,13 @@ MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) { - SubclassData |= (unsigned short)ETy; + LoadSDNodeBits.ExtTy = ETy; } ISD::LoadExtType getExtensionType() const { - return ISD::LoadExtType(SubclassData & 3); + return static_cast(LoadSDNodeBits.ExtTy); } + const SDValue &getSrc0() const { return getOperand(3); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD; @@ -1911,12 +1962,12 @@ MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, bool isTrunc, EVT MemVT, MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) { - SubclassData |= (unsigned short)isTrunc; + StoreSDNodeBits.IsTruncating = isTrunc; } /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. - bool isTruncatingStore() const { return SubclassData & 1; } + bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } const SDValue &getValue() const { return getOperand(3); } Index: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -521,24 +521,6 @@ AddNodeIDCustom(ID, N); } -/// encodeMemSDNodeFlags - Generic routine for computing a value for use in -/// the CSE map that carries volatility, temporalness, indexing mode, and -/// extension/truncation information. -/// -static inline unsigned -encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, - bool isNonTemporal, bool isInvariant) { - assert((ConvType & 3) == ConvType && - "ConvType may not require more than 2 bits!"); - assert((AM & 7) == AM && - "AM may not require more than 3 bits!"); - return ConvType | - (AM << 2) | - (isVolatile << 5) | - (isNonTemporal << 6) | - (isInvariant << 7); -} - //===----------------------------------------------------------------------===// // SelectionDAG Class //===----------------------------------------------------------------------===// @@ -5101,9 +5083,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); ID.AddInteger(MemVT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger(getSyntheticNodeSubclassData( + dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5200,8 +5181,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger(getSyntheticNodeSubclassData( + dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5265,8 +5246,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops); ID.AddInteger(SVT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger(getSyntheticNodeSubclassData( + dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5318,10 +5299,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED, - MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger(getSyntheticNodeSubclassData( + dl.getIROrder(), VTs, ExtTy, MemVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5349,8 +5328,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant())); + ID.AddInteger(getSyntheticNodeSubclassData( + dl.getIROrder(), VTs, isTrunc, MemVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5374,10 +5353,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ISD::NON_EXTLOAD, ISD::UNINDEXED, - MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger(getSyntheticNodeSubclassData( + dl.getIROrder(), VTs, VT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5411,9 +5388,8 @@ FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), - MMO->isInvariant())); + ID.AddInteger(getSyntheticNodeSubclassData( + dl.getIROrder(), VTs, VT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -6711,11 +6687,10 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, MachineMemOperand *mmo) : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { - SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant()); - assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); - assert(isNonTemporal() == MMO->isNonTemporal() && - "Non-temporal encoding error!"); + MemSDNodeBits.IsVolatile = MMO->isVolatile(); + MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); + MemSDNodeBits.IsInvariant = MMO->isInvariant(); + // We check here that the size of the memory operand fits within the size of // the MMO. This is because the MMO might indicate only a possible address // range instead of specifying the affected memory addresses precisely.