Index: llvm/trunk/include/llvm/IR/User.h =================================================================== --- llvm/trunk/include/llvm/IR/User.h +++ llvm/trunk/include/llvm/IR/User.h @@ -19,6 +19,7 @@ #ifndef LLVM_IR_USER_H #define LLVM_IR_USER_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Value.h" @@ -39,6 +40,9 @@ friend struct HungoffOperandTraits; virtual void anchor(); + LLVM_ATTRIBUTE_ALWAYS_INLINE inline static void * + allocateFixedOperandUser(size_t, unsigned, unsigned); + protected: /// Allocate a User with an operand pointer co-allocated. /// @@ -51,6 +55,16 @@ /// This is used for subclasses which have a fixed number of operands. void *operator new(size_t Size, unsigned Us); + /// Allocate a User with the operands co-allocated. If DescBytes is non-zero + /// then allocate an additional DescBytes bytes before the operands. These + /// bytes can be accessed by calling getDescriptor. + /// + /// DescBytes needs to be divisible by sizeof(void *). The allocated + /// descriptor, if any, is aligned to sizeof(void *) bytes. + /// + /// This is used for subclasses which have a fixed number of operands. + void *operator new(size_t Size, unsigned Us, unsigned DescBytes); + User(Type *ty, unsigned vty, Use *OpList, unsigned NumOps) : Value(ty, vty) { assert(NumOps < (1u << NumUserOperandsBits) && "Too many operands"); @@ -137,6 +151,12 @@ unsigned getNumOperands() const { return NumUserOperands; } + /// Returns the descriptor co-allocated with this User instance. + ArrayRef getDescriptor() const; + + /// Returns the descriptor co-allocated with this User instance. + MutableArrayRef getDescriptor(); + /// Set the number of operands on a GlobalVariable. /// /// GlobalVariable always allocates space for a single operands, but Index: llvm/trunk/include/llvm/IR/Value.h =================================================================== --- llvm/trunk/include/llvm/IR/Value.h +++ llvm/trunk/include/llvm/IR/Value.h @@ -104,12 +104,13 @@ /// /// Note, this should *NOT* be used directly by any class other than User. /// User uses this value to find the Use list. - enum : unsigned { NumUserOperandsBits = 29 }; + enum : unsigned { NumUserOperandsBits = 28 }; unsigned NumUserOperands : NumUserOperandsBits; bool IsUsedByMD : 1; bool HasName : 1; bool HasHungOffUses : 1; + bool HasDescriptor : 1; private: template // UseT == 'Use' or 'const Use' Index: llvm/trunk/lib/IR/User.cpp =================================================================== --- llvm/trunk/lib/IR/User.cpp +++ llvm/trunk/lib/IR/User.cpp @@ -87,22 +87,70 @@ Use::zap(OldOps, OldOps + OldNumUses, true); } + +// This is a private struct used by `User` to track the co-allocated descriptor +// section. +struct DescriptorInfo { + intptr_t SizeInBytes; +}; + +ArrayRef User::getDescriptor() const { + auto MutableARef = const_cast(this)->getDescriptor(); + return {MutableARef.begin(), MutableARef.end()}; +} + +MutableArrayRef User::getDescriptor() { + assert(HasDescriptor && "Don't call otherwise!"); + assert(!HasHungOffUses && "Invariant!"); + + auto *DI = reinterpret_cast(getIntrusiveOperands()) - 1; + assert(DI->SizeInBytes != 0 && "Should not have had a descriptor otherwise!"); + + return MutableArrayRef( + reinterpret_cast(DI) - DI->SizeInBytes, DI->SizeInBytes); +} + //===----------------------------------------------------------------------===// // User operator new Implementations //===----------------------------------------------------------------------===// -void *User::operator new(size_t Size, unsigned Us) { +void *User::allocateFixedOperandUser(size_t Size, unsigned Us, + unsigned DescBytes) { assert(Us < (1u << NumUserOperandsBits) && "Too many operands"); - void *Storage = ::operator new(Size + sizeof(Use) * Us); - Use *Start = static_cast(Storage); + + static_assert(sizeof(DescriptorInfo) % sizeof(void *) == 0, "Required below"); + + unsigned DescBytesToAllocate = + DescBytes == 0 ? 0 : (DescBytes + sizeof(DescriptorInfo)); + assert(DescBytesToAllocate % sizeof(void *) == 0 && + "We need this to satisfy alignment constraints for Uses"); + + uint8_t *Storage = static_cast( + ::operator new(Size + sizeof(Use) * Us + DescBytesToAllocate)); + Use *Start = reinterpret_cast(Storage + DescBytesToAllocate); Use *End = Start + Us; User *Obj = reinterpret_cast(End); Obj->NumUserOperands = Us; Obj->HasHungOffUses = false; + Obj->HasDescriptor = DescBytes != 0; Use::initTags(Start, End); + + if (DescBytes != 0) { + auto *DescInfo = reinterpret_cast(Storage + DescBytes); + DescInfo->SizeInBytes = DescBytes; + } + return Obj; } +void *User::operator new(size_t Size, unsigned Us) { + return allocateFixedOperandUser(Size, Us, 0); +} + +void *User::operator new(size_t Size, unsigned Us, unsigned DescBytes) { + return allocateFixedOperandUser(Size, Us, DescBytes); +} + void *User::operator new(size_t Size) { // Allocate space for a single Use* void *Storage = ::operator new(Size + sizeof(Use *)); @@ -110,6 +158,7 @@ User *Obj = reinterpret_cast(HungOffOperandList + 1); Obj->NumUserOperands = 0; Obj->HasHungOffUses = true; + Obj->HasDescriptor = false; *HungOffOperandList = nullptr; return Obj; } @@ -123,11 +172,20 @@ // use a Use[] allocated prior to the user. User *Obj = static_cast(Usr); if (Obj->HasHungOffUses) { + assert(!Obj->HasDescriptor && "not supported!"); + Use **HungOffOperandList = static_cast(Usr) - 1; // drop the hung off uses. Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->NumUserOperands, /* Delete */ true); ::operator delete(HungOffOperandList); + } else if (Obj->HasDescriptor) { + Use *UseBegin = static_cast(Usr) - Obj->NumUserOperands; + Use::zap(UseBegin, UseBegin + Obj->NumUserOperands, /* Delete */ false); + + auto *DI = reinterpret_cast(UseBegin) - 1; + uint8_t *Storage = reinterpret_cast(DI) - DI->SizeInBytes; + ::operator delete(Storage); } else { Use *Storage = static_cast(Usr) - Obj->NumUserOperands; Use::zap(Storage, Storage + Obj->NumUserOperands,