Index: llvm/include/llvm/IR/Metadata.h =================================================================== --- llvm/include/llvm/IR/Metadata.h +++ llvm/include/llvm/IR/Metadata.h @@ -928,8 +928,41 @@ friend class LLVMContextImpl; friend class DIArgList; - unsigned NumOperands; - unsigned NumUnresolved; + /// The header that is coallocated with an MDNode, along with the operands. + /// It is located immediately before the main body of the node. The operands + /// are in turn located immediately before the header. + struct Header { + unsigned NumOperands; + unsigned NumUnresolved = 0; + + static constexpr size_t getOpSize(unsigned NumOps) { + return sizeof(MDOperand) * NumOps; + } + static constexpr size_t getAllocSize(unsigned NumOps) { + return getOpSize(NumOps) + sizeof(Header); + } + void *getAllocation() { + return reinterpret_cast(this + 1) - + alignTo(getAllocSize(NumOperands), alignof(uint64_t)); + } + + explicit Header(unsigned NumOperands); + ~Header(); + MutableArrayRef operands() { + return makeMutableArrayRef( + reinterpret_cast(this) - NumOperands, NumOperands); + } + ArrayRef operands() const { + return makeArrayRef( + reinterpret_cast(this) - NumOperands, NumOperands); + } + }; + + Header &getHeader() { return *(reinterpret_cast
(this) - 1); } + + const Header &getHeader() const { + return *(reinterpret_cast(this) - 1); + } ContextAndReplaceableUses Context; @@ -938,7 +971,7 @@ ArrayRef Ops1, ArrayRef Ops2 = None); ~MDNode() = default; - void *operator new(size_t Size, unsigned NumOps); + void *operator new(size_t Size, unsigned NumOps, StorageType Storage); void operator delete(void *Mem); /// Required by std, but never called. @@ -953,8 +986,8 @@ void dropAllReferences(); - MDOperand *mutable_begin() { return mutable_end() - NumOperands; } - MDOperand *mutable_end() { return reinterpret_cast(this); } + MDOperand *mutable_begin() { return getHeader().operands().begin(); } + MDOperand *mutable_end() { return getHeader().operands().end(); } using mutable_op_range = iterator_range; @@ -1000,7 +1033,7 @@ /// As forward declarations are resolved, their containers should get /// resolved automatically. However, if this (or one of its operands) is /// involved in a cycle, \a resolveCycles() needs to be called explicitly. - bool isResolved() const { return !isTemporary() && !NumUnresolved; } + bool isResolved() const { return !isTemporary() && !getNumUnresolved(); } bool isUniqued() const { return Storage == Uniqued; } bool isDistinct() const { return Storage == Distinct; } @@ -1094,6 +1127,9 @@ /// Sets the operand directly, without worrying about uniquing. void setOperand(unsigned I, Metadata *New); + unsigned getNumUnresolved() const { return getHeader().NumUnresolved; } + + void setNumUnresolved(unsigned N) { getHeader().NumUnresolved = N; } void storeDistinctInContext(); template static T *storeImpl(T *N, StorageType Storage, StoreT &Store); @@ -1155,12 +1191,12 @@ op_range operands() const { return op_range(op_begin(), op_end()); } const MDOperand &getOperand(unsigned I) const { - assert(I < NumOperands && "Out of range"); - return op_begin()[I]; + assert(I < getNumOperands() && "Out of range"); + return getHeader().operands()[I]; } /// Return number of MDNode operands. - unsigned getNumOperands() const { return NumOperands; } + unsigned getNumOperands() const { return getHeader().NumOperands; } /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Metadata *MD) { Index: llvm/lib/IR/DebugInfoMetadata.cpp =================================================================== --- llvm/lib/IR/DebugInfoMetadata.cpp +++ llvm/lib/IR/DebugInfoMetadata.cpp @@ -78,8 +78,8 @@ Ops.push_back(Scope); if (InlinedAt) Ops.push_back(InlinedAt); - return storeImpl(new (Ops.size()) DILocation(Context, Storage, Line, Column, - Ops, ImplicitCode), + return storeImpl(new (Ops.size(), Storage) DILocation( + Context, Storage, Line, Column, Ops, ImplicitCode), Storage, Context.pImpl->DILocations); } @@ -304,7 +304,7 @@ // Use a nullptr for empty headers. assert(isCanonical(Header) && "Expected canonical MDString"); Metadata *PreOps[] = {Header}; - return storeImpl(new (DwarfOps.size() + 1) GenericDINode( + return storeImpl(new (DwarfOps.size() + 1, Storage) GenericDINode( Context, Storage, Hash, Tag, PreOps, DwarfOps), Storage, Context.pImpl->GenericDINodes); } @@ -329,17 +329,19 @@ } \ } while (false) #define DEFINE_GETIMPL_STORE(CLASS, ARGS, OPS) \ - return storeImpl(new (array_lengthof(OPS)) \ + return storeImpl(new (array_lengthof(OPS), Storage) \ CLASS(Context, Storage, UNWRAP_ARGS(ARGS), OPS), \ Storage, Context.pImpl->CLASS##s) #define DEFINE_GETIMPL_STORE_NO_OPS(CLASS, ARGS) \ - return storeImpl(new (0u) CLASS(Context, Storage, UNWRAP_ARGS(ARGS)), \ + return storeImpl(new (0u, Storage) \ + CLASS(Context, Storage, UNWRAP_ARGS(ARGS)), \ Storage, Context.pImpl->CLASS##s) #define DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(CLASS, OPS) \ - return storeImpl(new (array_lengthof(OPS)) CLASS(Context, Storage, OPS), \ + return storeImpl(new (array_lengthof(OPS), Storage) \ + CLASS(Context, Storage, OPS), \ Storage, Context.pImpl->CLASS##s) #define DEFINE_GETIMPL_STORE_N(CLASS, ARGS, OPS, NUM_OPS) \ - return storeImpl(new (NUM_OPS) \ + return storeImpl(new (NUM_OPS, Storage) \ CLASS(Context, Storage, UNWRAP_ARGS(ARGS), OPS), \ Storage, Context.pImpl->CLASS##s) @@ -848,7 +850,7 @@ Macros, SysRoot, SDK}; - return storeImpl(new (array_lengthof(Ops)) DICompileUnit( + return storeImpl(new (array_lengthof(Ops), Storage) DICompileUnit( Context, Storage, SourceLanguage, IsOptimized, RuntimeVersion, EmissionKind, DWOId, SplitDebugInlining, DebugInfoForProfiling, NameTableKind, RangesBaseAddress, Index: llvm/lib/IR/Metadata.cpp =================================================================== --- llvm/lib/IR/Metadata.cpp +++ llvm/lib/IR/Metadata.cpp @@ -523,35 +523,26 @@ "Alignment is insufficient after objects prepended to " #CLASS); #include "llvm/IR/Metadata.def" -void *MDNode::operator new(size_t Size, unsigned NumOps) { - size_t OpSize = NumOps * sizeof(MDOperand); +void *MDNode::operator new(size_t Size, unsigned NumOps, + StorageType /* Storage */) { // uint64_t is the most aligned type we need support (ensured by static_assert // above) - OpSize = alignTo(OpSize, alignof(uint64_t)); - void *Ptr = reinterpret_cast(::operator new(OpSize + Size)) + OpSize; - MDOperand *O = static_cast(Ptr); - for (MDOperand *E = O - NumOps; O != E; --O) - (void)new (O - 1) MDOperand; - return Ptr; + size_t AllocSize = alignTo(Header::getAllocSize(NumOps), alignof(uint64_t)); + char *Mem = reinterpret_cast(::operator new(AllocSize + Size)); + Header *H = new (Mem + AllocSize - sizeof(Header)) Header(NumOps); + return reinterpret_cast(H + 1); } -// Repress memory sanitization, due to use-after-destroy by operator -// delete. Bug report 24578 identifies this issue. -LLVM_NO_SANITIZE_MEMORY_ATTRIBUTE void MDNode::operator delete(void *Mem) { - MDNode *N = static_cast(Mem); - size_t OpSize = N->NumOperands * sizeof(MDOperand); - OpSize = alignTo(OpSize, alignof(uint64_t)); - - MDOperand *O = static_cast(Mem); - for (MDOperand *E = O - N->NumOperands; O != E; --O) - (O - 1)->~MDOperand(); - ::operator delete(reinterpret_cast(Mem) - OpSize); +void MDNode::operator delete(void *N) { + Header *H = reinterpret_cast
(N) - 1; + void *Mem = H->getAllocation(); + H->~Header(); + ::operator delete(Mem); } MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage, ArrayRef Ops1, ArrayRef Ops2) - : Metadata(ID, Storage), NumOperands(Ops1.size() + Ops2.size()), - NumUnresolved(0), Context(Context) { + : Metadata(ID, Storage), Context(Context) { unsigned Op = 0; for (Metadata *MD : Ops1) setOperand(Op++, MD); @@ -577,6 +568,19 @@ } } +MDNode::Header::Header(unsigned NumOps) { + NumOperands = NumOps; + MDOperand *O = reinterpret_cast(this); + for (MDOperand *E = O - NumOps; O != E; --O) + (void)new (O - 1) MDOperand(); +} + +MDNode::Header::~Header() { + MDOperand *O = reinterpret_cast(this) - NumOperands; + for (MDOperand *E = O + NumOperands; O != E; ++O) + (void)O->~MDOperand(); +} + static bool isOperandUnresolved(Metadata *Op) { if (auto *N = dyn_cast_or_null(Op)) return !N->isResolved(); @@ -584,9 +588,9 @@ } void MDNode::countUnresolvedOperands() { - assert(NumUnresolved == 0 && "Expected unresolved ops to be uncounted"); + assert(getNumUnresolved() == 0 && "Expected unresolved ops to be uncounted"); assert(isUniqued() && "Expected this to be uniqued"); - NumUnresolved = count_if(operands(), isOperandUnresolved); + setNumUnresolved(count_if(operands(), isOperandUnresolved)); } void MDNode::makeUniqued() { @@ -600,7 +604,7 @@ // Make this 'uniqued'. Storage = Uniqued; countUnresolvedOperands(); - if (!NumUnresolved) { + if (!getNumUnresolved()) { dropReplaceableUses(); assert(isResolved() && "Expected this to be resolved"); } @@ -624,14 +628,14 @@ assert(isUniqued() && "Expected this to be uniqued"); assert(!isResolved() && "Expected this to be unresolved"); - NumUnresolved = 0; + setNumUnresolved(0); dropReplaceableUses(); assert(isResolved() && "Expected this to be resolved"); } void MDNode::dropReplaceableUses() { - assert(!NumUnresolved && "Unexpected unresolved operand"); + assert(!getNumUnresolved() && "Unexpected unresolved operand"); // Drop any RAUW support. if (Context.hasReplaceableUses()) @@ -640,13 +644,13 @@ void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) { assert(isUniqued() && "Expected this to be uniqued"); - assert(NumUnresolved != 0 && "Expected unresolved operands"); + assert(getNumUnresolved() != 0 && "Expected unresolved operands"); // Check if an operand was resolved. if (!isOperandUnresolved(Old)) { if (isOperandUnresolved(New)) // An operand was un-resolved! - ++NumUnresolved; + setNumUnresolved(getNumUnresolved() + 1); } else if (!isOperandUnresolved(New)) decrementUnresolvedOperandCount(); } @@ -657,7 +661,8 @@ return; assert(isUniqued() && "Expected this to be uniqued"); - if (--NumUnresolved) + setNumUnresolved(getNumUnresolved() - 1); + if (getNumUnresolved()) return; // Last unresolved operand has just been resolved. @@ -732,7 +737,7 @@ } void MDNode::dropAllReferences() { - for (unsigned I = 0, E = NumOperands; I != E; ++I) + for (unsigned I = 0, E = getNumOperands(); I != E; ++I) setOperand(I, nullptr); if (Context.hasReplaceableUses()) { Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false); @@ -868,7 +873,8 @@ assert(ShouldCreate && "Expected non-uniqued nodes to always be created"); } - return storeImpl(new (MDs.size()) MDTuple(Context, Storage, Hash, MDs), + return storeImpl(new (MDs.size(), Storage) + MDTuple(Context, Storage, Hash, MDs), Storage, Context.pImpl->MDTuples); } @@ -880,7 +886,7 @@ void MDNode::storeDistinctInContext() { assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses"); - assert(!NumUnresolved && "Unexpected unresolved nodes"); + assert(!getNumUnresolved() && "Unexpected unresolved nodes"); Storage = Distinct; assert(isResolved() && "Expected this to be resolved"); @@ -913,7 +919,7 @@ } void MDNode::setOperand(unsigned I, Metadata *New) { - assert(I < NumOperands); + assert(I < getNumOperands()); mutable_begin()[I].reset(New, isUniqued() ? this : nullptr); }