diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h --- a/llvm/include/llvm/ADT/SmallVector.h +++ b/llvm/include/llvm/ADT/SmallVector.h @@ -97,6 +97,10 @@ AlignedCharArrayUnion FirstEl; }; +namespace detail { +template class GrowBufferBase; +} // namespace detail + /// This is the part of SmallVectorTemplateBase which does not depend on whether /// the type T is a POD. The extra dummy template argument is used by ArrayRef /// to avoid unnecessarily requiring T to be complete. @@ -104,6 +108,7 @@ class SmallVectorTemplateCommon : public SmallVectorBase> { using Base = SmallVectorBase>; + friend class detail::GrowBufferBase; /// Find the address of the first element. For this pointer math to be valid /// with small-size of 0 for T with lots of alignment, it's important that @@ -132,6 +137,40 @@ this->Size = this->Capacity = 0; // FIXME: Setting Capacity to 0 is suspect. } + size_t grow_size(size_t MinSize = 0) { + // Ensure we can fit the new capacity. + // This is only going to be applicable when the capacity is 32 bit. + if (MinSize > this->SizeTypeMax()) { + std::string Reason = "SmallVector unable to grow. Requested capacity (" + + std::to_string(MinSize) + + ") is larger than maximum value for size type (" + + std::to_string(this->SizeTypeMax()) + ")"; +#ifdef LLVM_ENABLE_EXCEPTIONS + throw std::length_error(Reason); +#else + report_fatal_error(Reason); +#endif + } + + // Ensure we can meet the guarantee of space for at least one more element. + // The above check alone will not catch the case where grow is called with a + // default MinSize of 0, but the current capacity cannot be increased. + // This is only going to be applicable when the capacity is 32 bit. + if (this->capacity() == this->SizeTypeMax()) { + std::string Reason = + "SmallVector capacity unable to grow. Already at maximum size " + + std::to_string(this->SizeTypeMax()); +#ifdef LLVM_ENABLE_EXCEPTIONS + throw std::length_error(Reason); +#else + report_fatal_error(Reason); +#endif + } + // Always grow, even from zero. + size_t NewCapacity = size_t(NextPowerOf2(this->capacity() + 2)); + return std::min(std::max(NewCapacity, MinSize), this->SizeTypeMax()); + } + public: using size_type = size_t; using difference_type = ptrdiff_t; @@ -203,6 +242,151 @@ } }; +namespace detail { +template +static constexpr bool + is_pod_like_v = (is_trivially_copy_constructible::value) && + (is_trivially_move_constructible::value) && + std::is_trivially_destructible::value; + +template class GrowBufferBase { +public: + using Size_T = SmallVectorSizeType; + + GrowBufferBase(Size_T Size, Size_T Capacity) + : Size(Size), Capacity(Capacity) { + BeginX = llvm::safe_malloc(sizeof(T) * Capacity); + } + + ~GrowBufferBase() { + assert(BeginX == nullptr && "Buffers haven't been swapped out"); + } + +protected: + T *begin() { return static_cast(BeginX); } + T *end() { return begin() + Size; } + void *BeginX; + Size_T Size, Capacity; + + void take_buffers(SmallVectorTemplateCommon &Vec) { + if (!Vec.isSmall()) + free(Vec.begin()); + Vec.BeginX = BeginX; + Vec.Size = Size; + Vec.Capacity = Capacity; + BeginX = nullptr; + Size = 0U; + Capacity = 0U; + } +}; + +template > +class GrowBuffer : public GrowBufferBase { +public: + using GrowBufferBase::GrowBufferBase; + + using Size_T = typename GrowBufferBase::Size_T; + void default_construct_at_end(Size_T Count) { + assert((this->Size + Count) <= this->Capacity && + "Overflowed GrowBuffer storage"); + T *First = this->end(); + this->Size += Count; + T *Last = this->end(); + for (; First != Last; ++First) { + new (static_cast(First)) T; + } + } + + void push_back(const T &Elt) { + assert(this->Size < this->Capacity && "Overflowed GrowBuffer storage"); + new (static_cast(this->end())) T(Elt); + ++this->Size; + } + + void push_back(T &&Elt) { + assert(this->Size < this->Capacity && "Overflowed GrowBuffer storage"); + new (static_cast(this->end())) T(std::move(Elt)); + ++this->Size; + } + + void append(Size_T Count, const T &Elt) { + assert((this->Size + Count) <= this->Capacity && + "Overflowed GrowBuffer storage"); + std::uninitialized_fill_n(this->end(), Count, Elt); + this->Size += Count; + } + + void swap_out_buffer(SmallVectorTemplateCommon &Vec) { + auto Begin = std::make_move_iterator(Vec.rbegin()); + auto End = std::make_move_iterator(Vec.rend()); + auto Src = std::reverse_iterator(this->begin() + Vec.size()); + std::uninitialized_copy(Begin, End, Src); + // Destory the items in the old vector. + std::for_each(Vec.begin(), Vec.end(), [](T &I) { I.~T(); }); + this->take_buffers(Vec); + } + + void swap_out_split_buffer(SmallVectorTemplateCommon &Vec, + Size_T SplitIndex) { + std::uninitialized_copy( + std::make_move_iterator( + std::reverse_iterator(Vec.begin() + SplitIndex)), + std::make_move_iterator(Vec.rend()), + std::reverse_iterator(this->begin() + SplitIndex)); + std::uninitialized_copy(std::make_move_iterator(&Vec[SplitIndex]), + std::make_move_iterator(Vec.end()), this->end()); + this->Size += Vec.size() - SplitIndex; + // Destory the items in the old vector. + std::for_each(Vec.begin(), Vec.end(), [](T &I) { I.~T(); }); + this->take_buffers(Vec); + } +}; + +template class GrowBuffer : public GrowBufferBase { +public: + using GrowBufferBase::GrowBufferBase; + using Size_T = typename GrowBufferBase::Size_T; + void default_construct_at_end(Size_T Count) { + assert((this->Size + Count) <= this->Capacity && + "Overflowed GrowBuffer storage"); + // No need to do any construction here. + this->Size += Count; + } + + void push_back(const T &Elt) { + assert(this->Size < this->Capacity && "Overflowed GrowBuffer storage"); + memcpy(this->end(), &Elt, sizeof(Elt)); + ++this->Size; + } + + void append(Size_T Count, const T &Elt) { + assert((this->Size + Count) <= this->Capacity && + "Overflowed GrowBuffer storage"); + T *First = this->end(); + this->Size += Count; + T *Last = this->end(); + for (; First != Last; ++First) { + memcpy(First, &Elt, sizeof(Elt)); + } + } + + void swap_out_buffer(SmallVectorTemplateCommon &Vec) { + memcpy(this->begin(), Vec.begin(), Vec.size_in_bytes()); + this->take_buffers(Vec); + } + + void swap_out_split_buffer(SmallVectorTemplateCommon &Vec, + Size_T SplitIndex) { + memcpy(this->begin(), Vec.begin(), SplitIndex * sizeof(T)); + memcpy(this->end(), &Vec[SplitIndex], + (Vec.size() - SplitIndex) * sizeof(T)); + this->Size += Vec.size() - SplitIndex; + this->take_buffers(Vec); + } +}; + +} // namespace detail + /// SmallVectorTemplateBase - This is where we put /// method implementations that are designed to work with non-trivial T's. /// @@ -211,9 +395,7 @@ /// copy these types with memcpy, there is no way for the type to observe this. /// This catches the important case of std::pair, which is not /// trivially assignable. -template ::value) && - (is_trivially_move_constructible::value) && - std::is_trivially_destructible::value> +template > class SmallVectorTemplateBase : public SmallVectorTemplateCommon { protected: SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} @@ -247,15 +429,23 @@ public: void push_back(const T &Elt) { - if (LLVM_UNLIKELY(this->size() >= this->capacity())) - this->grow(); + if (LLVM_UNLIKELY(this->size() >= this->capacity())) { + detail::GrowBuffer Buffer(this->size(), this->grow_size()); + Buffer.push_back(Elt); + Buffer.swap_out_buffer(*this); + return; + } ::new ((void*) this->end()) T(Elt); this->set_size(this->size() + 1); } void push_back(T &&Elt) { - if (LLVM_UNLIKELY(this->size() >= this->capacity())) - this->grow(); + if (LLVM_UNLIKELY(this->size() >= this->capacity())) { + detail::GrowBuffer Buffer(this->size(), this->grow_size()); + Buffer.push_back(std::move(Elt)); + Buffer.swap_out_buffer(*this); + return; + } ::new ((void*) this->end()) T(::std::move(Elt)); this->set_size(this->size() + 1); } @@ -269,37 +459,7 @@ // Define this out-of-line to dissuade the C++ compiler from inlining it. template void SmallVectorTemplateBase::grow(size_t MinSize) { - // Ensure we can fit the new capacity. - // This is only going to be applicable when the capacity is 32 bit. - if (MinSize > this->SizeTypeMax()) { - std::string Reason = "SmallVector unable to grow. Requested capacity (" + - std::to_string(MinSize) + - ") is larger than maximum value for size type (" + - std::to_string(this->SizeTypeMax()) + ")"; -#ifdef LLVM_ENABLE_EXCEPTIONS - throw std::length_error(Reason); -#else - report_fatal_error(Reason); -#endif - } - - // Ensure we can meet the guarantee of space for at least one more element. - // The above check alone will not catch the case where grow is called with a - // default MinSize of 0, but the current capacity cannot be increased. - // This is only going to be applicable when the capacity is 32 bit. - if (this->capacity() == this->SizeTypeMax()) { - std::string Reason = - "SmallVector capacity unable to grow. Already at maximum size " + - std::to_string(this->SizeTypeMax()); -#ifdef LLVM_ENABLE_EXCEPTIONS - throw std::length_error(Reason); -#else - report_fatal_error(Reason); -#endif - } - // Always grow, even from zero. - size_t NewCapacity = size_t(NextPowerOf2(this->capacity() + 2)); - NewCapacity = std::min(std::max(NewCapacity, MinSize), this->SizeTypeMax()); + size_t NewCapacity = this->grow_size(MinSize); T *NewElts = static_cast(llvm::safe_malloc(NewCapacity*sizeof(T))); // Move the elements over. @@ -365,8 +525,12 @@ public: void push_back(const T &Elt) { - if (LLVM_UNLIKELY(this->size() >= this->capacity())) - this->grow(); + if (LLVM_UNLIKELY(this->size() >= this->capacity())) { + detail::GrowBuffer Buffer(this->size(), this->grow_size()); + Buffer.push_back(Elt); + Buffer.swap_out_buffer(*this); + return; + } memcpy(reinterpret_cast(this->end()), &Elt, sizeof(T)); this->set_size(this->size() + 1); } @@ -424,8 +588,12 @@ this->destroy_range(this->begin()+N, this->end()); this->set_size(N); } else if (N > this->size()) { - if (this->capacity() < N) - this->grow(N); + if (this->capacity() < N) { + detail::GrowBuffer Buffer(this->size(), this->grow_size(N)); + Buffer.append(N - this->size(), NV); + Buffer.swap_out_buffer(*this); + return; + } std::uninitialized_fill(this->end(), this->begin()+N, NV); this->set_size(N); } @@ -460,8 +628,13 @@ /// Append \p NumInputs copies of \p Elt to the end. void append(size_type NumInputs, const T &Elt) { - if (NumInputs > this->capacity() - this->size()) - this->grow(this->size()+NumInputs); + if (NumInputs > this->capacity() - this->size()) { + detail::GrowBuffer Buffer(this->size(), + this->grow_size(this->size() + NumInputs)); + Buffer.append(NumInputs, Elt); + Buffer.swap_out_buffer(*this); + return; + } std::uninitialized_fill_n(this->end(), NumInputs, Elt); this->set_size(this->size() + NumInputs); @@ -540,8 +713,10 @@ if (this->size() >= this->capacity()) { size_t EltNo = I-this->begin(); - this->grow(); - I = this->begin()+EltNo; + detail::GrowBuffer Buffer(EltNo, this->grow_size()); + Buffer.push_back(std::move(Elt)); + Buffer.swap_out_split_buffer(*this, EltNo); + return this->begin() + EltNo; } ::new ((void*) this->end()) T(::std::move(this->back())); @@ -570,8 +745,10 @@ if (this->size() >= this->capacity()) { size_t EltNo = I-this->begin(); - this->grow(); - I = this->begin()+EltNo; + detail::GrowBuffer Buffer(EltNo, this->grow_size()); + Buffer.push_back(Elt); + Buffer.swap_out_split_buffer(*this, EltNo); + return this->begin() + EltNo; } ::new ((void*) this->end()) T(std::move(this->back())); // Push everything else over.