Optimisations and bug-fixes for BitVector and SmallBitVector

[bmoody]

Summary

The main purpose of this is to address the TODO in SmallBitVector to store
the large mode in a single allocation. I made some other optimisations and
fixed a few bugs while I was at it.

Behaviour changes:

• operator== now returns false when sizes are not equal
• Hashing
  • Fixed bug where BitVectors with equal contents but different capacities would get different hashes
  • Fixed bug where SmallBitVector would hash differently in small and large modes
  • Allow empty BitVectors and SmallBitVectors to be used as DenseMap keys
• BitVector::getData now returns only in-use words

Internal changes:

• SmallBitVector Large-mode now lives in a single allocation
• sizeof(BitVector) is now 16, previously 24
• BitVector(unsigned, bool) no longer allocates when s==0
• Most uses of Bits now call getActiveBits, returning only the BitWords currently in use.
• set_unused_bits now only affects bits in the top used word. Unused BitWords are now uninitialised, which should make many operations faster on very over-allocated vectors. This change should be safe because of the above change - accessing the uninitialised data should trip a bounds check. See commit f93a82f08f29163c342f320b88710b8eb0beb804 which introduced this behaviour
• grow no longer initialises data
• Re-implemented the two-vector operations in SmallBitVector to improve performance in the mixed small/large case
• SmallBitVector::getSmallBits now masks out size bits only. setSmallSize already ensures that unused data bits are zero.

Also included a bug-fix to RegScavenger, which started tripping an assert that I added. This
change should probably go in on its own but I don't know how to write a test exercising that
code, or who might be able to help me.

[nikic]

Behaviour changes:

I'd recommend to split out these changes (together with the corresponding test additions) into a separate revision, so functional changes and performance improvements are in separate commits.

[bmoody]

I've made another review request for the bug-fixes at https://reviews.llvm.org/D77027. Would it be better to update this review or abandon and make a new one once the other changes are accepted?

llvm/include/llvm/ADT/BitVector.h

public:
 
   bool operator!=(const const_set_bits_iterator_impl &Other) const {
     assert(&Parent == &Other.Parent &&
            "Comparing iterators from different BitVectors");
     return Current != Other.Current;
   }
 };
 
-class BitVector {
+class BitVectorBase {
+protected:
   typedef uintptr_t BitWord;
 
   enum { BITWORD_SIZE = (unsigned)sizeof(BitWord) * CHAR_BIT };
 
   static_assert(BITWORD_SIZE == 64 || BITWORD_SIZE == 32,
                 "Unsupported word size");
 
-  MutableArrayRef<BitWord> Bits; // Actual bits.
-  unsigned Size;                 // Size of bitvector in bits.
+  static unsigned NumBitWords(unsigned S) {
+    return (S + BITWORD_SIZE - 1) / BITWORD_SIZE;
+  }
 
 public:
   typedef unsigned size_type;
+
   // Encapsulation of a single bit.
   class reference {
     friend class BitVector;
 
     BitWord *WordRef;
     unsigned BitPos;
 
   public:
-    reference(BitVector &b, unsigned Idx) {
-      WordRef = &b.Bits[Idx / BITWORD_SIZE];
+    reference(BitWord *Bits, unsigned Idx) {
+      WordRef = Bits + (Idx / BITWORD_SIZE);
       BitPos = Idx % BITWORD_SIZE;
     }
 
     reference() = delete;
     reference(const reference&) = default;
 
     reference &operator=(reference t) {
       *this = bool(t);
       return *this;
     }
 
     reference& operator=(bool t) {
       if (t)
         *WordRef |= BitWord(1) << BitPos;
       else
         *WordRef &= ~(BitWord(1) << BitPos);
       return *this;
     }
 
     operator bool() const {
       return ((*WordRef) & (BitWord(1) << BitPos)) != 0;
     }
   };
+};
+
+/// A CRTP base class for logic and interface shared by BitVector and the
+/// large-mode of SmallBitVector.
+///
+/// The derived class should define the following interface:
+///
+/// class DerivedExample : BitVectorImpl<DerivedExample> {
+///   size_type getSize() const;
+///   void setSize(size_type N);
+///   size_type getCapacity() const;
+///   BitWord *getBitsPtr();
+///   const BitWord *getBitsPtr() const;
+///   void growImpl(size_type NewCapacity);
+/// };
+template <class Derived> class BitVectorImpl : public BitVectorBase {
+  template <class T> friend class BitVectorImpl;
+
+  Derived &self() { return static_cast<Derived &>(*this); }
+  const Derived &self() const { return static_cast<const Derived &>(*this); }
+
+protected:
+  MutableArrayRef<BitWord> getBits() {
+    return MutableArrayRef<BitWord>(self().getBitsPtr(), self().getCapacity());
+  }
+  ArrayRef<BitWord> getActiveBits() const {
+    return ArrayRef<BitWord>(self().getBitsPtr(),
+                             NumBitWords(self().getSize()));
+  }
+  MutableArrayRef<BitWord> getActiveBits() {
+    return MutableArrayRef<BitWord>(self().getBitsPtr(),
+                                    NumBitWords(self().getSize()));
+  }
 
-  typedef const_set_bits_iterator_impl<BitVector> const_set_bits_iterator;
+public:
+  typedef const_set_bits_iterator_impl<Derived> const_set_bits_iterator;
   typedef const_set_bits_iterator set_iterator;
 
   const_set_bits_iterator set_bits_begin() const {
-    return const_set_bits_iterator(*this);
+    return const_set_bits_iterator(self());
   }
   const_set_bits_iterator set_bits_end() const {
-    return const_set_bits_iterator(*this, -1);
+    return const_set_bits_iterator(self(), -1);
   }
   iterator_range<const_set_bits_iterator> set_bits() const {
     return make_range(set_bits_begin(), set_bits_end());
   }
 
-  /// BitVector default ctor - Creates an empty bitvector.
-  BitVector() : Size(0) {}
-
-  /// BitVector ctor - Creates a bitvector of specified number of bits. All
-  /// bits are initialized to the specified value.
-  explicit BitVector(unsigned s, bool t = false) : Size(s) {
-    size_t Capacity = NumBitWords(s);
-    Bits = allocate(Capacity);
-    init_words(Bits, t);
-    if (t)
-      clear_unused_bits();
-  }
-
-  /// BitVector copy ctor.
-  BitVector(const BitVector &RHS) : Size(RHS.size()) {
-    if (Size == 0) {
-      Bits = MutableArrayRef<BitWord>();
-      return;
-    }
-
-    size_t Capacity = NumBitWords(RHS.size());
-    Bits = allocate(Capacity);
-    std::memcpy(Bits.data(), RHS.Bits.data(), Capacity * sizeof(BitWord));
-  }
-
-  BitVector(BitVector &&RHS) : Bits(RHS.Bits), Size(RHS.Size) {
-    RHS.Bits = MutableArrayRef<BitWord>();
-    RHS.Size = 0;
-  }
-
-  ~BitVector() { std::free(Bits.data()); }
-
   /// empty - Tests whether there are no bits in this bitvector.
-  bool empty() const { return Size == 0; }
+  bool empty() const { return size() == 0; }
 
   /// size - Returns the number of bits in this bitvector.
-  size_type size() const { return Size; }
+  size_type size() const { return self().getSize(); }
 
   /// count - Returns the number of bits which are set.
   size_type count() const {
     unsigned NumBits = 0;
-    for (unsigned i = 0; i < NumBitWords(size()); ++i)
-      NumBits += countPopulation(Bits[i]);
+    for (BitWord Word : getActiveBits())
+      NumBits += countPopulation(Word);
     return NumBits;
   }
 
   /// any - Returns true if any bit is set.
   bool any() const {
-    for (unsigned i = 0; i < NumBitWords(size()); ++i)
-      if (Bits[i] != 0)
+    for (BitWord Word : getActiveBits())
+      if (Word != 0)
         return true;
     return false;
   }
 
   /// all - Returns true if all bits are set.
   bool all() const {
+    ArrayRef<BitWord> Bits = getActiveBits();
+    unsigned Size = size();
     for (unsigned i = 0; i < Size / BITWORD_SIZE; ++i)
       if (Bits[i] != ~BitWord(0))
         return false;
 
     // If bits remain check that they are ones. The unused bits are always zero.
     if (unsigned Remainder = Size % BITWORD_SIZE)
       return Bits[Size / BITWORD_SIZE] == (BitWord(1) << Remainder) - 1;
 
     return true;
   }
 
   /// none - Returns true if none of the bits are set.
-  bool none() const {
-    return !any();
-  }
+  bool none() const { return !any(); }
 
   /// find_first_in - Returns the index of the first set bit in the range
   /// [Begin, End).  Returns -1 if all bits in the range are unset.
   int find_first_in(unsigned Begin, unsigned End) const {
-    assert(Begin <= End && End <= Size);
+    ArrayRef<BitWord> Bits = getActiveBits();
+    assert(Begin <= End && End <= size());
     if (Begin == End)
       return -1;
 
     unsigned FirstWord = Begin / BITWORD_SIZE;
     unsigned LastWord = (End - 1) / BITWORD_SIZE;
 
     // Check subsequent words.
     for (unsigned i = FirstWord; i <= LastWord; ++i) {
for (unsigned i = FirstWord; i <= LastWord; ++i) {
         return i * BITWORD_SIZE + countTrailingZeros(Copy);
     }
     return -1;
   }
 
   /// find_last_in - Returns the index of the last set bit in the range
   /// [Begin, End).  Returns -1 if all bits in the range are unset.
   int find_last_in(unsigned Begin, unsigned End) const {
-    assert(Begin <= End && End <= Size);
+    ArrayRef<BitWord> Bits = getActiveBits();
+    assert(Begin <= End && End <= size());
     if (Begin == End)
       return -1;
 
     unsigned LastWord = (End - 1) / BITWORD_SIZE;
     unsigned FirstWord = Begin / BITWORD_SIZE;
 
     for (unsigned i = LastWord + 1; i >= FirstWord + 1; --i) {
       unsigned CurrentWord = i - 1;
int find_last_in(unsigned Begin, unsigned End) const {
     }
 
     return -1;
   }
 
   /// find_first_unset_in - Returns the index of the first unset bit in the
   /// range [Begin, End).  Returns -1 if all bits in the range are set.
   int find_first_unset_in(unsigned Begin, unsigned End) const {
+    size_type Size = size();
+    ArrayRef<BitWord> Bits = getActiveBits();
     assert(Begin <= End && End <= Size);
     if (Begin == End)
       return -1;
 
     unsigned FirstWord = Begin / BITWORD_SIZE;
     unsigned LastWord = (End - 1) / BITWORD_SIZE;
 
     // Check subsequent words.
     for (unsigned i = FirstWord; i <= LastWord; ++i) {
       BitWord Copy = Bits[i];
 
       if (i == FirstWord) {
         unsigned FirstBit = Begin % BITWORD_SIZE;
         Copy |= maskTrailingOnes<BitWord>(FirstBit);
       }
 
       if (i == LastWord) {
         unsigned LastBit = (End - 1) % BITWORD_SIZE;
         Copy |= maskTrailingZeros<BitWord>(LastBit + 1);
       }
       if (Copy != ~BitWord(0)) {
         unsigned Result = i * BITWORD_SIZE + countTrailingOnes(Copy);
-        return Result < size() ? Result : -1;
+        return Result < Size ? Result : -1;
       }
     }
     return -1;
   }
 
   /// find_last_unset_in - Returns the index of the last unset bit in the
   /// range [Begin, End).  Returns -1 if all bits in the range are set.
   int find_last_unset_in(unsigned Begin, unsigned End) const {
+    size_type Size = size();
+    ArrayRef<BitWord> Bits = getActiveBits();
     assert(Begin <= End && End <= Size);
     if (Begin == End)
       return -1;
 
     unsigned LastWord = (End - 1) / BITWORD_SIZE;
     unsigned FirstWord = Begin / BITWORD_SIZE;
 
     for (unsigned i = LastWord + 1; i >= FirstWord + 1; --i) {
for (unsigned i = LastWord + 1; i >= FirstWord + 1; --i) {
         return Result < Size ? Result : -1;
       }
     }
     return -1;
   }
 
   /// find_first - Returns the index of the first set bit, -1 if none
   /// of the bits are set.
-  int find_first() const { return find_first_in(0, Size); }
+  int find_first() const { return find_first_in(0, size()); }
 
   /// find_last - Returns the index of the last set bit, -1 if none of the bits
   /// are set.
-  int find_last() const { return find_last_in(0, Size); }
+  int find_last() const { return find_last_in(0, size()); }
 
   /// find_next - Returns the index of the next set bit following the
   /// "Prev" bit. Returns -1 if the next set bit is not found.
-  int find_next(unsigned Prev) const { return find_first_in(Prev + 1, Size); }
+  int find_next(unsigned Prev) const {
+    return find_first_in(Prev + 1, size());
+  }
 
   /// find_prev - Returns the index of the first set bit that precedes the
   /// the bit at \p PriorTo.  Returns -1 if all previous bits are unset.
   int find_prev(unsigned PriorTo) const { return find_last_in(0, PriorTo); }
 
   /// find_first_unset - Returns the index of the first unset bit, -1 if all
   /// of the bits are set.
-  int find_first_unset() const { return find_first_unset_in(0, Size); }
+  int find_first_unset() const { return find_first_unset_in(0, size()); }
 
   /// find_next_unset - Returns the index of the next unset bit following the
   /// "Prev" bit.  Returns -1 if all remaining bits are set.
   int find_next_unset(unsigned Prev) const {
-    return find_first_unset_in(Prev + 1, Size);
+    return find_first_unset_in(Prev + 1, size());
   }
 
   /// find_last_unset - Returns the index of the last unset bit, -1 if all of
   /// the bits are set.
-  int find_last_unset() const { return find_last_unset_in(0, Size); }
+  int find_last_unset() const { return find_last_unset_in(0, size()); }
 
   /// find_prev_unset - Returns the index of the first unset bit that precedes
   /// the bit at \p PriorTo.  Returns -1 if all previous bits are set.
   int find_prev_unset(unsigned PriorTo) {
     return find_last_unset_in(0, PriorTo);
   }
 
   /// clear - Removes all bits from the bitvector. Does not change capacity.
-  void clear() {
-    Size = 0;
-  }
+  void clear() { self().setSize(0); }
 
   /// resize - Grow or shrink the bitvector.
-  void resize(unsigned N, bool t = false) {
-    if (N > getBitCapacity()) {
-      unsigned OldCapacity = Bits.size();
+  void resize(size_type N, bool t = false) {
+    size_type OldSize = size();
+    if (N > getBitCapacity())
       grow(N);
-      init_words(Bits.drop_front(OldCapacity), t);
-    }
 
     // Set any old unused bits that are now included in the BitVector. This
     // may set bits that are not included in the new vector, but we will clear
     // them back out below.
-    if (N > Size)
+    if (N > OldSize) {
+      size_type OldActiveWords = NumBitWords(OldSize);
+      size_type NewlyActiveWords = NumBitWords(N) - OldActiveWords;
       set_unused_bits(t);
+      init_words(getBits().slice(OldActiveWords, NewlyActiveWords), t);
+    }
 
     // Update the size, and clear out any bits that are now unused
-    unsigned OldSize = Size;
-    Size = N;
+    self().setSize(N);
     if (t || N < OldSize)
       clear_unused_bits();
   }
 
   void reserve(unsigned N) {
     if (N > getBitCapacity())
-      grow(N);
+      self().grow(N);
   }
 
   // Set, reset, flip
-  BitVector &set() {
-    init_words(Bits, true);
+  Derived &set() {
+    init_words(getActiveBits(), true);
     clear_unused_bits();
-    return *this;
+    return self();
   }
 
-  BitVector &set(unsigned Idx) {
-    assert(Bits.data() && "Bits never allocated");
+  Derived &set(unsigned Idx) {
+    MutableArrayRef<BitWord> Bits = getActiveBits();
     Bits[Idx / BITWORD_SIZE] |= BitWord(1) << (Idx % BITWORD_SIZE);
-    return *this;
+    return self();
   }
 
   /// set - Efficiently set a range of bits in [I, E)
-  BitVector &set(unsigned I, unsigned E) {
+  Derived &set(unsigned I, unsigned E) {
     assert(I <= E && "Attempted to set backwards range!");
     assert(E <= size() && "Attempted to set out-of-bounds range!");
 
-    if (I == E) return *this;
+    if (I == E)
+      return self();
+
+    MutableArrayRef<BitWord> Bits = getActiveBits();
 
     if (I / BITWORD_SIZE == E / BITWORD_SIZE) {
       BitWord EMask = BitWord(1) << (E % BITWORD_SIZE);
       BitWord IMask = BitWord(1) << (I % BITWORD_SIZE);
       BitWord Mask = EMask - IMask;
       Bits[I / BITWORD_SIZE] |= Mask;
-      return *this;
+      return self();
     }
 
     BitWord PrefixMask = ~BitWord(0) << (I % BITWORD_SIZE);
     Bits[I / BITWORD_SIZE] |= PrefixMask;
     I = alignTo(I, BITWORD_SIZE);
 
     for (; I + BITWORD_SIZE <= E; I += BITWORD_SIZE)
       Bits[I / BITWORD_SIZE] = ~BitWord(0);
 
     BitWord PostfixMask = (BitWord(1) << (E % BITWORD_SIZE)) - 1;
     if (I < E)
       Bits[I / BITWORD_SIZE] |= PostfixMask;
 
-    return *this;
+    return self();
   }
 
-  BitVector &reset() {
-    init_words(Bits, false);
-    return *this;
+  Derived &reset() {
+    init_words(getActiveBits(), false);
+    return self();
   }
 
-  BitVector &reset(unsigned Idx) {
-    Bits[Idx / BITWORD_SIZE] &= ~(BitWord(1) << (Idx % BITWORD_SIZE));
-    return *this;
+  Derived &reset(unsigned Idx) {
+    getActiveBits()[Idx / BITWORD_SIZE] &=
+        ~(BitWord(1) << (Idx % BITWORD_SIZE));
+    return self();
   }
 
   /// reset - Efficiently reset a range of bits in [I, E)
-  BitVector &reset(unsigned I, unsigned E) {
+  Derived &reset(unsigned I, unsigned E) {
     assert(I <= E && "Attempted to reset backwards range!");
     assert(E <= size() && "Attempted to reset out-of-bounds range!");
 
-    if (I == E) return *this;
+    if (I == E)
+      return self();
+
+    MutableArrayRef<BitWord> Bits = getActiveBits();
 
     if (I / BITWORD_SIZE == E / BITWORD_SIZE) {
       BitWord EMask = BitWord(1) << (E % BITWORD_SIZE);
       BitWord IMask = BitWord(1) << (I % BITWORD_SIZE);
       BitWord Mask = EMask - IMask;
       Bits[I / BITWORD_SIZE] &= ~Mask;
-      return *this;
+      return self();
     }
 
     BitWord PrefixMask = ~BitWord(0) << (I % BITWORD_SIZE);
     Bits[I / BITWORD_SIZE] &= ~PrefixMask;
     I = alignTo(I, BITWORD_SIZE);
 
     for (; I + BITWORD_SIZE <= E; I += BITWORD_SIZE)
       Bits[I / BITWORD_SIZE] = BitWord(0);
 
     BitWord PostfixMask = (BitWord(1) << (E % BITWORD_SIZE)) - 1;
     if (I < E)
       Bits[I / BITWORD_SIZE] &= ~PostfixMask;
 
-    return *this;
+    return self();
   }
 
-  BitVector &flip() {
-    for (unsigned i = 0; i < NumBitWords(size()); ++i)
-      Bits[i] = ~Bits[i];
+  Derived &flip() {
+    for (BitWord &Word : getActiveBits()) {
+      Word = ~Word;
+    }
     clear_unused_bits();
-    return *this;
+    return self();
   }
 
-  BitVector &flip(unsigned Idx) {
-    Bits[Idx / BITWORD_SIZE] ^= BitWord(1) << (Idx % BITWORD_SIZE);
-    return *this;
+  Derived &flip(unsigned Idx) {
+    getActiveBits()[Idx / BITWORD_SIZE] ^= BitWord(1) << (Idx % BITWORD_SIZE);
+    return self();
   }
 
   // Indexing.
   reference operator[](unsigned Idx) {
-    assert (Idx < Size && "Out-of-bounds Bit access.");
-    return reference(*this, Idx);
+    assert(Idx < size() && "Out-of-bounds Bit access.");
+    return reference(self().getBitsPtr(), Idx);
   }
 
   bool operator[](unsigned Idx) const {
-    assert (Idx < Size && "Out-of-bounds Bit access.");
+    assert(Idx < size() && "Out-of-bounds Bit access.");
     BitWord Mask = BitWord(1) << (Idx % BITWORD_SIZE);
-    return (Bits[Idx / BITWORD_SIZE] & Mask) != 0;
+    return (getActiveBits()[Idx / BITWORD_SIZE] & Mask) != 0;
   }
 
   bool test(unsigned Idx) const {
     return (*this)[Idx];
   }
 
   // Push single bit to end of vector.
-  void push_back(bool Val) {
-    unsigned OldSize = Size;
-    unsigned NewSize = Size + 1;
-
-    // Resize, which will insert zeros.
-    // If we already fit then the unused bits will be already zero.
-    if (NewSize > getBitCapacity())
-      resize(NewSize, false);
-    else
-      Size = NewSize;
-
-    // If true, set single bit.
-    if (Val)
-      set(OldSize);
-  }
+  void push_back(bool Val) { resize(size() + 1, Val); }
 
   /// Test if any common bits are set.
-  bool anyCommon(const BitVector &RHS) const {
-    unsigned ThisWords = NumBitWords(size());
-    unsigned RHSWords  = NumBitWords(RHS.size());
+  bool anyCommon(const Derived &RHS) const {
+    ArrayRef<BitWord> ThisBits = getActiveBits();
+    ArrayRef<BitWord> RHSBits = RHS.getActiveBits();
+    unsigned ThisWords = ThisBits.size();
+    unsigned RHSWords = RHSBits.size();
     for (unsigned i = 0, e = std::min(ThisWords, RHSWords); i != e; ++i)
-      if (Bits[i] & RHS.Bits[i])
+      if (ThisBits[i] & RHSBits[i])
         return true;
     return false;
   }
 
   // Comparison operators.
-  bool operator==(const BitVector &RHS) const {
-    unsigned ThisWords = NumBitWords(size());
-    unsigned RHSWords  = NumBitWords(RHS.size());
-    unsigned i;
-    for (i = 0; i != std::min(ThisWords, RHSWords); ++i)
-      if (Bits[i] != RHS.Bits[i])
-        return false;
-
-    // Verify that any extra words are all zeros.
-    if (i != ThisWords) {
-      for (; i != ThisWords; ++i)
-        if (Bits[i])
-          return false;
-    } else if (i != RHSWords) {
-      for (; i != RHSWords; ++i)
-        if (RHS.Bits[i])
-          return false;
-    }
-    return true;
+  bool operator==(const Derived &RHS) const {
+    return size() == RHS.size() &&
+           getActiveBits() == RHS.getActiveBits();
   }
 
-  bool operator!=(const BitVector &RHS) const {
-    return !(*this == RHS);
-  }
+  bool operator!=(const Derived &RHS) const { return !(*this == RHS); }
 
   /// Intersection, union, disjoint union.
-  BitVector &operator&=(const BitVector &RHS) {
-    unsigned ThisWords = NumBitWords(size());
-    unsigned RHSWords  = NumBitWords(RHS.size());
+  Derived &operator&=(const Derived &RHS) {
+    MutableArrayRef<BitWord> ThisBits = getActiveBits();
+    ArrayRef<BitWord> RHSBits = RHS.getActiveBits();
+    unsigned ThisWords = ThisBits.size();
+    unsigned RHSWords = RHSBits.size();
     unsigned i;
     for (i = 0; i != std::min(ThisWords, RHSWords); ++i)
-      Bits[i] &= RHS.Bits[i];
+      ThisBits[i] &= RHSBits[i];
 
     // Any bits that are just in this bitvector become zero, because they aren't
     // in the RHS bit vector.  Any words only in RHS are ignored because they
     // are already zero in the LHS.
     for (; i != ThisWords; ++i)
-      Bits[i] = 0;
+      ThisBits[i] = 0;
 
-    return *this;
+    return self();
   }
 
   /// reset - Reset bits that are set in RHS. Same as *this &= ~RHS.
-  BitVector &reset(const BitVector &RHS) {
-    unsigned ThisWords = NumBitWords(size());
-    unsigned RHSWords  = NumBitWords(RHS.size());
+  Derived &reset(const Derived &RHS) {
+    MutableArrayRef<BitWord> ThisBits = getActiveBits();
+    ArrayRef<BitWord> RHSBits = RHS.getActiveBits();
+    unsigned ThisWords = ThisBits.size();
+    unsigned RHSWords = RHSBits.size();
     unsigned i;
     for (i = 0; i != std::min(ThisWords, RHSWords); ++i)
-      Bits[i] &= ~RHS.Bits[i];
-    return *this;
+      ThisBits[i] &= ~RHSBits[i];
+    return self();
   }
 
   /// test - Check if (This - RHS) is zero.
   /// This is the same as reset(RHS) and any().
-  bool test(const BitVector &RHS) const {
-    unsigned ThisWords = NumBitWords(size());
-    unsigned RHSWords  = NumBitWords(RHS.size());
+  bool test(const Derived &RHS) const {
+    ArrayRef<BitWord> ThisBits = getActiveBits();
+    ArrayRef<BitWord> RHSBits = RHS.getActiveBits();
+    unsigned ThisWords = ThisBits.size();
+    unsigned RHSWords = RHSBits.size();
     unsigned i;
     for (i = 0; i != std::min(ThisWords, RHSWords); ++i)
-      if ((Bits[i] & ~RHS.Bits[i]) != 0)
+      if ((ThisBits[i] & ~RHSBits[i]) != 0)
         return true;
 
-    for (; i != ThisWords ; ++i)
-      if (Bits[i] != 0)
+    for (; i != ThisWords; ++i)
+      if (ThisBits[i] != 0)
         return true;
 
     return false;
   }
 
-  BitVector &operator|=(const BitVector &RHS) {
+  Derived &operator|=(const Derived &RHS) {
     if (size() < RHS.size())
       resize(RHS.size());
-    for (size_t i = 0, e = NumBitWords(RHS.size()); i != e; ++i)
-      Bits[i] |= RHS.Bits[i];
-    return *this;
+    MutableArrayRef<BitWord> ThisBits = getActiveBits();
+    ArrayRef<BitWord> RHSBits = RHS.getActiveBits();
+    for (size_t i = 0, e = RHSBits.size(); i != e; ++i)
+      ThisBits[i] |= RHSBits[i];
+    return self();
   }
 
-  BitVector &operator^=(const BitVector &RHS) {
+  Derived &operator^=(const Derived &RHS) {
     if (size() < RHS.size())
       resize(RHS.size());
-    for (size_t i = 0, e = NumBitWords(RHS.size()); i != e; ++i)
-      Bits[i] ^= RHS.Bits[i];
-    return *this;
+    MutableArrayRef<BitWord> ThisBits = getActiveBits();
+    ArrayRef<BitWord> RHSBits = RHS.getActiveBits();
+    for (size_t i = 0, e = RHSBits.size(); i != e; ++i)
+      ThisBits[i] ^= RHSBits[i];
+    return self();
   }
 
-  BitVector &operator>>=(unsigned N) {
-    assert(N <= Size);
+  Derived &operator>>=(unsigned N) {
+    assert(N <= size());
     if (LLVM_UNLIKELY(empty() || N == 0))
-      return *this;
+      return self();
 
-    unsigned NumWords = NumBitWords(Size);
+    MutableArrayRef<BitWord> Bits = getActiveBits();
+    unsigned NumWords = Bits.size();
     assert(NumWords >= 1);
 
     wordShr(N / BITWORD_SIZE);
 
     unsigned BitDistance = N % BITWORD_SIZE;
     if (BitDistance == 0)
-      return *this;
+      return self();
 
     // When the shift size is not a multiple of the word size, then we have
     // a tricky situation where each word in succession needs to extract some
     // of the bits from the next word and or them into this word while
     // shifting this word to make room for the new bits.  This has to be done
     // for every word in the array.
 
     // Since we're shifting each word right, some bits will fall off the end
Derived &operator>>=(unsigned N) {
 
     for (unsigned I = 0; I < NumWords - 1; ++I) {
       Bits[I] >>= BitDistance;
       Bits[I] |= (Bits[I + 1] & Mask) << LSH;
     }
 
     Bits[NumWords - 1] >>= BitDistance;
 
-    return *this;
+    return self();
   }
 
-  BitVector &operator<<=(unsigned N) {
-    assert(N <= Size);
+  Derived &operator<<=(unsigned N) {
+    assert(N <= size());
     if (LLVM_UNLIKELY(empty() || N == 0))
-      return *this;
+      return self();
 
-    unsigned NumWords = NumBitWords(Size);
+    MutableArrayRef<BitWord> Bits = getActiveBits();
+    unsigned NumWords = Bits.size();
     assert(NumWords >= 1);
 
     wordShl(N / BITWORD_SIZE);
 
     unsigned BitDistance = N % BITWORD_SIZE;
     if (BitDistance == 0)
-      return *this;
+      return self();
 
     // When the shift size is not a multiple of the word size, then we have
     // a tricky situation where each word in succession needs to extract some
     // of the bits from the previous word and or them into this word while
     // shifting this word to make room for the new bits.  This has to be done
     // for every word in the array.  This is similar to the algorithm outlined
     // in operator>>=, but backwards.
 
Derived &operator<<=(unsigned N) {
 
     for (int I = NumWords - 1; I > 0; --I) {
       Bits[I] <<= BitDistance;
       Bits[I] |= (Bits[I - 1] & Mask) >> RSH;
     }
     Bits[0] <<= BitDistance;
     clear_unused_bits();
 
-    return *this;
-  }
-
-  // Assignment operator.
-  const BitVector &operator=(const BitVector &RHS) {
-    if (this == &RHS) return *this;
-
-    Size = RHS.size();
-
-    // Handle tombstone when the BitVector is a key of a DenseHash.
-    if (RHS.isInvalid()) {
-      std::free(Bits.data());
-      Bits = None;
-      return *this;
-    }
-
-    unsigned RHSWords = NumBitWords(Size);
-    if (Size <= getBitCapacity()) {
-      if (Size)
-        std::memcpy(Bits.data(), RHS.Bits.data(), RHSWords * sizeof(BitWord));
-      clear_unused_bits();
-      return *this;
-    }
-
-    // Grow the bitvector to have enough elements.
-    unsigned NewCapacity = RHSWords;
-    assert(NewCapacity > 0 && "negative capacity?");
-    auto NewBits = allocate(NewCapacity);
-    std::memcpy(NewBits.data(), RHS.Bits.data(), NewCapacity * sizeof(BitWord));
-
-    // Destroy the old bits.
-    std::free(Bits.data());
-    Bits = NewBits;
-
-    return *this;
-  }
-
-  const BitVector &operator=(BitVector &&RHS) {
-    if (this == &RHS) return *this;
-
-    std::free(Bits.data());
-    Bits = RHS.Bits;
-    Size = RHS.Size;
-
-    RHS.Bits = MutableArrayRef<BitWord>();
-    RHS.Size = 0;
-
-    return *this;
-  }
-
-  void swap(BitVector &RHS) {
-    std::swap(Bits, RHS.Bits);
-    std::swap(Size, RHS.Size);
-  }
-
-  void invalid() {
-    assert(!Size && Bits.empty());
-    Size = (unsigned)-1;
+    return self();
   }
-  bool isInvalid() const { return Size == (unsigned)-1; }
 
-  ArrayRef<BitWord> getData() const { return Bits; }
+  ArrayRef<BitWord> getData() const { return getActiveBits(); }
 
   //===--------------------------------------------------------------------===//
   // Portable bit mask operations.
   //===--------------------------------------------------------------------===//
   //
   // These methods all operate on arrays of uint32_t, each holding 32 bits. The
   // fixed word size makes it easier to work with literal bit vector constants
   // in portable code.
public:
   }
 
   /// clearBitsNotInMask - Clear a bit in this vector for every '0' bit in Mask.
   /// Don't resize.  This computes "*this &= Mask".
   void clearBitsNotInMask(const uint32_t *Mask, unsigned MaskWords = ~0u) {
     applyMask<false, true>(Mask, MaskWords);
   }
 
-private:
+protected:
   /// Perform a logical left shift of \p Count words by moving everything
   /// \p Count words to the right in memory.
   ///
   /// While confusing, words are stored from least significant at Bits[0] to
   /// most significant at Bits[NumWords-1].  A logical shift left, however,
   /// moves the current least significant bit to a higher logical index, and
   /// fills the previous least significant bits with 0.  Thus, we actually
   /// need to move the bytes of the memory to the right, not to the left.
   /// Example:
   ///   Words = [0xBBBBAAAA, 0xDDDDFFFF, 0x00000000, 0xDDDD0000]
   /// represents a BitVector where 0xBBBBAAAA contain the least significant
   /// bits.  So if we want to shift the BitVector left by 2 words, we need to
   /// turn this into 0x00000000 0x00000000 0xBBBBAAAA 0xDDDDFFFF by using a
   /// memmove which moves right, not left.
   void wordShl(uint32_t Count) {
     if (Count == 0)
       return;
 
-    uint32_t NumWords = NumBitWords(Size);
-
-    auto Src = Bits.take_front(NumWords).drop_back(Count);
-    auto Dest = Bits.take_front(NumWords).drop_front(Count);
+    MutableArrayRef<BitWord> Bits = getActiveBits();
+    auto Src = Bits.drop_back(Count);
+    auto Dest = Bits.drop_front(Count);
 
     // Since we always move Word-sized chunks of data with src and dest both
     // aligned to a word-boundary, we don't need to worry about endianness
     // here.
     std::memmove(Dest.begin(), Src.begin(), Dest.size() * sizeof(BitWord));
     std::memset(Bits.data(), 0, Count * sizeof(BitWord));
     clear_unused_bits();
   }
 
   /// Perform a logical right shift of \p Count words by moving those
   /// words to the left in memory.  See wordShl for more information.
   ///
   void wordShr(uint32_t Count) {
     if (Count == 0)
       return;
 
-    uint32_t NumWords = NumBitWords(Size);
-
-    auto Src = Bits.take_front(NumWords).drop_front(Count);
-    auto Dest = Bits.take_front(NumWords).drop_back(Count);
+    MutableArrayRef<BitWord> Bits = getActiveBits();
+    auto Src = Bits.drop_front(Count);
+    auto Dest = Bits.drop_back(Count);
     assert(Dest.size() == Src.size());
 
     std::memmove(Dest.begin(), Src.begin(), Dest.size() * sizeof(BitWord));
     std::memset(Dest.end(), 0, Count * sizeof(BitWord));
   }
 
-  MutableArrayRef<BitWord> allocate(size_t NumWords) {
-    BitWord *RawBits = static_cast<BitWord *>(
-        safe_malloc(NumWords * sizeof(BitWord)));
-    return MutableArrayRef<BitWord>(RawBits, NumWords);
-  }
-
-  int next_unset_in_word(int WordIndex, BitWord Word) const {
-    unsigned Result = WordIndex * BITWORD_SIZE + countTrailingOnes(Word);
-    return Result < size() ? Result : -1;
-  }
-
-  unsigned NumBitWords(unsigned S) const {
-    return (S + BITWORD_SIZE-1) / BITWORD_SIZE;
-  }
-
-  // Set the unused bits in the high words.
+  // Set the unused bits in the high word.
   void set_unused_bits(bool t = true) {
-    //  Set high words first.
-    unsigned UsedWords = NumBitWords(Size);
-    if (Bits.size() > UsedWords)
-      init_words(Bits.drop_front(UsedWords), t);
-
-    //  Then set any stray high bits of the last used word.
-    unsigned ExtraBits = Size % BITWORD_SIZE;
-    if (ExtraBits) {
+    if (unsigned ExtraBits = size() % BITWORD_SIZE) {
       BitWord ExtraBitMask = ~BitWord(0) << ExtraBits;
       if (t)
-        Bits[UsedWords-1] |= ExtraBitMask;
+        getActiveBits().back() |= ExtraBitMask;
       else
-        Bits[UsedWords-1] &= ~ExtraBitMask;
+        getActiveBits().back() &= ~ExtraBitMask;
     }
   }
 
-  // Clear the unused bits in the high words.
+  // Clear the unused bits in the high word.
   void clear_unused_bits() {
     set_unused_bits(false);
   }
 
+  // Increase capacity. This leaves newly allocated data uninitialised. The
+  // caller is responsible for initialising the newly allocated memory as
+  // appropriate.
   void grow(unsigned NewSize) {
-    size_t NewCapacity = std::max<size_t>(NumBitWords(NewSize), Bits.size() * 2);
+    size_type NewCapacity =
+        std::max<size_type>(NumBitWords(NewSize), self().getCapacity() * 2);
     assert(NewCapacity > 0 && "realloc-ing zero space");
-    BitWord *NewBits = static_cast<BitWord *>(
-        safe_realloc(Bits.data(), NewCapacity * sizeof(BitWord)));
-    Bits = MutableArrayRef<BitWord>(NewBits, NewCapacity);
-    clear_unused_bits();
+    // growImpl is required to also update the capacity.
+    self().growImpl(NewCapacity);
   }
 
-  void init_words(MutableArrayRef<BitWord> B, bool t) {
+  static void init_words(MutableArrayRef<BitWord> B, bool t) {
     if (B.size() > 0)
       memset(B.data(), 0 - (int)t, B.size() * sizeof(BitWord));
   }
 
   template<bool AddBits, bool InvertMask>
   void applyMask(const uint32_t *Mask, unsigned MaskWords) {
     static_assert(BITWORD_SIZE % 32 == 0, "Unsupported BitWord size.");
     MaskWords = std::min(MaskWords, (size() + 31) / 32);
     const unsigned Scale = BITWORD_SIZE / 32;
+    MutableArrayRef<BitWord> Bits = getActiveBits();
     unsigned i;
     for (i = 0; MaskWords >= Scale; ++i, MaskWords -= Scale) {
       BitWord BW = Bits[i];
       // This inner loop should unroll completely when BITWORD_SIZE > 32.
       for (unsigned b = 0; b != BITWORD_SIZE; b += 32) {
         uint32_t M = *Mask++;
         if (InvertMask) M = ~M;
         if (AddBits) BW |=   BitWord(M) << b;
         else         BW &= ~(BitWord(M) << b);
       }
       Bits[i] = BW;
     }
     for (unsigned b = 0; MaskWords; b += 32, --MaskWords) {
       uint32_t M = *Mask++;
       if (InvertMask) M = ~M;
       if (AddBits) Bits[i] |=   BitWord(M) << b;
       else         Bits[i] &= ~(BitWord(M) << b);
     }
     if (AddBits)
       clear_unused_bits();
   }
 
 public:
   /// Return the size (in bytes) of the bit vector.
-  size_t getMemorySize() const { return Bits.size() * sizeof(BitWord); }
-  size_t getBitCapacity() const { return Bits.size() * BITWORD_SIZE; }
+  size_t getMemorySize() const {
+    return self().getCapacity() * sizeof(BitWord);
+  }
+  size_t getBitCapacity() const { return self().getCapacity() * BITWORD_SIZE; }
+};
+
+class BitVector : public BitVectorImpl<BitVector> {
+  friend class BitVectorImpl<BitVector>;
+
+  BitWord *BitsPtr;   // Pointer to data array.
+  size_type Size;     // Size of bitvector in bits.
+  size_type Capacity; // Capacity of data array in BitWords.
+
+  // Methods required by BitVectorImpl
+  size_type getSize() const { return Size; }
+  void setSize(size_type N) { Size = N; }
+
+  size_type getCapacity() const { return Capacity; }
+
+  BitWord *getBitsPtr() { return BitsPtr; }
+  const BitWord *getBitsPtr() const { return BitsPtr; }
+
+  void growImpl(size_type NewCapacity) {
+
+    BitsPtr = static_cast<BitWord *>(
+        safe_realloc(BitsPtr, NewCapacity * sizeof(BitWord)));
+    Capacity = NewCapacity;
+  }
+
+public:
+  /// BitVector default ctor - Creates an empty bitvector.
+  BitVector() : BitsPtr(nullptr), Size(0), Capacity(0) {}
+
+  /// BitVector ctor - Creates a bitvector of specified number of bits. All
+  /// bits are initialized to the specified value.
+  explicit BitVector(unsigned s, bool t = false)
+      : Size(s), Capacity(NumBitWords(s)) {
+    if (Size == 0) {
+      BitsPtr = nullptr;
+      return;
+    }
+    BitsPtr = allocate(Capacity);
+    init_words(getBits(), t);
+    if (t)
+      clear_unused_bits();
+  }
+
+  /// BitVector copy ctor.
+  BitVector(const BitVector &RHS)
+      : Size(RHS.size()), Capacity(NumBitWords(Size)) {
+    if (Size == 0) {
+      BitsPtr = nullptr;
+      return;
+    }
+
+    BitsPtr = allocate(Capacity);
+    std::memcpy(BitsPtr, RHS.BitsPtr, Capacity * sizeof(BitWord));
+  }
+
+  BitVector(BitVector &&RHS)
+      : BitsPtr(RHS.BitsPtr), Size(RHS.Size), Capacity(RHS.Capacity) {
+    RHS.BitsPtr = nullptr;
+    RHS.Size = 0;
+    RHS.Capacity = 0;
+  }
+
+  ~BitVector() { std::free(BitsPtr); }
+
+  /// size - Returns the number of bits in this bitvector.
+  size_type size() const { return Size; }
+
+  // Assignment operator.
+  const BitVector &operator=(const BitVector &RHS) {
+    if (this == &RHS)
+      return *this;
+
+    Size = RHS.size();
+
+    // Handle tombstone when the BitVector is a key of a DenseHash.
+    if (RHS.isInvalid()) {
+      std::free(BitsPtr);
+      BitsPtr = nullptr;
+      Capacity = 0;
+      return *this;
+    }
+
+    unsigned RHSWords = NumBitWords(Size);
+    if (Size <= getBitCapacity()) {
+      if (Size)
+        std::memcpy(BitsPtr, RHS.BitsPtr, RHSWords * sizeof(BitWord));
+      return *this;
+    }
+
+    // Grow the bitvector to have enough elements.
+    unsigned NewCapacity = RHSWords;
+    assert(NewCapacity > 0 && "negative capacity?");
+    BitWord *NewBits = allocate(NewCapacity);
+    std::memcpy(NewBits, RHS.BitsPtr, NewCapacity * sizeof(BitWord));
+
+    // Destroy the old bits.
+    std::free(BitsPtr);
+    BitsPtr = NewBits;
+
+    return *this;
+  }
+
+  const BitVector &operator=(BitVector &&RHS) {
+    if (this == &RHS)
+      return *this;
+
+    std::free(BitsPtr);
+    BitsPtr = RHS.BitsPtr;
+    Size = RHS.Size;
+    Capacity = RHS.Capacity;
+
+    RHS.BitsPtr = nullptr;
+    RHS.Size = 0;
+    RHS.Capacity = 0;
+
+    return *this;
+  }
+
+  void swap(BitVector &RHS) {
+    std::swap(BitsPtr, RHS.BitsPtr);
+    std::swap(Size, RHS.Size);
+    std::swap(Capacity, RHS.Capacity);
+  }
+
+  static BitVector getDenseMapEmptyKey() {
+    BitVector K;
+    K.Size = size_type(-1);
+    return K;
+  }
+  static BitVector getDenseMapTombstoneKey() {
+    BitVector K;
+    K.Size = size_type(-2);
+    return K;
+  }
+  bool isInvalid() const { return Size >= size_type(-2); }
+
+private:
+  BitWord *allocate(size_t NumWords) {
+    return static_cast<BitWord *>(safe_malloc(NumWords * sizeof(BitWord)));
+  }
 };
 
 inline size_t capacity_in_bytes(const BitVector &X) {
   return X.getMemorySize();
 }
 
 template <> struct DenseMapInfo<BitVector> {
-  static inline BitVector getEmptyKey() { return BitVector(); }
+  static inline BitVector getEmptyKey() {
+    return BitVector::getDenseMapEmptyKey();
+  }
   static inline BitVector getTombstoneKey() {
-    BitVector V;
-    V.invalid();
-    return V;
+    return BitVector::getDenseMapTombstoneKey();
   }
   static unsigned getHashValue(const BitVector &V) {
     return DenseMapInfo<std::pair<unsigned, ArrayRef<uintptr_t>>>::getHashValue(
         std::make_pair(V.size(), V.getData()));
   }
   static bool isEqual(const BitVector &LHS, const BitVector &RHS) {
     if (LHS.isInvalid() || RHS.isInvalid())
-      return LHS.isInvalid() == RHS.isInvalid();
+      return LHS.size() == RHS.size();
     return LHS == RHS;
   }
 };
 } // end namespace llvm
 
 namespace std {
   /// Implement std::swap in terms of BitVector swap.
   inline void
   swap(llvm::BitVector &LHS, llvm::BitVector &RHS) {
     LHS.swap(RHS);
   }
 } // end namespace std
 
 #endif // LLVM_ADT_BITVECTOR_H

llvm/include/llvm/ADT/SmallBitVector.h

 namespace llvm {
 
 /// This is a 'bitvector' (really, a variable-sized bit array), optimized for
 /// the case when the array is small. It contains one pointer-sized field, which
 /// is directly used as a plain collection of bits when possible, or as a
 /// pointer to a larger heap-allocated array when necessary. This allows normal
 /// "small" cases to be fast without losing generality for large inputs.
 class SmallBitVector {
-  // TODO: In "large" mode, a pointer to a BitVector is used, leading to an
-  // unnecessary level of indirection. It would be more efficient to use a
-  // pointer to memory containing size, allocation size, and the array of bits.
-  uintptr_t X = 1;
-
   enum {
     // The number of bits in this class.
     NumBaseBits = sizeof(uintptr_t) * CHAR_BIT,
 
     // One bit is used to discriminate between small and large mode. The
     // remaining bits are used for the small-mode representation.
     SmallNumRawBits = NumBaseBits - 1,
 
     // A few more bits are used to store the size of the bit set in small mode.
     // Theoretically this is a ceil-log2. These bits are encoded in the most
     // significant bits of the raw bits.
     SmallNumSizeBits = (NumBaseBits == 32 ? 5 :
                         NumBaseBits == 64 ? 6 :
                         SmallNumRawBits),
 
     // The remaining bits are used to store the actual set in small mode.
     SmallNumDataBits = SmallNumRawBits - SmallNumSizeBits
   };
 
   static_assert(NumBaseBits == 64 || NumBaseBits == 32,
                 "Unsupported word size");
 
+  class LargeImpl : public BitVectorImpl<LargeImpl> {
+  public:
+    friend class BitVectorImpl<LargeImpl>;
+    friend class SmallBitVector;
+
+    static_assert((sizeof(size_type) * 2) % sizeof(BitWord) == 0,
+                  "Data array must be correctly aligned");
+
+    // When the containing SmallBitVector is in large-mode, this is a pointer
+    // to the large-mode data, accessed via the methods below in this class.
+    // In small-mode this field contains the bit-packed data accessed by
+    // methods in the SmallBitVector class.
+    uintptr_t X;
+
+    static size_t getAllocSize(size_type Capacity) {
+      return sizeof(size_type) * 2 + sizeof(BitWord) * Capacity;
+    }
+
+    // Methods required by BitVectorImpl.
+    size_type getSize() const { return reinterpret_cast<size_type *>(X)[0]; }
+    void setSize(size_type N) { reinterpret_cast<size_type *>(X)[0] = N; }
+
+    size_type getCapacity() const {
+      return reinterpret_cast<size_type *>(X)[1];
+    }
+
+    BitWord *getBitsPtr() {
+      return reinterpret_cast<BitWord *>(reinterpret_cast<size_type *>(X) + 2);
+    }
+    const BitWord *getBitsPtr() const {
+      return const_cast<LargeImpl *>(this)->getBitsPtr();
+    }
+
+    void growImpl(size_type NewCapacity) {
+      X = reinterpret_cast<uintptr_t>(
+          safe_realloc(reinterpret_cast<void *>(X), getAllocSize(NewCapacity)));
+      setCapacity(NewCapacity);
+    }
+
+    void setCapacity(size_type C) { reinterpret_cast<size_type *>(X)[1] = C; }
+
+    void allocate(size_type Size) {
+      size_type Capacity = NumBitWords(Size);
+      X = reinterpret_cast<uintptr_t>(safe_malloc(getAllocSize(Capacity)));
+      setCapacity(Capacity);
+    }
+    void init(size_type Size, bool t) {
+      allocate(Size);
+      setSize(Size);
+      init_words(getBits(), t);
+    }
+    void init(const LargeImpl &RHS) {
+      size_type Size = RHS.size();
+      allocate(Size);
+      setSize(Size);
+      std::memcpy(getBitsPtr(), RHS.getBitsPtr(),
+                  getCapacity() * sizeof(BitWord));
+    }
+    void assign(const LargeImpl &RHS) {
+      size_type Size = RHS.size();
+      unsigned RHSWords = NumBitWords(Size);
+      if (RHSWords <= getCapacity()) {
+        if (RHSWords)
+          std::memcpy(getBitsPtr(), RHS.getBitsPtr(),
+                      RHSWords * sizeof(BitWord));
+        setSize(Size);
+        setCapacity(RHSWords);
+      } else {
+        release();
+        init(RHS);
+      }
+    }
+    void release() { std::free(reinterpret_cast<void *>(X)); }
+
+    LargeImpl() = default;
+    LargeImpl(const LargeImpl &) = delete;
+    LargeImpl(LargeImpl &&) = delete;
+    LargeImpl &operator=(const LargeImpl &) = delete;
+    LargeImpl &operator=(LargeImpl &&) = delete;
+  };
+  // We assume throughout the implementation of SmallBitVector small-mode data
+  // will always fit in a single BitWord.
+  static_assert(unsigned(SmallNumDataBits) <= unsigned(LargeImpl::BITWORD_SIZE),
+                "Unexpected SmallNumDataBits");
+
+  LargeImpl Data;
+
 public:
-  using size_type = unsigned;
+  using size_type = LargeImpl::size_type;
 
   // Encapsulation of a single bit.
   class reference {
     SmallBitVector &TheVector;
     unsigned BitPos;
 
   public:
     reference(SmallBitVector &b, unsigned Idx) : TheVector(b), BitPos(Idx) {}
public:
     }
 
     operator bool() const {
       return const_cast<const SmallBitVector &>(TheVector).operator[](BitPos);
     }
   };
 
 private:
-  BitVector *getPointer() const {
-    assert(!isSmall());
-    return reinterpret_cast<BitVector *>(X);
-  }
-
   void switchToSmall(uintptr_t NewSmallBits, size_t NewSize) {
-    X = 1;
+    Data.X = 1;
     setSmallSize(NewSize);
     setSmallBits(NewSmallBits);
   }
 
-  void switchToLarge(BitVector *BV) {
-    X = reinterpret_cast<uintptr_t>(BV);
-    assert(!isSmall() && "Tried to use an unaligned pointer");
-  }
-
   // Return all the bits used for the "small" representation; this includes
   // bits for the size as well as the element bits.
   uintptr_t getSmallRawBits() const {
     assert(isSmall());
-    return X >> 1;
+    return Data.X >> 1;
   }
 
   void setSmallRawBits(uintptr_t NewRawBits) {
     assert(isSmall());
-    X = (NewRawBits << 1) | uintptr_t(1);
+    Data.X = (NewRawBits << 1) | uintptr_t(1);
   }
 
   // Return the size.
   size_t getSmallSize() const { return getSmallRawBits() >> SmallNumDataBits; }
 
-  void setSmallSize(size_t Size) {
-    setSmallRawBits(getSmallBits() | (Size << SmallNumDataBits));
+  void setSmallSize(size_type Size) {
+    setSmallRawBits(getSmallBits() | (uintptr_t(Size) << SmallNumDataBits));
   }
 
   // Return the element bits.
   uintptr_t getSmallBits() const {
-    return getSmallRawBits() & ~(~uintptr_t(0) << getSmallSize());
+    return getSmallRawBits() & ~(~uintptr_t(0) << SmallNumDataBits);
   }
 
   void setSmallBits(uintptr_t NewBits) {
     setSmallRawBits((NewBits & ~(~uintptr_t(0) << getSmallSize())) |
                     (getSmallSize() << SmallNumDataBits));
   }
 
 public:
   /// Creates an empty bitvector.
-  SmallBitVector() = default;
+  SmallBitVector() { Data.X = 1; }
 
   /// Creates a bitvector of specified number of bits. All bits are initialized
   /// to the specified value.
   explicit SmallBitVector(unsigned s, bool t = false) {
     if (s <= SmallNumDataBits)
       switchToSmall(t ? ~uintptr_t(0) : 0, s);
     else
-      switchToLarge(new BitVector(s, t));
+      Data.init(s, t);
   }
 
   /// SmallBitVector copy ctor.
   SmallBitVector(const SmallBitVector &RHS) {
     if (RHS.isSmall())
-      X = RHS.X;
+      Data.X = RHS.Data.X;
     else
-      switchToLarge(new BitVector(*RHS.getPointer()));
+      Data.init(RHS.Data);
   }
 
-  SmallBitVector(SmallBitVector &&RHS) : X(RHS.X) {
-    RHS.X = 1;
+  SmallBitVector(SmallBitVector &&RHS) {
+    Data.X = RHS.Data.X;
+    RHS.Data.X = 1;
   }
 
   ~SmallBitVector() {
     if (!isSmall())
-      delete getPointer();
+      Data.release();
   }
 
   using const_set_bits_iterator = const_set_bits_iterator_impl<SmallBitVector>;
   using set_iterator = const_set_bits_iterator;
 
   const_set_bits_iterator set_bits_begin() const {
     return const_set_bits_iterator(*this);
   }
 
   const_set_bits_iterator set_bits_end() const {
     return const_set_bits_iterator(*this, -1);
   }
 
   iterator_range<const_set_bits_iterator> set_bits() const {
     return make_range(set_bits_begin(), set_bits_end());
   }
 
-  bool isSmall() const { return X & uintptr_t(1); }
+  bool isSmall() const { return Data.X & uintptr_t(1); }
 
   /// Tests whether there are no bits in this bitvector.
-  bool empty() const {
-    return isSmall() ? getSmallSize() == 0 : getPointer()->empty();
-  }
+  bool empty() const { return isSmall() ? getSmallSize() == 0 : Data.empty(); }
 
   /// Returns the number of bits in this bitvector.
-  size_t size() const {
-    return isSmall() ? getSmallSize() : getPointer()->size();
-  }
+  size_t size() const { return isSmall() ? getSmallSize() : Data.size(); }
 
   /// Returns the number of bits which are set.
   size_type count() const {
     if (isSmall()) {
       uintptr_t Bits = getSmallBits();
       return countPopulation(Bits);
     }
-    return getPointer()->count();
+    return Data.count();
   }
 
   /// Returns true if any bit is set.
   bool any() const {
     if (isSmall())
       return getSmallBits() != 0;
-    return getPointer()->any();
+    return Data.any();
   }
 
   /// Returns true if all bits are set.
   bool all() const {
     if (isSmall())
       return getSmallBits() == (uintptr_t(1) << getSmallSize()) - 1;
-    return getPointer()->all();
+    return Data.all();
   }
 
   /// Returns true if none of the bits are set.
   bool none() const {
     if (isSmall())
       return getSmallBits() == 0;
-    return getPointer()->none();
+    return Data.none();
   }
 
   /// Returns the index of the first set bit, -1 if none of the bits are set.
   int find_first() const {
     if (isSmall()) {
       uintptr_t Bits = getSmallBits();
       if (Bits == 0)
         return -1;
       return countTrailingZeros(Bits);
     }
-    return getPointer()->find_first();
+    return Data.find_first();
   }
 
   int find_last() const {
     if (isSmall()) {
       uintptr_t Bits = getSmallBits();
       if (Bits == 0)
         return -1;
       return NumBaseBits - countLeadingZeros(Bits) - 1;
     }
-    return getPointer()->find_last();
+    return Data.find_last();
   }
 
   /// Returns the index of the first unset bit, -1 if all of the bits are set.
   int find_first_unset() const {
     if (isSmall()) {
       if (count() == getSmallSize())
         return -1;
 
       uintptr_t Bits = getSmallBits();
       return countTrailingOnes(Bits);
     }
-    return getPointer()->find_first_unset();
+    return Data.find_first_unset();
   }
 
   int find_last_unset() const {
     if (isSmall()) {
       if (count() == getSmallSize())
         return -1;
 
       uintptr_t Bits = getSmallBits();
       // Set unused bits.
       Bits |= ~uintptr_t(0) << getSmallSize();
       return NumBaseBits - countLeadingOnes(Bits) - 1;
     }
-    return getPointer()->find_last_unset();
+    return Data.find_last_unset();
   }
 
   /// Returns the index of the next set bit following the "Prev" bit.
   /// Returns -1 if the next set bit is not found.
   int find_next(unsigned Prev) const {
     if (isSmall()) {
       uintptr_t Bits = getSmallBits();
       // Mask off previous bits.
       Bits &= ~uintptr_t(0) << (Prev + 1);
       if (Bits == 0 || Prev + 1 >= getSmallSize())
         return -1;
       return countTrailingZeros(Bits);
     }
-    return getPointer()->find_next(Prev);
+    return Data.find_next(Prev);
   }
 
   /// Returns the index of the next unset bit following the "Prev" bit.
   /// Returns -1 if the next unset bit is not found.
   int find_next_unset(unsigned Prev) const {
     if (isSmall()) {
       ++Prev;
       uintptr_t Bits = getSmallBits();
       // Mask in previous bits.
       uintptr_t Mask = (uintptr_t(1) << Prev) - 1;
       Bits |= Mask;
 
       if (Bits == ~uintptr_t(0) || Prev + 1 >= getSmallSize())
         return -1;
       return countTrailingOnes(Bits);
     }
-    return getPointer()->find_next_unset(Prev);
+    return Data.find_next_unset(Prev);
   }
 
   /// find_prev - Returns the index of the first set bit that precedes the
   /// the bit at \p PriorTo.  Returns -1 if all previous bits are unset.
   int find_prev(unsigned PriorTo) const {
     if (isSmall()) {
       if (PriorTo == 0)
         return -1;
 
       --PriorTo;
       uintptr_t Bits = getSmallBits();
       Bits &= maskTrailingOnes<uintptr_t>(PriorTo + 1);
       if (Bits == 0)
         return -1;
 
       return NumBaseBits - countLeadingZeros(Bits) - 1;
     }
-    return getPointer()->find_prev(PriorTo);
+    return Data.find_prev(PriorTo);
   }
 
   /// Clear all bits.
   void clear() {
     if (!isSmall())
-      delete getPointer();
-    switchToSmall(0, 0);
+      Data.release();
+    Data.X = 1;
   }
 
   /// Grow or shrink the bitvector.
-  void resize(unsigned N, bool t = false) {
+  void resize(size_type N, bool t = false) {
     if (!isSmall()) {
-      getPointer()->resize(N, t);
-    } else if (SmallNumDataBits >= N) {
+      Data.resize(N, t);
+    } else if (N <= SmallNumDataBits) {
       uintptr_t NewBits = t ? ~uintptr_t(0) << getSmallSize() : 0;
       setSmallSize(N);
       setSmallBits(NewBits | getSmallBits());
     } else {
-      BitVector *BV = new BitVector(N, t);
       uintptr_t OldBits = getSmallBits();
-      for (size_t i = 0, e = getSmallSize(); i != e; ++i)
-        (*BV)[i] = (OldBits >> i) & 1;
-      switchToLarge(BV);
+      size_type OldSize = getSmallSize();
+      Data.allocate(N);
+      Data.setSize(OldSize);
+      Data.getBitsPtr()[0] = OldBits;
+      Data.resize(N, t);
     }
   }
 
-  void reserve(unsigned N) {
+  void reserve(size_type N) {
     if (isSmall()) {
       if (N > SmallNumDataBits) {
-        uintptr_t OldBits = getSmallRawBits();
-        size_t SmallSize = getSmallSize();
-        BitVector *BV = new BitVector(SmallSize);
-        for (size_t i = 0; i < SmallSize; ++i)
-          if ((OldBits >> i) & 1)
-            BV->set(i);
-        BV->reserve(N);
-        switchToLarge(BV);
+        size_type Size = getSmallSize();
+        uintptr_t OldBits = getSmallBits();
+        Data.allocate(N);
+        Data.setSize(Size);
+        Data.getBitsPtr()[0] = OldBits;
       }
     } else {
-      getPointer()->reserve(N);
+      Data.reserve(N);
     }
   }
 
   // Set, reset, flip
   SmallBitVector &set() {
     if (isSmall())
       setSmallBits(~uintptr_t(0));
     else
-      getPointer()->set();
+      Data.set();
     return *this;
   }
 
   SmallBitVector &set(unsigned Idx) {
     if (isSmall()) {
-      assert(Idx <= static_cast<unsigned>(
-                        std::numeric_limits<uintptr_t>::digits) &&
-             "undefined behavior");
+      assert(Idx < getSmallSize());
       setSmallBits(getSmallBits() | (uintptr_t(1) << Idx));
-    }
-    else
-      getPointer()->set(Idx);
+    } else
+      Data.set(Idx);
     return *this;
   }
 
   /// Efficiently set a range of bits in [I, E)
   SmallBitVector &set(unsigned I, unsigned E) {
     assert(I <= E && "Attempted to set backwards range!");
     assert(E <= size() && "Attempted to set out-of-bounds range!");
     if (I == E) return *this;
     if (isSmall()) {
       uintptr_t EMask = ((uintptr_t)1) << E;
       uintptr_t IMask = ((uintptr_t)1) << I;
       uintptr_t Mask = EMask - IMask;
       setSmallBits(getSmallBits() | Mask);
     } else
-      getPointer()->set(I, E);
+      Data.set(I, E);
     return *this;
   }
 
   SmallBitVector &reset() {
     if (isSmall())
       setSmallBits(0);
     else
-      getPointer()->reset();
+      Data.reset();
     return *this;
   }
 
   SmallBitVector &reset(unsigned Idx) {
-    if (isSmall())
+    if (isSmall()) {
+      assert(Idx < getSmallSize());
       setSmallBits(getSmallBits() & ~(uintptr_t(1) << Idx));
-    else
-      getPointer()->reset(Idx);
+    } else
+      Data.reset(Idx);
     return *this;
   }
 
   /// Efficiently reset a range of bits in [I, E)
   SmallBitVector &reset(unsigned I, unsigned E) {
     assert(I <= E && "Attempted to reset backwards range!");
     assert(E <= size() && "Attempted to reset out-of-bounds range!");
     if (I == E) return *this;
     if (isSmall()) {
       uintptr_t EMask = ((uintptr_t)1) << E;
       uintptr_t IMask = ((uintptr_t)1) << I;
       uintptr_t Mask = EMask - IMask;
       setSmallBits(getSmallBits() & ~Mask);
     } else
-      getPointer()->reset(I, E);
+      Data.reset(I, E);
     return *this;
   }
 
   SmallBitVector &flip() {
     if (isSmall())
       setSmallBits(~getSmallBits());
     else
-      getPointer()->flip();
+      Data.flip();
     return *this;
   }
 
   SmallBitVector &flip(unsigned Idx) {
-    if (isSmall())
+    if (isSmall()) {
+      assert(Idx < getSmallSize());
       setSmallBits(getSmallBits() ^ (uintptr_t(1) << Idx));
-    else
-      getPointer()->flip(Idx);
+    } else
+      Data.flip(Idx);
     return *this;
   }
 
   // No argument flip.
   SmallBitVector operator~() const {
     return SmallBitVector(*this).flip();
   }
 
   // Indexing.
   reference operator[](unsigned Idx) {
     assert(Idx < size() && "Out-of-bounds Bit access.");
     return reference(*this, Idx);
   }
 
   bool operator[](unsigned Idx) const {
     assert(Idx < size() && "Out-of-bounds Bit access.");
     if (isSmall())
       return ((getSmallBits() >> Idx) & 1) != 0;
-    return getPointer()->operator[](Idx);
+    return Data[Idx];
   }
 
   bool test(unsigned Idx) const {
     return (*this)[Idx];
   }
 
   // Push single bit to end of vector.
-  void push_back(bool Val) {
-    resize(size() + 1, Val);
-  }
+  void push_back(bool Val) { resize(size() + 1, Val); }
 
   /// Test if any common bits are set.
   bool anyCommon(const SmallBitVector &RHS) const {
     if (isSmall() && RHS.isSmall())
       return (getSmallBits() & RHS.getSmallBits()) != 0;
     if (!isSmall() && !RHS.isSmall())
-      return getPointer()->anyCommon(*RHS.getPointer());
-
-    for (unsigned i = 0, e = std::min(size(), RHS.size()); i != e; ++i)
-      if (test(i) && RHS.test(i))
-        return true;
-    return false;
+      return Data.anyCommon(RHS.Data);
+    return (getFirstWord() & RHS.getFirstWord()) != 0;
   }
 
   // Comparison operators.
   bool operator==(const SmallBitVector &RHS) const {
-    if (size() != RHS.size())
-      return false;
     if (isSmall() && RHS.isSmall())
-      return getSmallBits() == RHS.getSmallBits();
+      return Data.X == RHS.Data.X;
     else if (!isSmall() && !RHS.isSmall())
-      return *getPointer() == *RHS.getPointer();
-    else {
-      for (size_t i = 0, e = size(); i != e; ++i) {
-        if ((*this)[i] != RHS[i])
-          return false;
-      }
-      return true;
-    }
-  }
+      return Data == RHS.Data;
+    else if (size() == RHS.size())
+      return getFirstWord() == RHS.getFirstWord();
 
   bool operator!=(const SmallBitVector &RHS) const {
     return !(*this == RHS);
   }
 
   // Intersection, union, disjoint union.
   // FIXME BitVector::operator&= does not resize the LHS but this does
   SmallBitVector &operator&=(const SmallBitVector &RHS) {
     resize(std::max(size(), RHS.size()));
     if (isSmall() && RHS.isSmall())
       setSmallBits(getSmallBits() & RHS.getSmallBits());
     else if (!isSmall() && !RHS.isSmall())
-      getPointer()->operator&=(*RHS.getPointer());
+      Data &= RHS.Data;
     else {
-      size_t i, e;
-      for (i = 0, e = std::min(size(), RHS.size()); i != e; ++i)
-        (*this)[i] = test(i) && RHS.test(i);
-      for (e = size(); i != e; ++i)
-        reset(i);
+      setFirstWord(getFirstWord() & RHS.getFirstWord());
+      clearUpperWords();
     }
     return *this;
   }
 
   /// Reset bits that are set in RHS. Same as *this &= ~RHS.
   SmallBitVector &reset(const SmallBitVector &RHS) {
     if (isSmall() && RHS.isSmall())
       setSmallBits(getSmallBits() & ~RHS.getSmallBits());
     else if (!isSmall() && !RHS.isSmall())
-      getPointer()->reset(*RHS.getPointer());
+      Data.reset(RHS.Data);
     else
-      for (unsigned i = 0, e = std::min(size(), RHS.size()); i != e; ++i)
-        if (RHS.test(i))
-          reset(i);
-
+      setFirstWord(getFirstWord() & ~RHS.getFirstWord());
     return *this;
   }
 
   /// Check if (This - RHS) is zero. This is the same as reset(RHS) and any().
   bool test(const SmallBitVector &RHS) const {
     if (isSmall() && RHS.isSmall())
       return (getSmallBits() & ~RHS.getSmallBits()) != 0;
     if (!isSmall() && !RHS.isSmall())
-      return getPointer()->test(*RHS.getPointer());
-
-    unsigned i, e;
-    for (i = 0, e = std::min(size(), RHS.size()); i != e; ++i)
-      if (test(i) && !RHS.test(i))
-        return true;
-
-    for (e = size(); i != e; ++i)
-      if (test(i))
+      return Data.test(RHS.Data);
+    if ((getFirstWord() & ~RHS.getFirstWord()) != 0)
+    for (LargeImpl::BitWord Word : getUpperWords()) {
+      if (Word)
         return true;
-
+    }
     return false;
   }
 
   SmallBitVector &operator|=(const SmallBitVector &RHS) {
     resize(std::max(size(), RHS.size()));
     if (isSmall() && RHS.isSmall())
       setSmallBits(getSmallBits() | RHS.getSmallBits());
     else if (!isSmall() && !RHS.isSmall())
-      getPointer()->operator|=(*RHS.getPointer());
-    else {
-      for (size_t i = 0, e = RHS.size(); i != e; ++i)
-        (*this)[i] = test(i) || RHS.test(i);
-    }
+      Data |= RHS.Data;
+    else
+      setFirstWord(getFirstWord() | RHS.getFirstWord());
     return *this;
   }
 
   SmallBitVector &operator^=(const SmallBitVector &RHS) {
     resize(std::max(size(), RHS.size()));
     if (isSmall() && RHS.isSmall())
       setSmallBits(getSmallBits() ^ RHS.getSmallBits());
     else if (!isSmall() && !RHS.isSmall())
-      getPointer()->operator^=(*RHS.getPointer());
-    else {
-      for (size_t i = 0, e = RHS.size(); i != e; ++i)
-        (*this)[i] = test(i) != RHS.test(i);
-    }
+      Data ^= RHS.Data;
+    else
+      setFirstWord(getFirstWord() ^ RHS.getFirstWord());
     return *this;
   }
 
   SmallBitVector &operator<<=(unsigned N) {
     if (isSmall())
       setSmallBits(getSmallBits() << N);
     else
-      getPointer()->operator<<=(N);
+      Data <<= N;
     return *this;
   }
 
   SmallBitVector &operator>>=(unsigned N) {
     if (isSmall())
       setSmallBits(getSmallBits() >> N);
     else
-      getPointer()->operator>>=(N);
+      Data >>= N;
     return *this;
   }
 
   // Assignment operator.
   const SmallBitVector &operator=(const SmallBitVector &RHS) {
-    if (isSmall()) {
+    if (this == &RHS) {
+      // Do nothing
+    } else if (isSmall()) {
       if (RHS.isSmall())
-        X = RHS.X;
+        Data.X = RHS.Data.X;
       else
-        switchToLarge(new BitVector(*RHS.getPointer()));
+        Data.init(RHS.Data);
     } else {
-      if (!RHS.isSmall())
-        *getPointer() = *RHS.getPointer();
-      else {
-        delete getPointer();
-        X = RHS.X;
-      }
+      if (RHS.isSmall()) {
+        Data.release();
+        Data.X = RHS.Data.X;
+      } else
+        Data.assign(RHS.Data);
     }
     return *this;
   }
 
   const SmallBitVector &operator=(SmallBitVector &&RHS) {
     if (this != &RHS) {
       clear();
       swap(RHS);
     }
     return *this;
   }
 
   void swap(SmallBitVector &RHS) {
-    std::swap(X, RHS.X);
+    std::swap(Data.X, RHS.Data.X);
   }
 
   /// Add '1' bits from Mask to this vector. Don't resize.
   /// This computes "*this |= Mask".
   void setBitsInMask(const uint32_t *Mask, unsigned MaskWords = ~0u) {
     if (isSmall())
       applyMask<true, false>(Mask, MaskWords);
     else
-      getPointer()->setBitsInMask(Mask, MaskWords);
+      Data.setBitsInMask(Mask, MaskWords);
   }
 
   /// Clear any bits in this vector that are set in Mask. Don't resize.
   /// This computes "*this &= ~Mask".
   void clearBitsInMask(const uint32_t *Mask, unsigned MaskWords = ~0u) {
     if (isSmall())
       applyMask<false, false>(Mask, MaskWords);
     else
-      getPointer()->clearBitsInMask(Mask, MaskWords);
+      Data.clearBitsInMask(Mask, MaskWords);
   }
 
   /// Add a bit to this vector for every '0' bit in Mask. Don't resize.
   /// This computes "*this |= ~Mask".
   void setBitsNotInMask(const uint32_t *Mask, unsigned MaskWords = ~0u) {
     if (isSmall())
       applyMask<true, true>(Mask, MaskWords);
     else
-      getPointer()->setBitsNotInMask(Mask, MaskWords);
+      Data.setBitsNotInMask(Mask, MaskWords);
   }
 
   /// Clear a bit in this vector for every '0' bit in Mask. Don't resize.
   /// This computes "*this &= Mask".
   void clearBitsNotInMask(const uint32_t *Mask, unsigned MaskWords = ~0u) {
     if (isSmall())
       applyMask<false, true>(Mask, MaskWords);
     else
-      getPointer()->clearBitsNotInMask(Mask, MaskWords);
+      Data.clearBitsNotInMask(Mask, MaskWords);
   }
 
-  void invalid() {
-    assert(empty());
-    X = (uintptr_t)-1;
+  // These are invalid forms, because the maximum size value able to be
+  // represented in the small size is always more than number of data bits
+  // available. These forms are valid for =, == and != but nothing else.
+  static SmallBitVector getDenseMapEmptyKey() {
+    SmallBitVector K;
+    K.setSmallSize((1 << SmallNumSizeBits) - 1);
+    return K;
+  }
+  static SmallBitVector getDenseMapTombstoneKey() {
+    SmallBitVector K;
+    K.setSmallSize((1 << SmallNumSizeBits) - 2);
+    return K;
+  }
+  unsigned getDenseMapHashValue() const {
+    using HashType = std::pair<size_type, ArrayRef<LargeImpl::BitWord>>;
+    HashType H;
+    H.first = size();
+    LargeImpl::BitWord Word;
+    if (isSmall()) {
+      Word = getSmallBits();
+      H.second = ArrayRef<LargeImpl::BitWord>(Word);
+    } else {
+      H.second = Data.getActiveBits();
     }
-  bool isInvalid() const { return X == (uintptr_t)-1; }
-
-  ArrayRef<uintptr_t> getData() const {
-    return isSmall() ? makeArrayRef(X) : getPointer()->getData();
+    return DenseMapInfo<HashType>::getHashValue(H);
   }
 
 private:
+  LargeImpl::BitWord getFirstWord() const {
+    if (isSmall())
+      return getSmallBits();
+    if (Data.empty())
+      return 0;
+    return Data.getBitsPtr()[0];
+  }
+  void setFirstWord(uintptr_t NewBits) {
+    if (isSmall()) {
+      setSmallBits(NewBits);
+    } else {
+      MutableArrayRef<LargeImpl::BitWord> Bits = Data.getActiveBits();
+      if (!Bits.empty())
+        Bits[0] = NewBits;
+    }
+  }
+  void clearUpperWords() {
+    if (isSmall())
+      return;
+    MutableArrayRef<LargeImpl::BitWord> Bits = Data.getActiveBits();
+    if (!Bits.empty())
+      LargeImpl::init_words(Bits.drop_front(), false);
+  }
+  ArrayRef<LargeImpl::BitWord> getUpperWords() const {
+    if (isSmall())
+      return {};
+    ArrayRef<LargeImpl::BitWord> Bits = Data.getActiveBits();
+    if (Bits.empty())
+      return Bits;
+    return Bits.drop_front();
+  }
+
   template <bool AddBits, bool InvertMask>
   void applyMask(const uint32_t *Mask, unsigned MaskWords) {
     assert(MaskWords <= sizeof(uintptr_t) && "Mask is larger than base!");
     uintptr_t M = Mask[0];
     if (NumBaseBits == 64)
       M |= uint64_t(Mask[1]) << 32;
     if (InvertMask)
       M = ~M;

 inline SmallBitVector
 operator^(const SmallBitVector &LHS, const SmallBitVector &RHS) {
   SmallBitVector Result(LHS);
   Result ^= RHS;
   return Result;
 }
 
 template <> struct DenseMapInfo<SmallBitVector> {
-  static inline SmallBitVector getEmptyKey() { return SmallBitVector(); }
+  static inline SmallBitVector getEmptyKey() {
+    return SmallBitVector::getDenseMapEmptyKey();
+  }
   static inline SmallBitVector getTombstoneKey() {
-    SmallBitVector V;
-    V.invalid();
-    return V;
+    return SmallBitVector::getDenseMapTombstoneKey();
   }
   static unsigned getHashValue(const SmallBitVector &V) {
-    return DenseMapInfo<std::pair<unsigned, ArrayRef<uintptr_t>>>::getHashValue(
-        std::make_pair(V.size(), V.getData()));
+    return V.getDenseMapHashValue();
   }
   static bool isEqual(const SmallBitVector &LHS, const SmallBitVector &RHS) {
-    if (LHS.isInvalid() || RHS.isInvalid())
-      return LHS.isInvalid() == RHS.isInvalid();
     return LHS == RHS;
   }
 };
 } // end namespace llvm
 
 namespace std {
 
 /// Implement std::swap in terms of BitVector swap.
 inline void
 swap(llvm::SmallBitVector &LHS, llvm::SmallBitVector &RHS) {
   LHS.swap(RHS);
 }
 
 } // end namespace std
 
 #endif // LLVM_ADT_SMALLBITVECTOR_H

llvm/lib/CodeGen/RegisterScavenging.cpp

void RegScavenger::determineKillsAndDefs() {
   assert(!MI.isDebugInstr() && "Debug values have no kills or defs");
 
   // Find out which registers are early clobbered, killed, defined, and marked
   // def-dead in this instruction.
   KillRegUnits.reset();
   DefRegUnits.reset();
   for (const MachineOperand &MO : MI.operands()) {
     if (MO.isRegMask()) {
-      TmpRegUnits.clear();
+      TmpRegUnits.reset();
       for (unsigned RU = 0, RUEnd = TRI->getNumRegUnits(); RU != RUEnd; ++RU) {
         for (MCRegUnitRootIterator RURI(RU, TRI); RURI.isValid(); ++RURI) {
           if (MO.clobbersPhysReg(*RURI)) {
             TmpRegUnits.set(RU);
             break;
           }
         }
       }

llvm/unittests/ADT/BitVectorTest.cpp

TYPED_TEST(BitVectorTest, TrivialOperation) {
   EXPECT_EQ(0U, Vec.count());
   EXPECT_EQ(0U, Vec.size());
   EXPECT_FALSE(Vec.any());
   EXPECT_TRUE(Vec.all());
   EXPECT_TRUE(Vec.none());
   EXPECT_TRUE(Vec.empty());
 }
 
+TYPED_TEST(BitVectorTest, Equality) {
+  TypeParam A;
+  TypeParam B;
+  EXPECT_TRUE(A == B);
+  A.resize(10);
+  EXPECT_FALSE(A == B);
+  B.resize(10);
+  EXPECT_TRUE(A == B);
+  A.set(5);
+  EXPECT_FALSE(A == B);
+  B.set(5);
+  EXPECT_TRUE(A == B);
+  A.resize(20);
+  EXPECT_FALSE(A == B);
+  B.resize(20);
+  EXPECT_TRUE(A == B);
+}
+
+TYPED_TEST(BitVectorTest, Resize) {
+  // Increase size and capacity
+  {
+    TypeParam A;
+    A.resize(100);
+    A.set(12);
+    A.set(13);
+    A.set(75);
+    A.resize(400);
+    EXPECT_EQ(400U, A.size());
+    EXPECT_EQ(3U, A.count());
+    EXPECT_TRUE(A.test(12));
+    EXPECT_TRUE(A.test(13));
+    EXPECT_TRUE(A.test(75));
+  }
+  {
+    TypeParam A;
+    A.resize(100);
+    A.set(12);
+    A.set(13);
+    A.set(75);
+    A.resize(400, true);
+    EXPECT_EQ(400U, A.size());
+    EXPECT_EQ(303U, A.count());
+    EXPECT_TRUE(A.test(12));
+    EXPECT_TRUE(A.test(13));
+    EXPECT_TRUE(A.test(75));
+  }
+  // Increase size and but not capacity
+  {
+    TypeParam A;
+    A.reserve(600);
+    A.resize(100);
+    A.set(12);
+    A.set(13);
+    A.set(75);
+    A.resize(400);
+    EXPECT_EQ(400U, A.size());
+    EXPECT_EQ(3U, A.count());
+    EXPECT_TRUE(A.test(12));
+    EXPECT_TRUE(A.test(13));
+    EXPECT_TRUE(A.test(75));
+  }
+  {
+    TypeParam A;
+    A.reserve(600);
+    A.resize(100);
+    A.set(12);
+    A.set(13);
+    A.set(75);
+    A.resize(400, true);
+    EXPECT_EQ(400U, A.size());
+    EXPECT_EQ(303U, A.count());
+    EXPECT_TRUE(A.test(12));
+    EXPECT_TRUE(A.test(13));
+    EXPECT_TRUE(A.test(75));
+  }
+  // Reduce size
+  {
+    TypeParam A;
+    A.resize(100);
+    A.set(12);
+    A.set(13);
+    A.set(75);
+    A.resize(50);
+    EXPECT_EQ(50U, A.size());
+    EXPECT_EQ(2U, A.count());
+    EXPECT_TRUE(A.test(12));
+    EXPECT_TRUE(A.test(13));
+  }
+  {
+    TypeParam A;
+    A.resize(100);
+    A.set(12);
+    A.set(13);
+    A.set(75);
+    A.resize(50, true);
+    EXPECT_EQ(50U, A.size());
+    EXPECT_EQ(2U, A.count());
+    EXPECT_TRUE(A.test(12));
+    EXPECT_TRUE(A.test(13));
+  }
+}
+
 TYPED_TEST(BitVectorTest, SimpleFindOpsMultiWord) {
   TypeParam A;
 
   // Test finding next set and unset bits in a BitVector with multiple words
   A.resize(100);
   A.set(12);
   A.set(13);
   A.set(75);
TYPED_TEST(BitVectorTest, DenseSet) {
   I = Set.insert(B);
   EXPECT_EQ(true, I.second);
 
   TypeParam C(20, false);
   C.set(19);
   I = Set.insert(C);
   EXPECT_EQ(true, I.second);
 
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   TypeParam D;
-  EXPECT_DEATH(Set.insert(D),
-               "Empty/Tombstone value shouldn't be inserted into map!");
-#endif
+  I = Set.insert(D);
+  EXPECT_EQ(true, I.second);
 
-  EXPECT_EQ(3U, Set.size());
+  EXPECT_EQ(4U, Set.size());
   EXPECT_EQ(1U, Set.count(A));
   EXPECT_EQ(1U, Set.count(B));
   EXPECT_EQ(1U, Set.count(C));
+  EXPECT_EQ(1U, Set.count(D));
 
   EXPECT_EQ(true, Set.erase(B));
-  EXPECT_EQ(2U, Set.size());
+  EXPECT_EQ(3U, Set.size());
 
   EXPECT_EQ(true, Set.erase(C));
-  EXPECT_EQ(1U, Set.size());
+  EXPECT_EQ(2U, Set.size());
 
   EXPECT_EQ(true, Set.erase(A));
+  EXPECT_EQ(1U, Set.size());
+
+  EXPECT_EQ(true, Set.erase(D));
   EXPECT_EQ(0U, Set.size());
 }
+
+/// Test that capacity and small/large mode don't affect hashing.
+TYPED_TEST(BitVectorTest, DenseMapHashing) {
+  using DMI = DenseMapInfo<TypeParam>;
+  {
+    TypeParam A;
+    A.resize(200);
+    A.set(100);
+
+    TypeParam B;
+    B.resize(200);
+    B.set(100);
+    B.reserve(1000);
+
+    EXPECT_EQ(DMI::getHashValue(A), DMI::getHashValue(B));
+  }
+  {
+    TypeParam A;
+    A.resize(20);
+    A.set(10);
+
+    TypeParam B;
+    B.resize(20);
+    B.set(10);
+    B.reserve(1000);
+
+    EXPECT_EQ(DMI::getHashValue(A), DMI::getHashValue(B));
+  }
+}
+
 } // namespace