Index: llvm/include/llvm/ADT/SmallVector.h
===================================================================
--- llvm/include/llvm/ADT/SmallVector.h
+++ llvm/include/llvm/ADT/SmallVector.h
@@ -235,7 +235,14 @@
                              (is_trivially_move_constructible<T>::value) &&
                              std::is_trivially_destructible<T>::value>
 class SmallVectorTemplateBase : public SmallVectorTemplateCommon<T> {
+public:
+  /// Take parameters by reference (not by value) since T is not trivially
+  /// copyable.
+  static constexpr bool TakesParamByValue = false;
+
 protected:
+  using ValueParamT = const T &;
+
   SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {}
 
   static void destroy_range(T *S, T *E) {
@@ -266,8 +273,9 @@
   void grow(size_t MinSize = 0);
 
 public:
-  void push_back(const T &Elt) {
-    this->assertSafeToAdd(&Elt);
+  void push_back(ValueParamT Elt) {
+    if (!TakesParamByValue)
+      this->assertSafeToAdd(&Elt);
     if (LLVM_UNLIKELY(this->size() >= this->capacity()))
       this->grow();
     ::new ((void*) this->end()) T(Elt);
@@ -328,7 +336,16 @@
 /// skipping destruction.
 template <typename T>
 class SmallVectorTemplateBase<T, true> : public SmallVectorTemplateCommon<T> {
+public:
+  /// Take parameters by value when T is small since it's trivially copyable.
+  static constexpr bool TakesParamByValue =
+      std::conditional<sizeof(T) <= 2 * sizeof(void *), std::true_type,
+                       std::false_type>::type::value;
+
 protected:
+  using ValueParamT =
+      typename std::conditional<TakesParamByValue, T, const T &>::type;
+
   SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {}
 
   // No need to do a destroy loop for POD's.
@@ -370,8 +387,9 @@
   void grow(size_t MinSize = 0) { this->grow_pod(MinSize, sizeof(T)); }
 
 public:
-  void push_back(const T &Elt) {
-    this->assertSafeToAdd(&Elt);
+  void push_back(ValueParamT Elt) {
+    if (!TakesParamByValue)
+      this->assertSafeToAdd(&Elt);
     if (LLVM_UNLIKELY(this->size() >= this->capacity()))
       this->grow();
     memcpy(reinterpret_cast<void *>(this->end()), &Elt, sizeof(T));
@@ -393,7 +411,11 @@
   using reference = typename SuperClass::reference;
   using size_type = typename SuperClass::size_type;
 
+  using SmallVectorTemplateBase<T>::TakesParamByValue;
+
 protected:
+  using ValueParamT = typename SmallVectorTemplateBase<T>::ValueParamT;
+
   // Default ctor - Initialize to empty.
   explicit SmallVectorImpl(unsigned N)
       : SmallVectorTemplateBase<T>(N) {}
@@ -426,8 +448,9 @@
     }
   }
 
-  void resize(size_type N, const T &NV) {
-    this->assertSafeToReferenceAfterResize(&NV, N);
+  void resize(size_type N, ValueParamT NV) {
+    if (!TakesParamByValue)
+      this->assertSafeToReferenceAfterResize(&NV, N);
     if (N < this->size()) {
       this->destroy_range(this->begin()+N, this->end());
       this->set_size(N);
@@ -473,8 +496,9 @@
   }
 
   /// Append \p NumInputs copies of \p Elt to the end.
-  void append(size_type NumInputs, const T &Elt) {
-    this->assertSafeToAdd(&Elt, NumInputs);
+  void append(size_type NumInputs, ValueParamT Elt) {
+    if (!TakesParamByValue)
+      this->assertSafeToAdd(&Elt, NumInputs);
     if (NumInputs > this->capacity() - this->size())
       this->grow(this->size()+NumInputs);
 
@@ -489,8 +513,9 @@
   // FIXME: Consider assigning over existing elements, rather than clearing &
   // re-initializing them - for all assign(...) variants.
 
-  void assign(size_type NumElts, const T &Elt) {
-    this->assertSafeToReferenceAfterResize(&Elt, 0);
+  void assign(size_type NumElts, ValueParamT Elt) {
+    if (!TakesParamByValue)
+      this->assertSafeToReferenceAfterResize(&Elt, 0);
     clear();
     if (this->capacity() < NumElts)
       this->grow(NumElts);
@@ -556,7 +581,8 @@
     assert(I <= this->end() && "Inserting past the end of the vector.");
 
     // Check that adding an element won't invalidate Elt.
-    this->assertSafeToAdd(&Elt);
+    if (!TakesParamByValue)
+      this->assertSafeToAdd(&Elt);
 
     if (this->size() >= this->capacity()) {
       size_t EltNo = I-this->begin();
@@ -572,21 +598,49 @@
     // If we just moved the element we're inserting, be sure to update
     // the reference.
     std::remove_reference_t<ArgType> *EltPtr = &Elt;
-    if (I <= EltPtr && EltPtr < this->end())
+    if (!TakesParamByValue && I <= EltPtr && EltPtr < this->end())
       ++EltPtr;
 
     *I = ::std::forward<ArgType>(*EltPtr);
     return I;
   }
 
+  template <
+      class ArgType,
+      std::enable_if_t<
+          std::is_same<std::remove_const_t<std::remove_reference_t<ArgType>>,
+                       T>::value &&
+              !TakesParamByValue,
+          bool> = true>
+  iterator insert_one_maybe_copy(iterator I, ArgType &&Elt) {
+    // TakesParamByValue is false. Just forward to insert_one_impl.
+    return insert_one_impl(I, std::forward<ArgType>(Elt));
+  }
+
+  /// insert a T, where TakesParamByValue is true. Calls insert_one_impl with a
+  /// copy of Elt to avoid invalidation.
+  template <
+      class ArgType,
+      std::enable_if_t<
+          std::is_same<std::remove_const_t<std::remove_reference_t<ArgType>>,
+                       T>::value &&
+              TakesParamByValue,
+          bool> = true>
+  iterator insert_one_maybe_copy(iterator I, ArgType &&Elt) {
+    // TakesParamByValue is true. Call insert_one_impl with a copy of Elt.
+    return insert_one_impl(I, T(Elt));
+  }
+
 public:
   iterator insert(iterator I, T &&Elt) {
-    return insert_one_impl(I, std::move(Elt));
+    return insert_one_maybe_copy(I, std::move(Elt));
   }
 
-  iterator insert(iterator I, const T &Elt) { return insert_one_impl(I, Elt); }
+  iterator insert(iterator I, const T &Elt) {
+    return insert_one_maybe_copy(I, Elt);
+  }
 
-  iterator insert(iterator I, size_type NumToInsert, const T &Elt) {
+  iterator insert(iterator I, size_type NumToInsert, ValueParamT Elt) {
     // Convert iterator to elt# to avoid invalidating iterator when we reserve()
     size_t InsertElt = I - this->begin();
 
@@ -599,7 +653,8 @@
     assert(I <= this->end() && "Inserting past the end of the vector.");
 
     // Check that adding NumToInsert elements won't invalidate Elt.
-    this->assertSafeToAdd(&Elt, NumToInsert);
+    if (!TakesParamByValue)
+      this->assertSafeToAdd(&Elt, NumToInsert);
 
     // Ensure there is enough space.
     reserve(this->size() + NumToInsert);
@@ -731,7 +786,8 @@
       class ArgType,
       std::enable_if_t<
           std::is_same<std::remove_const_t<std::remove_reference_t<ArgType>>,
-                       T>::value,
+                       T>::value &&
+              !TakesParamByValue,
           bool> = true>
   reference emplace_back(ArgType &&Arg) {
     // ArgType is a T and may be a reference into this vector. Check whether
@@ -740,6 +796,21 @@
     return emplace_back_impl(std::forward<ArgType>(Arg));
   }
 
+  template <
+      class ArgType,
+      std::enable_if_t<
+          std::is_same<std::remove_const_t<std::remove_reference_t<ArgType>>,
+                       T>::value &&
+              TakesParamByValue,
+          bool> = true>
+  reference emplace_back(ArgType &&Arg) {
+    // ArgType is a T and may be a reference into this vector, but
+    // TakesParamByValue is true. Forward to push_back, which will take Arg by
+    // value and avoid reference invalidation.
+    this->push_back(Arg);
+    return this->back();
+  }
+
   template <class ArgType1, class ArgType2, class... ArgTypes>
   reference emplace_back(ArgType1 &&Arg1, ArgType2 &&Arg2,
                          ArgTypes &&... Args) {
@@ -938,6 +1009,8 @@
 template <typename T, unsigned N>
 class LLVM_GSL_OWNER SmallVector : public SmallVectorImpl<T>,
                                    SmallVectorStorage<T, N> {
+  using ValueParamT = typename SmallVectorImpl<T>::ValueParamT;
+
 public:
   SmallVector() : SmallVectorImpl<T>(N) {}
 
@@ -946,8 +1019,8 @@
     this->destroy_range(this->begin(), this->end());
   }
 
-  explicit SmallVector(size_t Size, const T &Value = T())
-    : SmallVectorImpl<T>(N) {
+  explicit SmallVector(size_t Size, ValueParamT Value = T())
+      : SmallVectorImpl<T>(N) {
     this->assign(Size, Value);
   }
 
Index: llvm/unittests/ADT/SmallVectorTest.cpp
===================================================================
--- llvm/unittests/ADT/SmallVectorTest.cpp
+++ llvm/unittests/ADT/SmallVectorTest.cpp
@@ -1004,6 +1004,7 @@
 template <class VectorT>
 class SmallVectorReferenceInvalidationTest : public SmallVectorTestBase {
 protected:
+  bool TakesParamByValue = VectorT::TakesParamByValue;
   const char *AssertionMessage =
       "Attempting to reference an element of the vector in an operation \" "
       "\"that invalidates it";
@@ -1024,8 +1025,11 @@
 };
 
 // Test one type that's trivially copyable (int) and one that isn't
-// (Constructable) since reference invalidation may be fixed differently for
-// each.
+// (Constructable).
+static_assert(SmallVectorImpl<int>::TakesParamByValue,
+              "int should be trivially copyable and small enough");
+static_assert(!SmallVectorImpl<Constructable>::TakesParamByValue,
+              "Constructable should not be trivially copyable");
 using SmallVectorReferenceInvalidationTestTypes =
     ::testing::Types<SmallVector<int, 1>, SmallVector<Constructable, 1>>;
 
@@ -1034,7 +1038,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, PushBack) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.push_back(V.back());
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.push_back(V.back()), this->AssertionMessage);
 #endif
@@ -1042,7 +1049,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, PushBackMoved) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.push_back(std::move(V.back()));
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.push_back(std::move(V.back())), this->AssertionMessage);
 #endif
@@ -1050,7 +1060,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, Resize) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.resize(2, V.back());
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.resize(2, V.back()), this->AssertionMessage);
 #endif
@@ -1058,7 +1071,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, Append) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.append(1, V.back());
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.append(1, V.back()), this->AssertionMessage);
 #endif
@@ -1066,7 +1082,12 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, Assign) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.assign(1, V.back());
+    V.assign(2, V.back());
+    V.assign(3, V.back());
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   // Regardless of capacity, assign should never reference an internal element.
   EXPECT_DEATH(V.assign(1, V.back()), this->AssertionMessage);
@@ -1077,7 +1098,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, Insert) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.insert(V.begin(), V.back());
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.insert(V.begin(), V.back()), this->AssertionMessage);
 #endif
@@ -1085,7 +1109,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, InsertMoved) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.insert(V.begin(), std::move(V.back()));
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.insert(V.begin(), std::move(V.back())),
                this->AssertionMessage);
@@ -1094,7 +1121,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, InsertN) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.insert(V.begin(), 2, V.back());
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.insert(V.begin(), 2, V.back()), this->AssertionMessage);
 #endif
@@ -1102,7 +1132,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, EmplaceBack) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.emplace_back(V.back());
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.emplace_back(V.back()), this->AssertionMessage);
 #endif
@@ -1110,7 +1143,10 @@
 
 TYPED_TEST(SmallVectorReferenceInvalidationTest, EmplaceBackMoved) {
   auto &V = this->V;
-  (void)V;
+  if (this->TakesParamByValue) {
+    V.emplace_back(std::move(V.back()));
+    return;
+  }
 #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST
   EXPECT_DEATH(V.emplace_back(std::move(V.back())), this->AssertionMessage);
 #endif