diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -301,12 +301,18 @@
       return false;
     auto *Const =
         isConstant(IV) ? Solver.getConstant(IV) : UndefValue::get(V->getType());
-    V->replaceAllUsesWith(Const);
 
-    for (auto *U : Const->users())
-      if (auto *I = dyn_cast<Instruction>(U))
+    // Using replaceUsesWithIf is probably the easiest way to do this without
+    // duplicating code.
+    SmallVector<Instruction *, 8> ReplacedUsers;
+    V->replaceUsesWithIf(Const, [&](auto &U) {
+      if (auto *I = dyn_cast<Instruction>(U.getUser()))
         if (Solver.isBlockExecutable(I->getParent()))
-          Solver.visit(I);
+          ReplacedUsers.push_back(I);
+      return true;
+    });
+    for (auto *I : ReplacedUsers)
+      Solver.visit(I);
 
     // Remove the instruction from Block and Solver.
     if (auto *I = dyn_cast<Instruction>(V)) {