Index: llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2593,6 +2593,74 @@
       }
     }
 
+    // Walk up the def chain unscheduling all nodes encountered
+    // until a frontier of unscheduled nodes is found.
+    void unschedule(ScheduleData *SD) {
+      assert(SD && SD->isSchedulingEntity() && SD->IsScheduled);
+      LLVM_DEBUG(dbgs() << "SLP:   unschedule " << *SD << "\n");
+
+      SmallVector<ScheduleData*> Worklist;
+      Worklist.push_back(SD);
+
+      while (!Worklist.empty()) {
+        ScheduleData *SD = Worklist.pop_back_val();
+        if (!SD->IsScheduled)
+          continue;
+
+        SD->IsScheduled = false;
+
+        // If BundleMember is a vector bundle, its operands may have been
+        // reordered duiring buildTree(). We therefore need to get its operands
+        // through the TreeEntry.
+        if (TreeEntry *TE = SD->TE) {
+          int Lane = SD->Lane;
+          assert(Lane >= 0 && "Lane not set");
+
+          // Since vectorization tree is being built recursively this assertion
+          // ensures that the tree entry has all operands set before reaching
+          // this code. Couple of exceptions known at the moment are extracts
+          // where their second (immediate) operand is not added. Since
+          // immediates do not affect scheduler behavior this is considered
+          // okay.
+          auto *In = TE->getMainOp();
+          assert(In &&
+                 (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) ||
+                  In->getNumOperands() == TE->getNumOperands()) &&
+                 "Missed TreeEntry operands?");
+          (void)In; // fake use to avoid build failure when assertions disabled
+
+          for (unsigned OpIdx = 0, NumOperands = TE->getNumOperands();
+               OpIdx != NumOperands; ++OpIdx)
+            if (auto *I = dyn_cast<Instruction>(TE->getOperand(OpIdx)[Lane])) {
+              auto *OpSD = getScheduleData(I);
+              if (OpSD && isInSchedulingRegion(OpSD))
+                Worklist.push_back(OpSD);
+            }
+        } else {
+          // Note: We can't see a bundle which doesn't correspond yet to a tree
+          // node.  Such bundles only exist transiently during buildTree_rec,
+          // and the sole callsite of this routine is before the new bundle
+          // is formed.
+          assert(!SD->NextInBundle);
+
+          // If BundleMember is a stand-alone instruction, no operand reordering
+          // has taken place, so we directly access its operands.
+          for (Use &U : SD->Inst->operands())
+            if (auto *I = dyn_cast<Instruction>(U.get())) {
+              auto *OpSD = getScheduleData(I);
+              if (OpSD && isInSchedulingRegion(OpSD))
+                Worklist.push_back(OpSD);
+            }
+        }
+        // Handle the memory dependencies.
+        for (ScheduleData *MemoryDepSD : SD->MemoryDependencies) {
+          assert(isInSchedulingRegion(MemoryDepSD));
+          Worklist.push_back(MemoryDepSD);
+        }
+      }
+    }
+
+
     void doForAllOpcodes(Value *V,
                          function_ref<void(ScheduleData *SD)> Action) {
       if (ScheduleData *SD = getScheduleData(V))
@@ -7272,13 +7340,13 @@
   Instruction *OldScheduleEnd = ScheduleEnd;
   LLVM_DEBUG(dbgs() << "SLP:  bundle: " << *S.OpValue << "\n");
 
-  auto TryScheduleBundleImpl = [this, OldScheduleEnd, SLP](bool ReSchedule,
-                                                         ScheduleData *Bundle) {
+  auto TryScheduleBundleImpl = [this, OldScheduleEnd, SLP](ScheduleData *Bundle) {
     // The scheduling region got new instructions at the lower end (or it is a
     // new region for the first bundle). This makes it necessary to
     // recalculate all dependencies.
     // It is seldom that this needs to be done a second time after adding the
     // initial bundle to the region.
+    bool ReSchedule = false;
     if (ScheduleEnd != OldScheduleEnd) {
       for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode())
         doForAllOpcodes(I, [](ScheduleData *SD) { SD->clearDependencies(); });
@@ -7316,12 +7384,11 @@
       // Otherwise the compiler may crash trying to incorrectly calculate
       // dependencies and emit instruction in the wrong order at the actual
       // scheduling.
-      TryScheduleBundleImpl(/*ReSchedule=*/false, nullptr);
+      TryScheduleBundleImpl(nullptr);
       return None;
     }
   }
 
-  bool ReSchedule = false;
   for (Value *V : VL) {
     ScheduleData *BundleMember = getScheduleData(V);
     assert(BundleMember &&
@@ -7329,15 +7396,14 @@
     if (!BundleMember->IsScheduled)
       continue;
     // A bundle member was scheduled as single instruction before and now
-    // needs to be scheduled as part of the bundle. We just get rid of the
-    // existing schedule.
-    LLVM_DEBUG(dbgs() << "SLP:  reset schedule because " << *BundleMember
-                      << " was already scheduled\n");
-    ReSchedule = true;
+    // needs to be scheduled as part of the bundle.
+    LLVM_DEBUG(dbgs() << "SLP:  unscheduling nodes used by " << *BundleMember
+                      << " to allow bundle scheduling\n");
+    unschedule(BundleMember);
   }
 
   auto *Bundle = buildBundle(VL);
-  TryScheduleBundleImpl(ReSchedule, Bundle);
+  TryScheduleBundleImpl(Bundle);
   if (!Bundle->isReady()) {
     cancelScheduling(VL, S.OpValue);
     return None;