Index: lib/Transforms/Vectorize/SLPVectorizer.cpp
===================================================================
--- lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1113,6 +1113,10 @@
     /// Sets all instruction in the scheduling region to un-scheduled.
     void resetSchedule();
 
+    /// Reorder ScheduleData data after scheduling, so that the first instruction
+    /// in the list would be the last one scheduled.
+    void reorderBundles();
+
     BasicBlock *BB;
 
     /// Simple memory allocation for ScheduleData.
@@ -2869,17 +2873,15 @@
   // The last instruction in the bundle in program order.
   Instruction *LastInst = nullptr;
 
-  // Find the last instruction. The common case should be that BB has been
-  // scheduled, and the last instruction is VL.back(). So we start with
-  // VL.back() and iterate over schedule data until we reach the end of the
-  // bundle. The end of the bundle is marked by null ScheduleData.
+  // Find the last instruction. If the bundle is not scheduled then
+  // the first in the bundle is the last one in BB, because we discover
+  // bundles in a backward walk. And if the bundle is already scheduled
+  // the we reorder one to be the last one scheduled in the first position.
   if (BlocksSchedules.count(BB)) {
     auto *Bundle =
         BlocksSchedules[BB]->getScheduleData(isOneOf(S, VL.back()));
     if (Bundle && Bundle->isPartOfBundle())
-      for (; Bundle; Bundle = Bundle->NextInBundle)
-        if (Bundle->OpValue == Bundle->Inst)
-          LastInst = Bundle->Inst;
+      LastInst = Bundle->FirstInBundle->Inst;
   }
 
   // LastInst can still be null at this point if there's either not an entry
@@ -4190,6 +4192,47 @@
   ReadyInsts.clear();
 }
 
+void BoUpSLP::BlockScheduling::reorderBundles() {
+  DenseMap<ScheduleData *, ScheduleData *> ReorderMap;
+  // Walk backward in the scheduling region to descover last instruction
+  // for a bundle.
+  for (auto *I = ScheduleEnd; I != ScheduleStart; I = I->getPrevNode()) {
+    doForAllOpcodes(I, [&ReorderMap, &I](ScheduleData *SD) {
+      if (SD->isPartOfBundle() &&
+          ReorderMap.find(SD->FirstInBundle) == ReorderMap.end()) {
+        ReorderMap[SD->FirstInBundle] = SD;
+      }
+    });
+  }
+  DenseMap<ScheduleData *, ScheduleData *>::iterator it;
+  // Swap the last scheduled instruction with the first one in the bundle.
+  for (it = ReorderMap.begin(); it != ReorderMap.end(); it++) {
+    ScheduleData *FirstSD = it->first;
+    ScheduleData *LastSD = it->second;
+    SmallVector<ScheduleData *, 4> Bundle;
+    unsigned LastPos = 0;
+    // The first instruction in the bundle is already the last one scheduled.
+    if (FirstSD == LastSD)
+      continue;
+    ScheduleData *SD = FirstSD;
+    while (SD) {
+      if (SD == LastSD)
+        LastPos = Bundle.size();
+      Bundle.push_back(SD);
+      SD = SD->NextInBundle;
+    }
+    std::swap(Bundle[0], Bundle[LastPos]);
+    for (ScheduleData *SD : Bundle)
+      SD->FirstInBundle = Bundle[0];
+    Bundle[0]->NextInBundle = Bundle[1];
+    Bundle[LastPos - 1]->NextInBundle = Bundle[LastPos];
+    if (LastPos == Bundle.size() - 1)
+      Bundle[LastPos]->NextInBundle = nullptr;
+    else
+      Bundle[LastPos]->NextInBundle = Bundle[LastPos + 1];
+  }
+}
+
 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
   if (!BS->ScheduleStart)
     return;
@@ -4252,6 +4295,7 @@
     NumToSchedule--;
   }
   assert(NumToSchedule == 0 && "could not schedule all instructions");
+  BS->reorderBundles();
 
   // Avoid duplicate scheduling of the block.
   BS->ScheduleStart = nullptr;