Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9638,14 +9638,37 @@
 
   auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
   const Align Alignment = getLoadStoreAlignment(&Ingredient);
-  bool CreateGatherScatter = !isConsecutive();
 
   auto &Builder = State.Builder;
-  InnerLoopVectorizer::VectorParts BlockInMaskParts(State.UF);
-  bool isMaskRequired = getMask();
-  if (isMaskRequired)
-    for (unsigned Part = 0; Part < State.UF; ++Part)
-      BlockInMaskParts[Part] = State.get(getMask(), Part);
+
+  if (!isConsecutive()) {
+    State.setDebugLocFromInst(&Ingredient);
+    // Handle Stores:
+    if (SI) {
+      for (unsigned Part = 0; Part < State.UF; ++Part) {
+        Value *StoredVal = State.get(StoredValue, Part);
+        Value *MaskPart = isMasked() ? State.get(getMask(), Part) : nullptr;
+        Value *VectorGep = State.get(getAddr(), Part);
+        Instruction *NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep,
+                                                         Alignment, MaskPart);
+        State.addMetadata(NewSI, SI);
+      }
+      return;
+    }
+
+    // Handle loads.
+    assert(LI && "Must have a load instruction");
+    for (unsigned Part = 0; Part < State.UF; ++Part) {
+      Value *MaskPart = isMasked() ? State.get(getMask(), Part) : nullptr;
+      Value *VectorGep = State.get(getAddr(), Part);
+      Instruction *NewLI =
+          Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart,
+                                     nullptr, "wide.masked.gather");
+      State.addMetadata(NewLI, LI);
+      State.set(getVPSingleValue(), NewLI, Part);
+    }
+    return;
+  }
 
   const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * {
     // Calculate the pointer for the specific unroll-part.
@@ -9676,9 +9699,6 @@
       PartPtr = Builder.CreateGEP(ScalarDataTy, Ptr, NumElt, "", InBounds);
       PartPtr =
           Builder.CreateGEP(ScalarDataTy, PartPtr, LastLane, "", InBounds);
-      if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
-        BlockInMaskParts[Part] =
-            Builder.CreateVectorReverse(BlockInMaskParts[Part], "reverse");
     } else {
       Value *Increment = createStepForVF(Builder, IndexTy, State.VF, Part);
       PartPtr = Builder.CreateGEP(ScalarDataTy, Ptr, Increment, "", InBounds);
@@ -9695,27 +9715,23 @@
     for (unsigned Part = 0; Part < State.UF; ++Part) {
       Instruction *NewSI = nullptr;
       Value *StoredVal = State.get(StoredValue, Part);
-      if (CreateGatherScatter) {
-        Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
-        Value *VectorGep = State.get(getAddr(), Part);
-        NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
-                                            MaskPart);
-      } else {
-        if (isReverse()) {
-          // If we store to reverse consecutive memory locations, then we need
-          // to reverse the order of elements in the stored value.
-          StoredVal = Builder.CreateVectorReverse(StoredVal, "reverse");
-          // We don't want to update the value in the map as it might be used in
-          // another expression. So don't call resetVectorValue(StoredVal).
-        }
-        auto *VecPtr =
-            CreateVecPtr(Part, State.get(getAddr(), VPIteration(0, 0)));
-        if (isMaskRequired)
-          NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
-                                            BlockInMaskParts[Part]);
-        else
-          NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
+      if (isReverse()) {
+        // If we store to reverse consecutive memory locations, then we need
+        // to reverse the order of elements in the stored value.
+        StoredVal = Builder.CreateVectorReverse(StoredVal, "reverse");
+        // We don't want to update the value in the map as it might be used in
+        // another expression. So don't call resetVectorValue(StoredVal).
       }
+      auto *VecPtr =
+        CreateVecPtr(Part, State.get(getAddr(), VPIteration(0, 0)));
+      if (isMasked()) {
+        Value *MaskVal = State.get(getMask(), Part);
+        if (isReverse())
+          MaskVal = Builder.CreateVectorReverse(MaskVal, "reverse");
+        NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
+                                          MaskVal);
+      } else
+        NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
       State.addMetadata(NewSI, SI);
     }
     return;
@@ -9725,30 +9741,23 @@
   assert(LI && "Must have a load instruction");
   State.setDebugLocFromInst(LI);
   for (unsigned Part = 0; Part < State.UF; ++Part) {
+    auto *VecPtr =
+      CreateVecPtr(Part, State.get(getAddr(), VPIteration(0, 0)));
     Value *NewLI;
-    if (CreateGatherScatter) {
-      Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
-      Value *VectorGep = State.get(getAddr(), Part);
-      NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart,
-                                         nullptr, "wide.masked.gather");
-      State.addMetadata(NewLI, LI);
-    } else {
-      auto *VecPtr =
-          CreateVecPtr(Part, State.get(getAddr(), VPIteration(0, 0)));
-      if (isMaskRequired)
-        NewLI = Builder.CreateMaskedLoad(
-            DataTy, VecPtr, Alignment, BlockInMaskParts[Part],
-            PoisonValue::get(DataTy), "wide.masked.load");
-      else
-        NewLI =
-            Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
-
-      // Add metadata to the load, but setVectorValue to the reverse shuffle.
-      State.addMetadata(NewLI, LI);
-      if (Reverse)
-        NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
-    }
+    if (isMasked()) {
+      Value *MaskVal = State.get(getMask(), Part);
+      if (isReverse())
+        MaskVal = Builder.CreateVectorReverse(MaskVal, "reverse");
+      NewLI = Builder.CreateMaskedLoad(DataTy, VecPtr, Alignment, MaskVal,
+        PoisonValue::get(DataTy), "wide.masked.load");
+    } else
+      NewLI =
+        Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
 
+    // Add metadata to the load, but setVectorValue to the reverse shuffle.
+    State.addMetadata(NewLI, LI);
+    if (Reverse)
+      NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
     State.set(getVPSingleValue(), NewLI, Part);
   }
 }