[LV] Reorganize VPWidenMemoryInstructionRecipe::execute [nfc]

[reames]

Summary

Pull out the generic scatter/gather lowering, and refactor the consecutive handling to localize impact of the masking manipulation.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
   assert((LI || SI) && "Invalid Load/Store instruction");
   assert((!SI || StoredValue) && "No stored value provided for widened store");
   assert((!LI || !StoredValue) && "Stored value provided for widened load");
 
   Type *ScalarDataTy = getLoadStoreType(&Ingredient);
 
   auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
   const Align Alignment = getLoadStoreAlignment(&Ingredient);
-  bool CreateGatherScatter = !isConsecutive();
 
   auto &Builder = State.Builder;
-  InnerLoopVectorizer::VectorParts BlockInMaskParts(State.UF);
-  bool isMaskRequired = getMask();
-  if (isMaskRequired)
-    for (unsigned Part = 0; Part < State.UF; ++Part)
-      BlockInMaskParts[Part] = State.get(getMask(), Part);
+
+  if (!isConsecutive()) {
+    State.setDebugLocFromInst(&Ingredient);
+    // Handle Stores:
+    if (SI) {
+      for (unsigned Part = 0; Part < State.UF; ++Part) {
+        Value *StoredVal = State.get(StoredValue, Part);
+        Value *MaskPart = isMasked() ? State.get(getMask(), Part) : nullptr;
+        Value *VectorGep = State.get(getAddr(), Part);
+        Instruction *NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep,
+                                                         Alignment, MaskPart);
+        State.addMetadata(NewSI, SI);
+      }
+      return;
+    }
+
+    // Handle loads.
+    assert(LI && "Must have a load instruction");
+    for (unsigned Part = 0; Part < State.UF; ++Part) {
+      Value *MaskPart = isMasked() ? State.get(getMask(), Part) : nullptr;
+      Value *VectorGep = State.get(getAddr(), Part);
+      Instruction *NewLI =
+          Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart,
+                                     nullptr, "wide.masked.gather");
+      State.addMetadata(NewLI, LI);
+      State.set(getVPSingleValue(), NewLI, Part);
+    }
+    return;
+  }
 
   const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * {
     // Calculate the pointer for the specific unroll-part.
     Value *PartPtr = nullptr;
 
     // Use i32 for the gep index type when the value is constant,
     // or query DataLayout for a more suitable index type otherwise.
     const DataLayout &DL =
if (isReverse()) {
       Value *NumElt =
           Builder.CreateMul(ConstantInt::get(IndexTy, -(int64_t)Part), RunTimeVF);
       // LastLane = 1 - RunTimeVF
       Value *LastLane =
           Builder.CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
       PartPtr = Builder.CreateGEP(ScalarDataTy, Ptr, NumElt, "", InBounds);
       PartPtr =
           Builder.CreateGEP(ScalarDataTy, PartPtr, LastLane, "", InBounds);
-      if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
-        BlockInMaskParts[Part] =
-            Builder.CreateVectorReverse(BlockInMaskParts[Part], "reverse");
     } else {
       Value *Increment = createStepForVF(Builder, IndexTy, State.VF, Part);
       PartPtr = Builder.CreateGEP(ScalarDataTy, Ptr, Increment, "", InBounds);
     }
 
     unsigned AddressSpace = Ptr->getType()->getPointerAddressSpace();
     return Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace));
   };
 
   // Handle Stores:
   if (SI) {
     State.setDebugLocFromInst(SI);
 
     for (unsigned Part = 0; Part < State.UF; ++Part) {
       Instruction *NewSI = nullptr;
       Value *StoredVal = State.get(StoredValue, Part);
-      if (CreateGatherScatter) {
-        Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
-        Value *VectorGep = State.get(getAddr(), Part);
-        NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
-                                            MaskPart);
-      } else {
-        if (isReverse()) {
-          // If we store to reverse consecutive memory locations, then we need
-          // to reverse the order of elements in the stored value.
-          StoredVal = Builder.CreateVectorReverse(StoredVal, "reverse");
-          // We don't want to update the value in the map as it might be used in
-          // another expression. So don't call resetVectorValue(StoredVal).
-        }
-        auto *VecPtr =
-            CreateVecPtr(Part, State.get(getAddr(), VPIteration(0, 0)));
-        if (isMaskRequired)
-          NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
-                                            BlockInMaskParts[Part]);
-        else
-          NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
-      }
+      if (isMasked()) {
+        Value *MaskVal = State.get(getMask(), Part);
+        if (isReverse())
+          MaskVal = Builder.CreateVectorReverse(MaskVal, "reverse");
+                                          MaskVal);
+      } else
       State.addMetadata(NewSI, SI);
     }
     return;
   }
 
   // Handle loads.
   assert(LI && "Must have a load instruction");
   State.setDebugLocFromInst(LI);
   for (unsigned Part = 0; Part < State.UF; ++Part) {
-    Value *NewLI;
-    if (CreateGatherScatter) {
-      Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
-      Value *VectorGep = State.get(getAddr(), Part);
-      NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart,
-                                         nullptr, "wide.masked.gather");
-      State.addMetadata(NewLI, LI);
-    } else {
-      auto *VecPtr =
-          CreateVecPtr(Part, State.get(getAddr(), VPIteration(0, 0)));
-      if (isMaskRequired)
-        NewLI = Builder.CreateMaskedLoad(
-            DataTy, VecPtr, Alignment, BlockInMaskParts[Part],
-            PoisonValue::get(DataTy), "wide.masked.load");
-      else
-        NewLI =
-            Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
+    Value *NewLI;
+    if (isMasked()) {
+      Value *MaskVal = State.get(getMask(), Part);
+      if (isReverse())
+        MaskVal = Builder.CreateVectorReverse(MaskVal, "reverse");
+      NewLI = Builder.CreateMaskedLoad(DataTy, VecPtr, Alignment, MaskVal,
+    } else
 
-      // Add metadata to the load, but setVectorValue to the reverse shuffle.
-      State.addMetadata(NewLI, LI);
-      if (Reverse)
-        NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
-    }
-
     State.set(getVPSingleValue(), NewLI, Part);
   }
 }
 
 // Determine how to lower the scalar epilogue, which depends on 1) optimising
 // for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
 // predication, and 4) a TTI hook that analyses whether the loop is suitable
 // for predication.