[BasicAA] BasicAA update for scalable quantity

[harviniriawan]

Summary

  Allow vscale GEPs to be analysed in BasicAA if comparison is performed
between 2 GEPs that are equal in scalability (i.e. vscale vs vscale)

  This improves Codegen that is dependent on alias analysis
framework (e.g. SVE on AArch64)

[harviniriawan]

I see cases where when we do Loop unrolling for SVE, that we move Load/Stores across iterations although they definitely not aliasing. This patch is trying to improve on that. In theory we should allow alias analysis when done between 2 GEPs that have the same scalable type, and memory location size.

This patch is the first series of patch to improve SVE loop unrolling, to get the full benefit, getSizeOrUnknown() must be updated as well (used in machinememoperand I believe).
Rudimentary prototyping suggests that doing this properly can give us a lot of uplift for SVE, especially in the LITTLE core

[nikic]

I'd suggest to split this into two parts: First add support for scalable LocationSize, and make sure code can handle it correctly, and then add support for scalable offsets to GEP decomposition later. I don't think the way you're currently doing it is what we want.

llvm/include/llvm/Analysis/MemoryLocation.h

107

precise() takes a TypeSize. You should be able to handle the isScalable() case in here only, rather than adjusting all callers to call a separate preciseScalable() method instead.

[harviniriawan]

Hi Nikita,

Yeap sure I can do the adjustment on the precise(method)
Meanwhile, could you please elaborate on what way would you prefer how we handle the scalable offsets on GEP decomposition?

Harvin

[nikic]

In D159451#4640575, @harviniriawan wrote:

Hi Nikita,

Yeap sure I can do the adjustment on the precise(method)
Meanwhile, could you please elaborate on what way would you prefer how we handle the scalable offsets on GEP decomposition?

I think vscale should be part of VariableGEPIndex, as we're modelling an expression of the form vscale * x.

llvm/include/llvm/Analysis/MemoryLocation.h

 
   uint64_t Value;
 
+  bool Scalable = false;
   // Hack to support implicit construction. This should disappear when the
   // public LocationSize ctor goes away.
   enum DirectConstruction { Direct };

   constexpr LocationSize(uint64_t Raw)
       : Value(Raw > MaxValue ? AfterPointer : Raw) {}
 
+  constexpr LocationSize(uint64_t Raw, bool Scalable)
+      : Value(Raw > MaxValue ? AfterPointer : Raw), Scalable(Scalable) {}
+
   static LocationSize precise(uint64_t Value) { return LocationSize(Value); }
   static LocationSize precise(TypeSize Value) {
     if (Value.isScalable())

     return precise(Value.getFixedValue());
   }
 
+  static LocationSize preciseScalable(TypeSize Value) {
+    return LocationSize(Value.getKnownMinValue(), Value.isScalable());
+  }
   static LocationSize upperBound(uint64_t Value) {
     // You can't go lower than 0, so give a precise result.
     if (LLVM_UNLIKELY(Value == 0))

     return (Value & ImpreciseBit) == 0;
   }
 
+  bool isScalable() const { return Scalable; }
   // Convenience method to check if this LocationSize's value is 0.
   bool isZero() const { return hasValue() && getValue() == 0; }
 

llvm/include/llvm/IR/DataLayout.h

     return getTypeSizeInBits(Ty) == getTypeStoreSizeInBits(Ty);
   }
 
+  bool isScalable(Type *Ty) const {
+    TypeSize BaseSize = getTypeSizeInBits(Ty);
+    return BaseSize.isScalable();
+  }
+
   /// Returns the offset in bytes between successive objects of the
   /// specified type, including alignment padding.
   ///

llvm/lib/Analysis/AliasAnalysisEvaluator.cpp

 
   // iterate over the worklist, and run the full (n^2)/2 disambiguations
   for (auto I1 = Pointers.begin(), E = Pointers.end(); I1 != E; ++I1) {
-    LocationSize Size1 = LocationSize::precise(DL.getTypeStoreSize(I1->second));
+    LocationSize Size1 =
+        LocationSize::preciseScalable(DL.getTypeStoreSize(I1->second));
     for (auto I2 = Pointers.begin(); I2 != I1; ++I2) {
       LocationSize Size2 =
-          LocationSize::precise(DL.getTypeStoreSize(I2->second));
+          LocationSize::preciseScalable(DL.getTypeStoreSize(I2->second));
       AliasResult AR = AA.alias(I1->first, Size1, I2->first, Size2);
       switch (AR) {
       case AliasResult::NoAlias:

   for (CallBase *Call : Calls) {
     for (const auto &Pointer : Pointers) {
       LocationSize Size =
-          LocationSize::precise(DL.getTypeStoreSize(Pointer.second));
+          LocationSize::preciseScalable(DL.getTypeStoreSize(Pointer.second));
       switch (AA.getModRefInfo(Call, Pointer.first, Size)) {
       case ModRefInfo::NoModRef:
         PrintModRefResults("NoModRef", PrintNoModRef, Call, Pointer,

llvm/lib/Analysis/BasicAliasAnalysis.cpp

 
 // The max limit of the search depth in DecomposeGEPExpression() and
 // getUnderlyingObject().
-static const unsigned MaxLookupSearchDepth = 6;
+static const unsigned MaxLookupSearchDepth = 7;
 
 bool BasicAAResult::invalidate(Function &Fn, const PreservedAnalyses &PA,
                                FunctionAnalysisManager::Invalidator &Inv) {

   const Value *Base;
   // Total constant offset from base.
   APInt Offset;
+  // Indicate if the offset is scalable (both variable and constant)
+  bool ScalableOffset;
+  // Indicate if there is only one constant offset
+  bool OnlyOneConstOffset;
   // Scaled variable (non-constant) indices.
   SmallVector<VariableGEPIndex, 4> VarIndices;
   // Are all operations inbounds GEPs or non-indexing operations?

     dbgs() << "\n";
   }
   void print(raw_ostream &OS) const {
-    OS << "(DecomposedGEP Base=" << Base->getName()
-       << ", Offset=" << Offset
-       << ", VarIndices=[";
+    OS << "(DecomposedGEP Base=" << Base->getName() << ", Offset=" << Offset
+       << ", ScalableOffset=" << ScalableOffset << ", VarIndices=[";
     for (size_t i = 0; i < VarIndices.size(); i++) {
       if (i != 0)
         OS << ", ";

   unsigned MaxIndexSize = DL.getMaxIndexSizeInBits();
   DecomposedGEP Decomposed;
   Decomposed.Offset = APInt(MaxIndexSize, 0);
+  Decomposed.ScalableOffset = false;
+  Decomposed.OnlyOneConstOffset = false;
   do {
     // See if this is a bitcast or GEP.
     const Operator *Op = dyn_cast<Operator>(V);

     // Walk the indices of the GEP, accumulating them into BaseOff/VarIndices.
     gep_type_iterator GTI = gep_type_begin(GEPOp);
     unsigned IndexSize = DL.getIndexSizeInBits(AS);
-    // Assume all GEP operands are constants until proven otherwise.
     bool GepHasConstantOffset = true;
     for (User::const_op_iterator I = GEPOp->op_begin() + 1, E = GEPOp->op_end();
          I != E; ++I, ++GTI) {

         Decomposed.Offset += DL.getStructLayout(STy)->getElementOffset(FieldNo);
         continue;
       }
-
+      TypeSize AllocTypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
       // For an array/pointer, add the element offset, explicitly scaled.
       if (const ConstantInt *CIdx = dyn_cast<ConstantInt>(Index)) {
-        if (CIdx->isZero())
-          continue;
-
-        // Don't attempt to analyze GEPs if the scalable index is not zero.
-        TypeSize AllocTypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
         if (AllocTypeSize.isScalable()) {
           Decomposed.Base = V;
-          return Decomposed;
+          Decomposed.ScalableOffset = true;
         }
+        if (CIdx->isZero())
+          continue;
 
-        Decomposed.Offset += AllocTypeSize.getFixedValue() *
+        Decomposed.Offset += AllocTypeSize.getKnownMinValue() *
                              CIdx->getValue().sextOrTrunc(MaxIndexSize);
+        if (!Decomposed.OnlyOneConstOffset)
+          Decomposed.OnlyOneConstOffset = true;
         continue;
       }
 
-      TypeSize AllocTypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
-      if (AllocTypeSize.isScalable()) {
-        Decomposed.Base = V;
-        return Decomposed;
-      }
-
       GepHasConstantOffset = false;
 
       // If the integer type is smaller than the index size, it is implicitly

           CastedValue(Index, 0, SExtBits, TruncBits), DL, 0, AC, DT);
 
       // Scale by the type size.
-      unsigned TypeSize = AllocTypeSize.getFixedValue();
+      unsigned TypeSize = AllocTypeSize.getKnownMinValue();
       LE = LE.mul(APInt(IndexSize, TypeSize), GEPOp->isInBounds());
       Decomposed.Offset += LE.Offset.sext(MaxIndexSize);
       APInt Scale = LE.Scale.sext(MaxIndexSize);

   if (DecompGEP1.Base == GEP1 && DecompGEP2.Base == V2)
     return AliasResult::MayAlias;
 
+  // If we compare 2 GEPs, one has Vscale quantity and one is not
+  // but the offset are both 0 and there's only one index,
+  // they will alias if the base address alias
+  if (((DecompGEP1.Offset == DecompGEP2.Offset) == 0) &&
+      (DecompGEP1.ScalableOffset != DecompGEP2.ScalableOffset) &&
+      (V1Size.hasValue() && V2Size.hasValue()) &&
+      DecompGEP1.VarIndices.empty() &&
+      ((DecompGEP1.OnlyOneConstOffset == DecompGEP2.OnlyOneConstOffset) ==
+       true))
+    return AAQI.AAR.alias(MemoryLocation::getBeforeOrAfter(DecompGEP1.Base),
+                          MemoryLocation::getBeforeOrAfter(DecompGEP2.Base),
+                          AAQI);
+
   // Subtract the GEP2 pointer from the GEP1 pointer to find out their
   // symbolic difference.
   subtractDecomposedGEPs(DecompGEP1, DecompGEP2, AAQI);

 
   // For GEPs with identical offsets, we can preserve the size and AAInfo
   // when performing the alias check on the underlying objects.
-  if (DecompGEP1.Offset == 0 && DecompGEP1.VarIndices.empty())
+  // The 2 GEPs must have equal scalable type
+  bool OffsetZeroCheck;
+  OffsetZeroCheck =
+      isa<GEPOperator>(V2)
+          ? (DecompGEP1.ScalableOffset == DecompGEP2.ScalableOffset)
+          : 1;
+  if (DecompGEP1.Offset == 0 && DecompGEP1.VarIndices.empty() &&
+      OffsetZeroCheck)
     return AAQI.AAR.alias(MemoryLocation(DecompGEP1.Base, V1Size),
                           MemoryLocation(DecompGEP2.Base, V2Size), AAQI);
 

     return BaseAlias;
   }
 
+  // If the two GEPs have differing ScalableOffset value, return MayAlias
+  if ((DecompGEP1.ScalableOffset != DecompGEP2.ScalableOffset) &&
+      isa<GEPOperator>(V2))
+    return AliasResult::MayAlias;
+
   // If there is a constant difference between the pointers, but the difference
   // is less than the size of the associated memory object, then we know
   // that the objects are partially overlapping.  If the difference is

     LocationSize VLeftSize = V2Size;
     LocationSize VRightSize = V1Size;
     const bool Swapped = Off.isNegative();
+    const bool SameScalableLoc =
+        VLeftSize.isScalable() == VRightSize.isScalable();
+    // const bool SameScalable = 1;
 
     if (Swapped) {
       // Swap if we have the situation where:

       return AliasResult::MayAlias;
 
     const uint64_t LSize = VLeftSize.getValue();
-    if (Off.ult(LSize)) {
+    if (Off.ult(LSize) && SameScalableLoc) {
       // Conservatively drop processing if a phi was visited and/or offset is
       // too big.
       AliasResult AR = AliasResult::PartialAlias;

       }
       return AR;
     }
-    return AliasResult::NoAlias;
+    if (SameScalableLoc)
+      return AliasResult::NoAlias;
   }
 
   // We need to know both acess sizes for all the following heuristics.
   if (!V1Size.hasValue() || !V2Size.hasValue())
     return AliasResult::MayAlias;
 
+  // TODO: Enable vscale analysis on variable quantities
+  if (V1Size.isScalable() || V2Size.isScalable())
+    return AliasResult::MayAlias;
+
+  if (DecompGEP1.ScalableOffset ||
+      (DecompGEP2.ScalableOffset && isa<GEPOperator>(V2)))
+    return AliasResult::MayAlias;
+
   APInt GCD;
   ConstantRange OffsetRange = ConstantRange(DecompGEP1.Offset);
   for (unsigned i = 0, e = DecompGEP1.VarIndices.size(); i != e; ++i) {

llvm/lib/Analysis/MemoryLocation.cpp

     OS << "mapEmpty";
   else if (*this == mapTombstone())
     OS << "mapTombstone";
-  else if (isPrecise())
+  else if (isPrecise() && !isScalable())
     OS << "precise(" << getValue() << ')';
+  else if (isPrecise() && isScalable())
+    OS << "precise(vscale x " << getValue() << ')';
   else
     OS << "upperBound(" << getValue() << ')';
 }

 MemoryLocation MemoryLocation::get(const LoadInst *LI) {
   const auto &DL = LI->getModule()->getDataLayout();
 
-  return MemoryLocation(
-      LI->getPointerOperand(),
-      LocationSize::precise(DL.getTypeStoreSize(LI->getType())),
-      LI->getAAMetadata());
+  if (DL.isScalable(LI->getType()))
+    return MemoryLocation(
+        LI->getPointerOperand(),
+        LocationSize::preciseScalable(DL.getTypeStoreSize(LI->getType())),
+        LI->getAAMetadata());
+  else
+    return MemoryLocation(
+        LI->getPointerOperand(),
+        LocationSize::precise(DL.getTypeStoreSize(LI->getType())),
+        LI->getAAMetadata());
 }
 
 MemoryLocation MemoryLocation::get(const StoreInst *SI) {
   const auto &DL = SI->getModule()->getDataLayout();
 
-  return MemoryLocation(SI->getPointerOperand(),
-                        LocationSize::precise(DL.getTypeStoreSize(
-                            SI->getValueOperand()->getType())),
-                        SI->getAAMetadata());
+  if (DL.isScalable(SI->getValueOperand()->getType())) {
+    return MemoryLocation(SI->getPointerOperand(),
+                          LocationSize::preciseScalable(DL.getTypeStoreSize(
+                              SI->getValueOperand()->getType())),
+                          SI->getAAMetadata());
+  } else
+    return MemoryLocation(SI->getPointerOperand(),
+                          LocationSize::precise(DL.getTypeStoreSize(
+                              SI->getValueOperand()->getType())),
+                          SI->getAAMetadata());
 }
 
 MemoryLocation MemoryLocation::get(const VAArgInst *VI) {

llvm/test/Analysis/BasicAA/gep-decomposition-limit.ll

 ; CHECK-DAG: NoAlias: i8* %gep.inc3, i8* %gep.inc5
 ; CHECK-DAG: NoAlias: i8* %gep.inc4, i8* %gep.inc5
 ;; At limit:
-; CHECK-DAG: MustAlias: i8* %gep.add6, i8* %gep.inc6
-; CHECK-DAG: NoAlias: i8* %gep.inc4, i8* %gep.inc6
-; CHECK-DAG: NoAlias: i8* %gep.inc5, i8* %gep.inc6
-;; After limit:
-; CHECK-DAG: MayAlias: i8* %gep.add7, i8* %gep.inc7
-; CHECK-DAG: MayAlias: i8* %gep.inc5, i8* %gep.inc7
+; CHECK-DAG: MustAlias: i8* %gep.add7, i8* %gep.inc7
 ; CHECK-DAG: NoAlias: i8* %gep.inc6, i8* %gep.inc7
+; CHECK-DAG: NoAlias: i8* %gep.inc5, i8* %gep.inc7
+;; After limit:
+; CHECK-DAG: MayAlias: i8* %gep.add8, i8* %gep.inc8
+; CHECK-DAG: NoAlias: i8* %gep.inc7, i8* %gep.inc8
+; CHECK-DAG: MayAlias: i8* %gep.inc6, i8* %gep.inc8
 
 define void @test(ptr %base) {
   %gep.add5 = getelementptr i8, ptr %base, i64 5
   %gep.add6 = getelementptr i8, ptr %base, i64 6
   %gep.add7 = getelementptr i8, ptr %base, i64 7
+  %gep.add8 = getelementptr i8, ptr %base, i64 8
 
   %gep.inc1 = getelementptr i8, ptr %base, i64 1
   %gep.inc2 = getelementptr i8, ptr %gep.inc1, i64 1

   %gep.inc5 = getelementptr i8, ptr %gep.inc4, i64 1
   %gep.inc6 = getelementptr i8, ptr %gep.inc5, i64 1
   %gep.inc7 = getelementptr i8, ptr %gep.inc6, i64 1
+  %gep.inc8 = getelementptr i8, ptr %gep.inc7, i64 1
 
   load i8, ptr %gep.add5
   load i8, ptr %gep.add6
   load i8, ptr %gep.add7
+  load i8, ptr %gep.add8
   load i8, ptr %gep.inc3
   load i8, ptr %gep.inc4
   load i8, ptr %gep.inc5
   load i8, ptr %gep.inc6
   load i8, ptr %gep.inc7
+  load i8, ptr %gep.inc8
 
   ret void
 }

llvm/test/Analysis/BasicAA/vscale.ll

 
 ; CHECK-LABEL: gep_alloca_const_offset_1
 ; CHECK-DAG:  MustAlias:    <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep1
-; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep2
-; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
+; CHECK-DAG:  NoAlias:     <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep2
+; CHECK-DAG:  NoAlias:     <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
 define void @gep_alloca_const_offset_1() {
   %alloc = alloca <vscale x 4 x i32>
   %gep1 = getelementptr <vscale x 4 x i32>, ptr %alloc, i64 0

 }
 
 ; CHECK-LABEL: gep_alloca_const_offset_2
-; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep1
-; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep2
-; TODO: AliasResult for gep1,gep2 can be improved as MustAlias
-; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
+; CHECK-DAG:  NoAlias:     <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep1
+; CHECK-DAG:  NoAlias:     <vscale x 4 x i32>* %alloc, <vscale x 4 x i32>* %gep2
+; CHECK-DAG:  MustAlias:     <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %gep2
 define void @gep_alloca_const_offset_2() {
   %alloc = alloca <vscale x 4 x i32>
   %gep1 = getelementptr <vscale x 4 x i32>, ptr %alloc, i64 1

 ; CHECK-LABEL: gep_same_base_const_offset
 ; CHECK-DAG:  MayAlias:     i32* %gep1, <vscale x 4 x i32>* %p
 ; CHECK-DAG:  MayAlias:     i32* %gep2, <vscale x 4 x i32>* %p
-; TODO: AliasResult for gep1,gep2 can be improved as NoAlias
-; CHECK-DAG:  MayAlias:     i32* %gep1, i32* %gep2
+; CHECK-DAG:  NoAlias:      i32* %gep1, i32* %gep2
 define void @gep_same_base_const_offset(ptr %p) {
   %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 0
   %gep2 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 1

 }
 
 ; CHECK-LABEL: gep_different_base_const_offset
-; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %p1
-; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %gep2, <vscale x 4 x i32>* %p2
+; CHECK-DAG:  NoAlias:     <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %p1
+; CHECK-DAG:  NoAlias:     <vscale x 4 x i32>* %gep2, <vscale x 4 x i32>* %p2
 ; CHECK-DAG:  NoAlias:      <vscale x 4 x i32>* %p1, <vscale x 4 x i32>* %p2
 ; CHECK-DAG:  NoAlias:      <vscale x 4 x i32>* %gep1, <vscale x 4 x i32>* %p2
 ; CHECK-DAG:  NoAlias:      <vscale x 4 x i32>* %gep2, <vscale x 4 x i32>* %p1

 ; CHECK-LABEL: gep_bitcast_1
 ; CHECK-DAG:   MustAlias:    i32* %p, <vscale x 4 x i32>* %p
 ; CHECK-DAG:   MayAlias:     i32* %gep1, <vscale x 4 x i32>* %p
-; CHECK-DAG:   MayAlias:     i32* %gep1, i32* %p
+; CHECK-DAG:   NoAlias:      i32* %gep1, i32* %p
 ; CHECK-DAG:   MayAlias:     i32* %gep2, <vscale x 4 x i32>* %p
 ; CHECK-DAG:   MayAlias:     i32* %gep1, i32* %gep2
 ; CHECK-DAG:   NoAlias:      i32* %gep2, i32* %p

 ; CHECK-DAG:  MayAlias:     i32* %gep1, <vscale x 4 x i32>* %p
 ; CHECK-DAG:  MayAlias:     i32* %gep1, <vscale x 4 x float>* %p
 ; CHECK-DAG:  MayAlias:     float* %gep2, <vscale x 4 x i32>* %p
-; CHECK-DAG:  MayAlias:     i32* %gep1, float* %gep2
+; CHECK-DAG:  MustAlias:    i32* %gep1, float* %gep2
 ; CHECK-DAG:  MayAlias:     float* %gep2, <vscale x 4 x float>* %p
 define void @gep_bitcast_2(ptr %p) {
   %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 0

 
 ; CHECK-LABEL: gep_recursion_level_1_bitcast
 ; CHECK-DAG:  MustAlias:    i32* %a, <vscale x 4 x i32>* %a
-; CHECK-DAG:  MayAlias:     i32* %a, i32* %gep
-; CHECK-DAG:  MayAlias:     i32* %a, i32* %gep_rec_1
+; CHECK-DAG:  NoAlias:      i32* %a, i32* %gep
+; CHECK-DAG:  NoAlias:      i32* %a, i32* %gep_rec_1
 ; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %a, i32* %gep
 ; CHECK-DAG:  MayAlias:     <vscale x 4 x i32>* %a, i32* %gep_rec_1
 ; CHECK-DAG:  NoAlias:      i32* %gep, i32* %gep_rec_1

llvm/test/Transforms/GVN/vscale.ll

 ; CHECK-LABEL: @load_clobber_load_gep3(
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 1, i64 0
 ; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, ptr [[GEP1]], align 4
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr <vscale x 4 x float>, ptr [[P]], i64 1, i64 0
-; CHECK-NEXT:    [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
-; CHECK-NEXT:    [[CAST:%.*]] = bitcast float [[LOAD2]] to i32
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LOAD1]], [[CAST]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LOAD1]], [[LOAD1]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 0

 ; CHECK-NEXT:    store i32 1, ptr [[GEP2]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[T:%.*]] = load i32, ptr [[GEP1]], align 4
-; CHECK-NEXT:    store i32 [[T]], ptr [[Q:%.*]], align 4
+; CHECK-NEXT:    store i32 0, ptr [[Q:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ; CHECK:       if.else:
 ; CHECK-NEXT:    ret void

 ; CHECK-NEXT:    store <vscale x 4 x i32> [[V:%.*]], ptr [[P1]], align 16
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[T:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16
-; CHECK-NEXT:    store <vscale x 4 x i32> [[T]], ptr [[Q:%.*]], align 16
+; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr [[Q:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ; CHECK:       if.else:
 ; CHECK-NEXT:    ret void