[LoopIdiomRecognize] Optimize cases when loop contains extra stores and loads

[eklepilkina]

Summary

The replacement with memset can be done only if there is no another instruction that aliases with optimized stores.
There are some cases when more precise memory access can be calculated for store and load instruction that allows to check aliasing of memory access more accurate and make optimizations in such cases.

[eklepilkina]

Gently ping

[eklepilkina]

Gently ping

llvm/include/llvm/Analysis/AliasAnalysis.h

public:
   ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc,
                            AAQueryInfo &AAQI);
   ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2,
                            AAQueryInfo &AAQI);
   ModRefInfo getModRefInfo(const VAArgInst *V, const MemoryLocation &Loc,
                            AAQueryInfo &AAQI);
   ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc,
                            AAQueryInfo &AAQI);
+  ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc,
+                           const std::optional<MemoryLocation> &LoadLocation,
+                           AAQueryInfo &AAQI);
+  ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc,
+                           AAQueryInfo &AAQI);
   ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc,
+                           const std::optional<MemoryLocation> &StoreLocation,
                            AAQueryInfo &AAQI);
   ModRefInfo getModRefInfo(const FenceInst *S, const MemoryLocation &Loc,
                            AAQueryInfo &AAQI);
   ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX,
                            const MemoryLocation &Loc, AAQueryInfo &AAQI);
   ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc,
                            AAQueryInfo &AAQI);
   ModRefInfo getModRefInfo(const CatchPadInst *I, const MemoryLocation &Loc,

llvm/lib/Analysis/AliasAnalysis.cpp

raw_ostream &llvm::operator<<(raw_ostream &OS, MemoryEffects ME) {
   }
   return OS;
 }
 
 //===----------------------------------------------------------------------===//
 // Helper method implementation
 //===----------------------------------------------------------------------===//
 
-ModRefInfo AAResults::getModRefInfo(const LoadInst *L,
-                                    const MemoryLocation &Loc,
-                                    AAQueryInfo &AAQI) {
+ModRefInfo
+AAResults::getModRefInfo(const LoadInst *L, const MemoryLocation &Loc,
+                         const std::optional<MemoryLocation> &LoadLocation,
   // Be conservative in the face of atomic.
   if (isStrongerThan(L->getOrdering(), AtomicOrdering::Unordered))
     return ModRefInfo::ModRef;
 
   // If the load address doesn't alias the given address, it doesn't read
   // or write the specified memory.
   if (Loc.Ptr) {
-    AliasResult AR = alias(MemoryLocation::get(L), Loc, AAQI, L);
+    AliasResult AR =
+        alias(LoadLocation.value_or(MemoryLocation::get(L)), Loc, AAQI, L);
     if (AR == AliasResult::NoAlias)
       return ModRefInfo::NoModRef;
   }
   // Otherwise, a load just reads.
   return ModRefInfo::Ref;
 }
-
-ModRefInfo AAResults::getModRefInfo(const StoreInst *S,
+ModRefInfo AAResults::getModRefInfo(const LoadInst *L,
                                     const MemoryLocation &Loc,
                                     AAQueryInfo &AAQI) {
+  return getModRefInfo(L, Loc, std::nullopt, AAQI);
+}
+
+ModRefInfo
+AAResults::getModRefInfo(const StoreInst *S, const MemoryLocation &Loc,
+                         const std::optional<MemoryLocation> &StoreLocation,
+                         AAQueryInfo &AAQI) {
   // Be conservative in the face of atomic.
   if (isStrongerThan(S->getOrdering(), AtomicOrdering::Unordered))
     return ModRefInfo::ModRef;
 
   if (Loc.Ptr) {
-    AliasResult AR = alias(MemoryLocation::get(S), Loc, AAQI, S);
+    AliasResult AR =
+        alias(StoreLocation.value_or(MemoryLocation::get(S)), Loc, AAQI, S);
     // If the store address cannot alias the pointer in question, then the
     // specified memory cannot be modified by the store.
     if (AR == AliasResult::NoAlias)
       return ModRefInfo::NoModRef;
 
     // Examine the ModRef mask. If Mod isn't present, then return NoModRef.
     // This ensures that if Loc is a constant memory location, we take into
     // account the fact that the store definitely could not modify the memory
     // location.
     if (!isModSet(getModRefInfoMask(Loc)))
       return ModRefInfo::NoModRef;
   }
 
   // Otherwise, a store just writes.
   return ModRefInfo::Mod;
 }
 
+ModRefInfo AAResults::getModRefInfo(const StoreInst *S,
+                                    const MemoryLocation &Loc,
+                                    AAQueryInfo &AAQI) {
+  return getModRefInfo(S, Loc, std::nullopt, AAQI);
+}
+
 ModRefInfo AAResults::getModRefInfo(const FenceInst *S,
                                     const MemoryLocation &Loc,
                                     AAQueryInfo &AAQI) {
   // All we know about a fence instruction is what we get from the ModRef
   // mask: if Loc is a constant memory location, the fence definitely could
   // not modify it.
   if (Loc.Ptr)
     return getModRefInfoMask(Loc);

llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp

bool processLoopStoreOfLoopLoad(Value *DestPtr, Value *SourcePtr,
                                   const SCEV *StoreSize, MaybeAlign StoreAlign,
                                   MaybeAlign LoadAlign, Instruction *TheStore,
                                   Instruction *TheLoad,
                                   const SCEVAddRecExpr *StoreEv,
                                   const SCEVAddRecExpr *LoadEv,
                                   const SCEV *BECount);
   bool avoidLIRForMultiBlockLoop(bool IsMemset = false,
                                  bool IsLoopMemset = false);
+  std::optional<MemoryLocation>
+  calculateStoreLoadMemoryLocation(Value *InstrPtr, const SCEV *BECount,
+                                   SCEVExpander &Expander);
+  bool mayLoopAccessLocation(Value *Ptr, ModRefInfo Access, Loop *L,
+                             const SCEV *BECount, const SCEV *StoreSizeSCEV,
+                             AliasAnalysis &AA,
+                             SmallPtrSetImpl<Instruction *> &IgnoredInsts,
+                             SCEVExpander &Expander);
 
   /// @}
   /// \name Noncountable Loop Idiom Handling
   /// @{
 
   bool runOnNoncountableLoop();
 
   bool recognizePopcount();
for (auto *BB : CurLoop->getBlocks()) {
     if (LI->getLoopFor(BB) != CurLoop)
       continue;
 
     MadeChange |= runOnLoopBlock(BB, BECount, ExitBlocks);
   }
   return MadeChange;
 }
 
-static APInt getStoreStride(const SCEVAddRecExpr *StoreEv) {
+static APInt getInstrStride(const SCEVAddRecExpr *StoreEv) {
   const SCEVConstant *ConstStride = cast<SCEVConstant>(StoreEv->getOperand(1));
   return ConstStride->getAPInt();
 }
 
 /// getMemSetPatternValue - If a strided store of the specified value is safe to
 /// turn into a memset_pattern16, return a ConstantArray of 16 bytes that should
 /// be passed in.  Otherwise, return null.
 ///
if (!UnorderedAtomic && HasMemsetPattern && !DisableLIRP::Memset &&
     // It looks like we can use PatternValue!
     return LegalStoreKind::MemsetPattern;
   }
 
   // Otherwise, see if the store can be turned into a memcpy.
   if (HasMemcpy && !DisableLIRP::Memcpy) {
     // Check to see if the stride matches the size of the store.  If so, then we
     // know that every byte is touched in the loop.
-    APInt Stride = getStoreStride(StoreEv);
+    APInt Stride = getInstrStride(StoreEv);
     unsigned StoreSize = DL->getTypeStoreSize(SI->getValueOperand()->getType());
     if (StoreSize != Stride && StoreSize != -Stride)
       return LegalStoreKind::None;
 
     // The store must be feeding a non-volatile load.
     LoadInst *LI = dyn_cast<LoadInst>(SI->getValueOperand());
 
     // Only allow non-volatile loads
bool LoopIdiomRecognize::processLoopStores(SmallVectorImpl<StoreInst *> &SL,
   SmallVector<unsigned, 16> IndexQueue;
   for (unsigned i = 0, e = SL.size(); i < e; ++i) {
     assert(SL[i]->isSimple() && "Expected only non-volatile stores.");
 
     Value *FirstStoredVal = SL[i]->getValueOperand();
     Value *FirstStorePtr = SL[i]->getPointerOperand();
     const SCEVAddRecExpr *FirstStoreEv =
         cast<SCEVAddRecExpr>(SE->getSCEV(FirstStorePtr));
-    APInt FirstStride = getStoreStride(FirstStoreEv);
+    APInt FirstStride = getInstrStride(FirstStoreEv);
     unsigned FirstStoreSize = DL->getTypeStoreSize(SL[i]->getValueOperand()->getType());
 
     // See if we can optimize just this store in isolation.
     if (FirstStride == FirstStoreSize || -FirstStride == FirstStoreSize) {
       Heads.insert(SL[i]);
       continue;
     }
 
for (unsigned i = 0, e = SL.size(); i < e; ++i) {
     for (j = i; j > 0; --j)
       IndexQueue.push_back(j - 1);
 
     for (auto &k : IndexQueue) {
       assert(SL[k]->isSimple() && "Expected only non-volatile stores.");
       Value *SecondStorePtr = SL[k]->getPointerOperand();
       const SCEVAddRecExpr *SecondStoreEv =
           cast<SCEVAddRecExpr>(SE->getSCEV(SecondStorePtr));
-      APInt SecondStride = getStoreStride(SecondStoreEv);
+      APInt SecondStride = getInstrStride(SecondStoreEv);
 
       if (FirstStride != SecondStride)
         continue;
 
       Value *SecondStoredVal = SL[k]->getValueOperand();
       Value *SecondSplatValue = nullptr;
       Constant *SecondPatternValue = nullptr;
 
while (Tails.count(I) || Heads.count(I)) {
       StoreSize += DL->getTypeStoreSize(I->getValueOperand()->getType());
       // Move to the next value in the chain.
       I = ConsecutiveChain[I];
     }
 
     Value *StoredVal = HeadStore->getValueOperand();
     Value *StorePtr = HeadStore->getPointerOperand();
     const SCEVAddRecExpr *StoreEv = cast<SCEVAddRecExpr>(SE->getSCEV(StorePtr));
-    APInt Stride = getStoreStride(StoreEv);
+    APInt Stride = getInstrStride(StoreEv);
 
     // Check to see if the stride matches the size of the stores.  If so, then
     // we know that every byte is touched in the loop.
     if (StoreSize != Stride && StoreSize != -Stride)
       continue;
 
     bool IsNegStride = StoreSize == -Stride;
 
bool LoopIdiomRecognize::processLoopMemSet(MemSetInst *MSI,
   return processLoopStridedStore(Pointer, SE->getSCEV(MSI->getLength()),
                                  MSI->getDestAlign(), SplatValue, MSI, MSIs, Ev,
                                  BECount, IsNegStride, /*IsLoopMemset=*/true);
 }
 
 /// mayLoopAccessLocation - Return true if the specified loop might access the
 /// specified pointer location, which is a loop-strided access.  The 'Access'
 /// argument specifies what the verboten forms of access are (read or write).
-static bool
-mayLoopAccessLocation(Value *Ptr, ModRefInfo Access, Loop *L,
-                      const SCEV *BECount, const SCEV *StoreSizeSCEV,
-                      AliasAnalysis &AA,
-                      SmallPtrSetImpl<Instruction *> &IgnoredInsts) {
+bool LoopIdiomRecognize::mayLoopAccessLocation(
+    Value *Ptr, ModRefInfo Access, Loop *L, const SCEV *BECount,
+    const SCEV *StoreSizeSCEV, AliasAnalysis &AA,
+    SmallPtrSetImpl<Instruction *> &IgnoredInsts, SCEVExpander &Expander) {
   // Get the location that may be stored across the loop.  Since the access is
   // strided positively through memory, we say that the modified location starts
   // at the pointer and has infinite size.
   LocationSize AccessSize = LocationSize::afterPointer();
 
   // If the loop iterates a fixed number of times, we can refine the access size
   // to be exactly the size of the memset, which is (BECount+1)*StoreSize
   const SCEVConstant *BECst = dyn_cast<SCEVConstant>(BECount);
   const SCEVConstant *ConstSize = dyn_cast<SCEVConstant>(StoreSizeSCEV);
   if (BECst && ConstSize)
     AccessSize = LocationSize::precise((BECst->getValue()->getZExtValue() + 1) *
                                        ConstSize->getValue()->getZExtValue());
 
-  // TODO: For this to be really effective, we have to dive into the pointer
-  // operand in the store.  Store to &A[i] of 100 will always return may alias
-  // with store of &A[100], we need to StoreLoc to be "A" with size of 100,
-  // which will then no-alias a store to &A[100].
   MemoryLocation StoreLoc(Ptr, AccessSize);
 
   for (BasicBlock *B : L->blocks())
-    for (Instruction &I : *B)
-      if (!IgnoredInsts.contains(&I) &&
-          isModOrRefSet(AA.getModRefInfo(&I, StoreLoc) & Access))
+    for (Instruction &I : *B) {
+      if (IgnoredInsts.contains(&I))
+        continue;
+      ModRefInfo RefInfo = AA.getModRefInfo(&I, StoreLoc);
+
+      // In case of store/load instructions try to calculate more precise
+      // memory location.
+      StoreInst *SI = dyn_cast<StoreInst>(&I);
+      LoadInst *LI = dyn_cast<LoadInst>(&I);
+      if (SI || LI) {
+        Value *InstPtr = SI ? SI->getPointerOperand() : LI->getPointerOperand();
+        auto MemLocation =
+            calculateStoreLoadMemoryLocation(InstPtr, BECount, Expander);
+        if (MemLocation) {
+          SimpleAAQueryInfo AAQIP(AA);
+          if (SI)
+            RefInfo = AA.getModRefInfo(SI, StoreLoc, MemLocation, AAQIP);
+          else if (LI)
+            RefInfo = AA.getModRefInfo(LI, StoreLoc, MemLocation, AAQIP);
+        }
+      }
+      if (isModOrRefSet(RefInfo & Access))
         return true;
+    }
   return false;
 }
 
 // If we have a negative stride, Start refers to the end of the memory location
 // we're trying to memset.  Therefore, we need to recompute the base pointer,
 // which is just Start - BECount*Size.
 static const SCEV *getStartForNegStride(const SCEV *Start, const SCEV *BECount,
                                         Type *IntPtr, const SCEV *StoreSizeSCEV,
static const SCEV *getNumBytes(const SCEV *BECount, Type *IntPtr,
                                const DataLayout *DL, ScalarEvolution *SE) {
   const SCEV *TripCountSCEV = getTripCount(BECount, IntPtr, CurLoop, DL, SE);
 
   return SE->getMulExpr(TripCountSCEV,
                         SE->getTruncateOrZeroExtend(StoreSizeSCEV, IntPtr),
                         SCEV::FlagNUW);
 }
 
+/// Calculate memory location used in store/load instructions in cases where
+/// this is possible.
+std::optional<MemoryLocation>
+LoopIdiomRecognize::calculateStoreLoadMemoryLocation(Value *InstrPtr,
+                                                     const SCEV *BECount,
+                                                     SCEVExpander &Expander) {
+  const SCEVAddRecExpr *StoreEv =
+      dyn_cast<SCEVAddRecExpr>(SE->getSCEV(InstrPtr));
+  // See if the pointer expression is an AddRec like {base,+,1} on the current
+  // loop, which indicates a strided istruction and more precise memory location
+  // can be calculated.
+  if (StoreEv && StoreEv->getLoop() == CurLoop && StoreEv->isAffine() &&
+      isa<SCEVConstant>(StoreEv->getOperand(1))) {
+    APInt Stride = getInstrStride(StoreEv);
+    uint64_t ConstSize = abs(Stride.getSExtValue());
+
+    // If the loop iterates a fixed number of times, we can define the used
+    // access size
+    const SCEVConstant *BECst = dyn_cast<SCEVConstant>(BECount);
+    if (BECst && ConstSize) {
+      LocationSize AccessSize = LocationSize::precise(
+          (BECst->getValue()->getZExtValue() + 1) * ConstSize);
+
+      BasicBlock *Preheader = CurLoop->getLoopPreheader();
+      IRBuilder<> Builder(Preheader->getTerminator());
+      unsigned DestAS = InstrPtr->getType()->getPointerAddressSpace();
+      Type *DestInt8PtrTy = Builder.getInt8PtrTy(DestAS);
+      unsigned InstAS = InstrPtr->getType()->getPointerAddressSpace();
+      Type *IntIdxTy = Builder.getIntNTy(DL->getIndexSizeInBits(InstAS));
+      const SCEV *Start = StoreEv->getStart();
+      const SCEV *StoreSizeSCEV = SE->getConstant(IntIdxTy, ConstSize);
+
+      // Calculate the right start in case of negative stride.
+      if (Stride.isNegative())
+        Start =
+            getStartForNegStride(Start, BECount, IntIdxTy, StoreSizeSCEV, SE);
+      // Added instructions will be removed later on the next phases.
+      Value *BasePtr = Expander.expandCodeFor(Start, DestInt8PtrTy,
+                                              Preheader->getTerminator());
+      return MemoryLocation(BasePtr, AccessSize);
+    }
+  }
+  return std::nullopt;
+}
+
 /// processLoopStridedStore - We see a strided store of some value.  If we can
 /// transform this into a memset or memset_pattern in the loop preheader, do so.
 bool LoopIdiomRecognize::processLoopStridedStore(
     Value *DestPtr, const SCEV *StoreSizeSCEV, MaybeAlign StoreAlignment,
     Value *StoredVal, Instruction *TheStore,
     SmallPtrSetImpl<Instruction *> &Stores, const SCEVAddRecExpr *Ev,
     const SCEV *BECount, bool IsNegStride, bool IsLoopMemset) {
   Module *M = TheStore->getModule();
bool LoopIdiomRecognize::processLoopStridedStore(
   // may be structural differences e.g. in the order of use lists not accounted
   // for in just a textual dump of the IR. This is written as a variable, even
   // though statically all the places this dominates could be replaced with
   // 'true', with the hope that anyone trying to be clever / "more precise" with
   // the return value will read this comment, and leave them alone.
   Changed = true;
 
   if (mayLoopAccessLocation(BasePtr, ModRefInfo::ModRef, CurLoop, BECount,
-                            StoreSizeSCEV, *AA, Stores))
+                            StoreSizeSCEV, *AA, Stores, Expander))
     return Changed;
 
   if (avoidLIRForMultiBlockLoop(/*IsMemset=*/true, IsLoopMemset))
     return Changed;
 
   // Okay, everything looks good, insert the memset.
 
   const SCEV *NumBytesS =
bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
 
   SCEVExpanderCleaner ExpCleaner(Expander);
 
   bool Changed = false;
   const SCEV *StrStart = StoreEv->getStart();
   unsigned StrAS = DestPtr->getType()->getPointerAddressSpace();
   Type *IntIdxTy = Builder.getIntNTy(DL->getIndexSizeInBits(StrAS));
 
-  APInt Stride = getStoreStride(StoreEv);
+  APInt Stride = getInstrStride(StoreEv);
   const SCEVConstant *ConstStoreSize = dyn_cast<SCEVConstant>(StoreSizeSCEV);
 
   // TODO: Deal with non-constant size; Currently expect constant store size
   assert(ConstStoreSize && "store size is expected to be a constant");
 
   int64_t StoreSize = ConstStoreSize->getValue()->getZExtValue();
   bool IsNegStride = StoreSize == -Stride;
 
bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
   SmallPtrSet<Instruction *, 2> IgnoredInsts;
   IgnoredInsts.insert(TheStore);
 
   bool IsMemCpy = isa<MemCpyInst>(TheStore);
   const StringRef InstRemark = IsMemCpy ? "memcpy" : "load and store";
 
   bool LoopAccessStore =
       mayLoopAccessLocation(StoreBasePtr, ModRefInfo::ModRef, CurLoop, BECount,
-                            StoreSizeSCEV, *AA, IgnoredInsts);
+                            StoreSizeSCEV, *AA, IgnoredInsts, Expander);
   if (LoopAccessStore) {
     // For memmove case it's not enough to guarantee that loop doesn't access
     // TheStore and TheLoad. Additionally we need to make sure that TheStore is
     // the only user of TheLoad.
     if (!TheLoad->hasOneUse())
       return Changed;
     IgnoredInsts.insert(TheLoad);
     if (mayLoopAccessLocation(StoreBasePtr, ModRefInfo::ModRef, CurLoop,
-                              BECount, StoreSizeSCEV, *AA, IgnoredInsts)) {
+                              BECount, StoreSizeSCEV, *AA, IgnoredInsts,
+                              Expander)) {
       ORE.emit([&]() {
         return OptimizationRemarkMissed(DEBUG_TYPE, "LoopMayAccessStore",
                                         TheStore)
                << ore::NV("Inst", InstRemark) << " in "
                << ore::NV("Function", TheStore->getFunction())
                << " function will not be hoisted: "
                << ore::NV("Reason", "The loop may access store location");
       });
Value *LoadBasePtr = Expander.expandCodeFor(
       LdStart, Builder.getInt8PtrTy(LdAS), Preheader->getTerminator());
 
   // If the store is a memcpy instruction, we must check if it will write to
   // the load memory locations. So remove it from the ignored stores.
   MemmoveVerifier Verifier(*LoadBasePtr, *StoreBasePtr, *DL);
   if (IsMemCpy && !Verifier.IsSameObject)
     IgnoredInsts.erase(TheStore);
   if (mayLoopAccessLocation(LoadBasePtr, ModRefInfo::Mod, CurLoop, BECount,
-                            StoreSizeSCEV, *AA, IgnoredInsts)) {
+                            StoreSizeSCEV, *AA, IgnoredInsts, Expander)) {
     ORE.emit([&]() {
       return OptimizationRemarkMissed(DEBUG_TYPE, "LoopMayAccessLoad", TheLoad)
              << ore::NV("Inst", InstRemark) << " in "
              << ore::NV("Function", TheStore->getFunction())
              << " function will not be hoisted: "
              << ore::NV("Reason", "The loop may access load location");
     });
     return Changed;

llvm/test/Transforms/LoopIdiom/memset-location-alias.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=loop-idiom < %s -S | FileCheck %s
 
 %struct.connection_data = type { [400 x i32], [400 x i32], [400 x i32], [400 x i32], [400 x i32], [400 x i32], i32, i32 }
 
 define dso_local void @extra_store(i32 %str, i32 %target, ptr nocapture %conn) {
 ; CHECK-LABEL: @extra_store(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[QUEUE_START:%.*]] = getelementptr inbounds [[STRUCT_CONNECTION_DATA:%.*]], ptr [[CONN:%.*]], i64 0, i32 6
 ; CHECK-NEXT:    store i32 0, ptr [[QUEUE_START]], align 4
 ; CHECK-NEXT:    [[QUEUE_END:%.*]] = getelementptr inbounds [[STRUCT_CONNECTION_DATA]], ptr [[CONN]], i64 0, i32 7
 ; CHECK-NEXT:    store i32 0, ptr [[QUEUE_END]], align 4
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[CONN]], i64 1684
+; CHECK-NEXT:    [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[CONN]], i64 84
+; CHECK-NEXT:    [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[CONN]], i64 3284
+; CHECK-NEXT:    [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[CONN]], i64 4884
+; CHECK-NEXT:    [[UGLYGEP4:%.*]] = getelementptr i8, ptr [[CONN]], i64 6484
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP2]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP3]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    [[UGLYGEP5:%.*]] = getelementptr i8, ptr [[CONN]], i64 84
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP4]], i8 0, i64 1516, i1 false)
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[POS:%.*]] = phi i32 [ 21, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[POS]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [400 x i32], ptr [[CONN]], i64 0, i64 [[IDXPROM]]
 ; CHECK-NEXT:    store i32 100000, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 1
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 2
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 3
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 4
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX8]], align 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[POS]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[POS]], 399
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %queue_start = getelementptr inbounds %struct.connection_data, ptr %conn, i64 0, i32 6

 
 define dso_local void @extra_store_alias(i32 %str, i32 %target, ptr nocapture %conn) {
 ; CHECK-LABEL: @extra_store_alias(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[QUEUE_START:%.*]] = getelementptr inbounds [[STRUCT_CONNECTION_DATA:%.*]], ptr [[CONN:%.*]], i64 0, i32 6
 ; CHECK-NEXT:    store i32 0, ptr [[QUEUE_START]], align 4
 ; CHECK-NEXT:    [[QUEUE_END:%.*]] = getelementptr inbounds [[STRUCT_CONNECTION_DATA]], ptr [[CONN]], i64 0, i32 7
 ; CHECK-NEXT:    store i32 0, ptr [[QUEUE_END]], align 4
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[CONN]], i64 3284
+; CHECK-NEXT:    [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[CONN]], i64 204
+; CHECK-NEXT:    [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[CONN]], i64 1684
+; CHECK-NEXT:    [[UGLYGEP4:%.*]] = getelementptr i8, ptr [[CONN]], i64 4884
+; CHECK-NEXT:    [[UGLYGEP5:%.*]] = getelementptr i8, ptr [[CONN]], i64 6484
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP4]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP5]], i8 0, i64 1516, i1 false)
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[POS:%.*]] = phi i32 [ 21, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[POS]], 30
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[ADD]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [400 x i32], ptr [[CONN]], i64 0, i64 [[IDXPROM]]
 ; CHECK-NEXT:    store i32 100000, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[POS]] to i64
 ; CHECK-NEXT:    [[DELTAS21:%.*]] = getelementptr [400 x i32], ptr [[CONN]], i64 0, i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[DELTAS21]], i64 0, i32 1
 ; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[DELTAS21]], i64 0, i32 2
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[DELTAS21]], i64 0, i32 3
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[DELTAS21]], i64 0, i32 4
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX8]], align 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[POS]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[POS]], 399
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %queue_start = getelementptr inbounds %struct.connection_data, ptr %conn, i64 0, i32 6

 
 define dso_local void @extra_store_negative_stride(i32 %str, i32 %target, ptr nocapture %conn) {
 ; CHECK-LABEL: @extra_store_negative_stride(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[QUEUE_START:%.*]] = getelementptr inbounds [[STRUCT_CONNECTION_DATA:%.*]], ptr [[CONN:%.*]], i64 0, i32 6
 ; CHECK-NEXT:    store i32 0, ptr [[QUEUE_START]], align 4
 ; CHECK-NEXT:    [[QUEUE_END:%.*]] = getelementptr inbounds [[STRUCT_CONNECTION_DATA]], ptr [[CONN]], i64 0, i32 7
 ; CHECK-NEXT:    store i32 0, ptr [[QUEUE_END]], align 4
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[CONN]], i64 1684
+; CHECK-NEXT:    [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[CONN]], i64 84
+; CHECK-NEXT:    [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[CONN]], i64 3284
+; CHECK-NEXT:    [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[CONN]], i64 4884
+; CHECK-NEXT:    [[UGLYGEP4:%.*]] = getelementptr i8, ptr [[CONN]], i64 6484
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP2]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP3]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    [[UGLYGEP5:%.*]] = getelementptr i8, ptr [[CONN]], i64 84
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP4]], i8 0, i64 1516, i1 false)
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[POS:%.*]] = phi i32 [ 399, [[ENTRY:%.*]] ], [ [[DEC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[POS]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [400 x i32], ptr [[CONN]], i64 0, i64 [[IDXPROM]]
 ; CHECK-NEXT:    store i32 100000, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 1
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 2
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 3
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 4
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX8]], align 4
 ; CHECK-NEXT:    [[DEC]] = add nsw i32 [[POS]], -1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[POS]], 21
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %queue_start = getelementptr inbounds %struct.connection_data, ptr %conn, i64 0, i32 6

 
 define dso_local void @extra_load(i32 %str, i32 %target, ptr nocapture %conn) {
 ; CHECK-LABEL: @extra_load(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[QUEUE_START:%.*]] = getelementptr inbounds [[STRUCT_CONNECTION_DATA:%.*]], ptr [[CONN:%.*]], i64 0, i32 6
 ; CHECK-NEXT:    store i32 0, ptr [[QUEUE_START]], align 4
 ; CHECK-NEXT:    [[QUEUE_END:%.*]] = getelementptr inbounds [[STRUCT_CONNECTION_DATA]], ptr [[CONN]], i64 0, i32 7
 ; CHECK-NEXT:    store i32 0, ptr [[QUEUE_END]], align 4
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[CONN]], i64 1684
+; CHECK-NEXT:    [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[CONN]], i64 84
+; CHECK-NEXT:    [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[CONN]], i64 3284
+; CHECK-NEXT:    [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[CONN]], i64 4884
+; CHECK-NEXT:    [[UGLYGEP4:%.*]] = getelementptr i8, ptr [[CONN]], i64 6484
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP2]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP3]], i8 0, i64 1516, i1 false)
+; CHECK-NEXT:    [[UGLYGEP5:%.*]] = getelementptr i8, ptr [[CONN]], i64 84
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP4]], i8 0, i64 1516, i1 false)
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[POS:%.*]] = phi i32 [ 21, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[POS]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [400 x i32], ptr [[CONN]], i64 0, i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr [[QUEUE_END]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 1
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX3]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 2
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 3
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX7]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr [[STRUCT_CONNECTION_DATA]], ptr [[ARRAYIDX]], i64 0, i32 4
-; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX9]], align 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[POS]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[POS]], 399
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %queue_start = getelementptr inbounds %struct.connection_data, ptr %conn, i64 0, i32 6