[SelectionDAGBuilder] Set NoUnsignedWrap on ISD::ADD for inbounds gep

[sunfish]

Summary

In an inbounds getelementptr, when an index produces a constant non-negative offset to add to the base, the add can be assumed to not have unsigned overflow.

This relies on the assumption that addresses can't occupy more than half the address space, which isn't possible in C because it wouldn't be possible to represent the difference between the start of the object and one-past-the-end in a ptrdiff_t.

Setting the NoUnsignedWrap flag is theoretically useful in general, and specifically useful to the WebAssembly backend, since it permits stronger constant offset folding.

[dschuff]

WebAssembly tests LGTM

[majnemer]

LGTM

llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

if (DAG.getDataLayout().isBigEndian()) {
     unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits();
     unsigned EVTStoreBits = ExtVT.getStoreSizeInBits();
     ShAmt = LVTStoreBits - EVTStoreBits - ShAmt;
   }
 
   uint64_t PtrOff = ShAmt / 8;
   unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
   SDLoc DL(LN0);
+  // The original load itself didn't wrap, so an offset within it doesn't.
+  SDNodeFlags Flags;
+  Flags.setNoUnsignedWrap(true);
   SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
                                PtrType, LN0->getBasePtr(),
-                               DAG.getConstant(PtrOff, DL, PtrType));
+                               DAG.getConstant(PtrOff, DL, PtrType),
+                               &Flags);
   AddToWorklist(NewPtr.getNode());
 
   SDValue Load;
   if (ExtType == ISD::NON_EXTLOAD)
     Load =  DAG.getLoad(VT, SDLoc(N0), LN0->getChain(), NewPtr,
                         LN0->getPointerInfo().getWithOffset(PtrOff),
                         LN0->isVolatile(), LN0->isNonTemporal(),
                         LN0->isInvariant(), NewAlign, LN0->getAAInfo());

llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
                                         DemoteReg, PtrValueVTs[0]);
     SDValue RetOp = getValue(I.getOperand(0));
 
     SmallVector<EVT, 4> ValueVTs;
     SmallVector<uint64_t, 4> Offsets;
     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
     unsigned NumValues = ValueVTs.size();
 
+    // An aggregate return value cannot wrap around the address space, so
+    // offsets to its parts don't wrap either.
+    SDNodeFlags Flags;
+    Flags.setNoUnsignedWrap(true);
+
     SmallVector<SDValue, 4> Chains(NumValues);
     for (unsigned i = 0; i != NumValues; ++i) {
       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
                                 RetPtr.getValueType(), RetPtr,
                                 DAG.getIntPtrConstant(Offsets[i],
-                                                      getCurSDLoc()));
+                                                      getCurSDLoc()),
+                                &Flags);
       Chains[i] =
         DAG.getStore(Chain, getCurSDLoc(),
                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
                      // FIXME: better loc info would be nice.
                      Add, MachinePointerInfo(), false, false, 0);
     }
 
     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
        OI != E; ++OI) {
     const Value *Idx = *OI;
     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
       if (Field) {
         // N = N + Offset
         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
+
+        // In an inbouds GEP with an offset that is nonnegative even when
+        // interpreted as signed, assume there is no unsigned overflow.
+        SDNodeFlags Flags;
+        if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
+          Flags.setNoUnsignedWrap(true);
+
         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
-                        DAG.getConstant(Offset, dl, N.getValueType()));
+                        DAG.getConstant(Offset, dl, N.getValueType()), &Flags);
       }
 
       Ty = StTy->getElementType(Field);
     } else {
       Ty = cast<SequentialType>(Ty)->getElementType();
       MVT PtrTy =
           DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout(), AS);
       unsigned PtrSize = PtrTy.getSizeInBits();
       APInt ElementSize(PtrSize, DL->getTypeAllocSize(Ty));
 
       // If this is a scalar constant or a splat vector of constants,
       // handle it quickly.
       const auto *CI = dyn_cast<ConstantInt>(Idx);
       if (!CI && isa<ConstantDataVector>(Idx) &&
           cast<ConstantDataVector>(Idx)->getSplatValue())
         CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
 
       if (CI) {
         if (CI->isZero())
           continue;
         APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
         SDValue OffsVal = VectorWidth ?
           DAG.getConstant(Offs, dl, MVT::getVectorVT(PtrTy, VectorWidth)) :
           DAG.getConstant(Offs, dl, PtrTy);
-        N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal);
+
+        // In an inbouds GEP with an offset that is nonnegative even when
+        // interpreted as signed, assume there is no unsigned overflow.
+        SDNodeFlags Flags;
+        if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
+          Flags.setNoUnsignedWrap(true);
+
+        N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, &Flags);
         continue;
       }
 
       // N = N + Idx * ElementSize;
       SDValue IdxN = getValue(Idx);
 
       if (!IdxN.getValueType().isVector() && VectorWidth) {
         MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth);
void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
   // the stack alignment, ignore it.  If the size is greater than or equal to
   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
   unsigned StackAlign =
       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
   if (Align <= StackAlign)
     Align = 0;
 
   // Round the size of the allocation up to the stack alignment size
-  // by add SA-1 to the size.
+  // by add SA-1 to the size. This doesn't overflow because we're computing
+  // an address inside an alloca.
+  SDNodeFlags Flags;
+  Flags.setNoUnsignedWrap(true);
   AllocSize = DAG.getNode(ISD::ADD, dl,
                           AllocSize.getValueType(), AllocSize,
-                          DAG.getIntPtrConstant(StackAlign - 1, dl));
+                          DAG.getIntPtrConstant(StackAlign - 1, dl), &Flags);
 
   // Mask out the low bits for alignment purposes.
   AllocSize = DAG.getNode(ISD::AND, dl,
                           AllocSize.getValueType(), AllocSize,
                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
                                                 dl));
 
   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
else if (AA->pointsToConstantMemory(MemoryLocation(
     Root = DAG.getRoot();
   }
 
   SDLoc dl = getCurSDLoc();
 
   if (isVolatile)
     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
 
+  // An aggregate load cannot wrap around the address space, so offsets to its
+  // parts don't wrap either.
+  SDNodeFlags Flags;
+  Flags.setNoUnsignedWrap(true);
+
   SmallVector<SDValue, 4> Values(NumValues);
   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
   EVT PtrVT = Ptr.getValueType();
   unsigned ChainI = 0;
   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
     // Serializing loads here may result in excessive register pressure, and
     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
     // could recover a bit by hoisting nodes upward in the chain by recognizing
     // they are side-effect free or do not alias. The optimizer should really
     // avoid this case by converting large object/array copies to llvm.memcpy
     // (MaxParallelChains should always remain as failsafe).
     if (ChainI == MaxParallelChains) {
       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                   makeArrayRef(Chains.data(), ChainI));
       Root = Chain;
       ChainI = 0;
     }
     SDValue A = DAG.getNode(ISD::ADD, dl,
                             PtrVT, Ptr,
-                            DAG.getConstant(Offsets[i], dl, PtrVT));
+                            DAG.getConstant(Offsets[i], dl, PtrVT),
+                            &Flags);
     SDValue L = DAG.getLoad(ValueVTs[i], dl, Root,
                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
                             isNonTemporal, isInvariant, Alignment, AAInfo,
                             Ranges);
 
     Values[i] = L;
     Chains[ChainI] = L.getValue(1);
   }
void SelectionDAGBuilder::visitStore(const StoreInst &I) {
   bool isVolatile = I.isVolatile();
   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
   unsigned Alignment = I.getAlignment();
   SDLoc dl = getCurSDLoc();
 
   AAMDNodes AAInfo;
   I.getAAMetadata(AAInfo);
 
+  // An aggregate load cannot wrap around the address space, so offsets to its
+  // parts don't wrap either.
+  SDNodeFlags Flags;
+  Flags.setNoUnsignedWrap(true);
+
   unsigned ChainI = 0;
   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
     // See visitLoad comments.
     if (ChainI == MaxParallelChains) {
       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                   makeArrayRef(Chains.data(), ChainI));
       Root = Chain;
       ChainI = 0;
     }
     SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
-                              DAG.getConstant(Offsets[i], dl, PtrVT));
+                              DAG.getConstant(Offsets[i], dl, PtrVT), &Flags);
     SDValue St = DAG.getStore(Root, dl,
                               SDValue(Src.getNode(), Src.getResNo() + i),
                               Add, MachinePointerInfo(PtrV, Offsets[i]),
                               isVolatile, isNonTemporal, Alignment, AAInfo);
     Chains[ChainI] = St;
   }
 
   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
if (!CanLowerReturn) {
     ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
     assert(PVTs.size() == 1 && "Pointers should fit in one register");
     EVT PtrVT = PVTs[0];
 
     unsigned NumValues = RetTys.size();
     ReturnValues.resize(NumValues);
     SmallVector<SDValue, 4> Chains(NumValues);
 
+    // An aggregate return value cannot wrap around the address space, so
+    // offsets to its parts don't wrap either.
+    SDNodeFlags Flags;
+    Flags.setNoUnsignedWrap(true);
+
     for (unsigned i = 0; i < NumValues; ++i) {
       SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
                                     CLI.DAG.getConstant(Offsets[i], CLI.DL,
-                                                        PtrVT));
+                                                        PtrVT), &Flags);
       SDValue L = CLI.DAG.getLoad(
           RetTys[i], CLI.DL, CLI.Chain, Add,
           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
                                             DemoteStackIdx, Offsets[i]),
           false, false, false, 1);
       ReturnValues[i] = L;
       Chains[i] = L.getValue(1);
     }

llvm/trunk/test/CodeGen/WebAssembly/offset.ll

 define i32 @load_i32_with_folded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   %t = load i32, i32* %s
   ret i32 %t
 }
 
+; With an inbounds gep, we can fold an offset.
+
+; CHECK-LABEL: load_i32_with_folded_gep_offset:
+; CHECK: i32.load  $push0=, 24($0){{$}}
+define i32 @load_i32_with_folded_gep_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 6
+  %t = load i32, i32* %s
+  ret i32 %t
+}
+
+; We can't fold a negative offset though, even with an inbounds gep.
+
+; CHECK-LABEL: load_i32_with_unfolded_gep_negative_offset:
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add   $push1=, $0, $pop0{{$}}
+; CHECK: i32.load  $push2=, 0($pop1){{$}}
+define i32 @load_i32_with_unfolded_gep_negative_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 -6
+  %t = load i32, i32* %s
+  ret i32 %t
+}
+
 ; Without nuw, and even with nsw, we can't fold an offset.
 
 ; CHECK-LABEL: load_i32_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i32.load  $push2=, 0($pop1){{$}}
 define i32 @load_i32_with_unfolded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   %t = load i32, i32* %s
   ret i32 %t
 }
 
+; Without inbounds, we can't fold a gep offset.
+
+; CHECK-LABEL: load_i32_with_unfolded_gep_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add   $push1=, $0, $pop0{{$}}
+; CHECK: i32.load  $push2=, 0($pop1){{$}}
+define i32 @load_i32_with_unfolded_gep_offset(i32* %p) {
+  %s = getelementptr i32, i32* %p, i32 6
+  %t = load i32, i32* %s
+  ret i32 %t
+}
+
 ; Same as above but with i64.
 
 ; CHECK-LABEL: load_i64_with_folded_offset:
 ; CHECK: i64.load  $push0=, 24($0){{$}}
 define i64 @load_i64_with_folded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   %t = load i64, i64* %s
   ret i64 %t
 }
 
 ; Same as above but with i64.
 
+; CHECK-LABEL: load_i64_with_folded_gep_offset:
+; CHECK: i64.load  $push0=, 24($0){{$}}
+define i64 @load_i64_with_folded_gep_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 3
+  %t = load i64, i64* %s
+  ret i64 %t
+}
+
+; Same as above but with i64.
+
+; CHECK-LABEL: load_i64_with_unfolded_gep_negative_offset:
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add   $push1=, $0, $pop0{{$}}
+; CHECK: i64.load  $push2=, 0($pop1){{$}}
+define i64 @load_i64_with_unfolded_gep_negative_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 -3
+  %t = load i64, i64* %s
+  ret i64 %t
+}
+
+; Same as above but with i64.
+
 ; CHECK-LABEL: load_i64_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i64.load  $push2=, 0($pop1){{$}}
 define i64 @load_i64_with_unfolded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   %t = load i64, i64* %s
   ret i64 %t
 }
 
+; Same as above but with i64.
+
+; CHECK-LABEL: load_i64_with_unfolded_gep_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add   $push1=, $0, $pop0{{$}}
+; CHECK: i64.load  $push2=, 0($pop1){{$}}
+define i64 @load_i64_with_unfolded_gep_offset(i64* %p) {
+  %s = getelementptr i64, i64* %p, i32 3
+  %t = load i64, i64* %s
+  ret i64 %t
+}
+
 ; Same as above but with store.
 
 ; CHECK-LABEL: store_i32_with_folded_offset:
 ; CHECK: i32.store $discard=, 24($0), $pop0{{$}}
 define void @store_i32_with_folded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   store i32 0, i32* %s
   ret void
 }
 
 ; Same as above but with store.
 
+; CHECK-LABEL: store_i32_with_folded_gep_offset:
+; CHECK: i32.store $discard=, 24($0), $pop0{{$}}
+define void @store_i32_with_folded_gep_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 6
+  store i32 0, i32* %s
+  ret void
+}
+
+; Same as above but with store.
+
+; CHECK-LABEL: store_i32_with_unfolded_gep_negative_offset:
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add   $push1=, $0, $pop0{{$}}
+; CHECK: i32.store $discard=, 0($pop1), $pop2{{$}}
+define void @store_i32_with_unfolded_gep_negative_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 -6
+  store i32 0, i32* %s
+  ret void
+}
+
+; Same as above but with store.
+
 ; CHECK-LABEL: store_i32_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i32.store $discard=, 0($pop1), $pop2{{$}}
 define void @store_i32_with_unfolded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i32*
   store i32 0, i32* %s
   ret void
 }
 
+; Same as above but with store.
+
+; CHECK-LABEL: store_i32_with_unfolded_gep_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add   $push1=, $0, $pop0{{$}}
+; CHECK: i32.store $discard=, 0($pop1), $pop2{{$}}
+define void @store_i32_with_unfolded_gep_offset(i32* %p) {
+  %s = getelementptr i32, i32* %p, i32 6
+  store i32 0, i32* %s
+  ret void
+}
+
 ; Same as above but with store with i64.
 
 ; CHECK-LABEL: store_i64_with_folded_offset:
 ; CHECK: i64.store $discard=, 24($0), $pop0{{$}}
 define void @store_i64_with_folded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   store i64 0, i64* %s
   ret void
 }
 
 ; Same as above but with store with i64.
 
+; CHECK-LABEL: store_i64_with_folded_gep_offset:
+; CHECK: i64.store $discard=, 24($0), $pop0{{$}}
+define void @store_i64_with_folded_gep_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 3
+  store i64 0, i64* %s
+  ret void
+}
+
+; Same as above but with store with i64.
+
+; CHECK-LABEL: store_i64_with_unfolded_gep_negative_offset:
+; CHECK: i32.const $push0=, -24{{$}}
+; CHECK: i32.add   $push1=, $0, $pop0{{$}}
+; CHECK: i64.store $discard=, 0($pop1), $pop2{{$}}
+define void @store_i64_with_unfolded_gep_negative_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 -3
+  store i64 0, i64* %s
+  ret void
+}
+
+; Same as above but with store with i64.
+
 ; CHECK-LABEL: store_i64_with_unfolded_offset:
 ; CHECK: i32.const $push0=, 24{{$}}
 ; CHECK: i32.add   $push1=, $0, $pop0{{$}}
 ; CHECK: i64.store $discard=, 0($pop1), $pop2{{$}}
 define void @store_i64_with_unfolded_offset(i64* %p) {
   %q = ptrtoint i64* %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to i64*
   store i64 0, i64* %s
   ret void
 }
 
+; Same as above but with store with i64.
+
+; CHECK-LABEL: store_i64_with_unfolded_gep_offset:
+; CHECK: i32.const $push0=, 24{{$}}
+; CHECK: i32.add   $push1=, $0, $pop0{{$}}
+; CHECK: i64.store $discard=, 0($pop1), $pop2{{$}}
+define void @store_i64_with_unfolded_gep_offset(i64* %p) {
+  %s = getelementptr i64, i64* %p, i32 3
+  store i64 0, i64* %s
+  ret void
+}
+
 ; When loading from a fixed address, materialize a zero.
 
 ; CHECK-LABEL: load_i32_from_numeric_address
 ; CHECK: i32.const $push0=, 0{{$}}
 ; CHECK: i32.load  $push1=, 42($pop0){{$}}
 define i32 @load_i32_from_numeric_address() {
   %s = inttoptr i32 42 to i32*
   %t = load i32, i32* %s
define i32 @load_i8_s_with_folded_offset(i8* %p) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
   %t = load i8, i8* %s
   %u = sext i8 %t to i32
   ret i32 %u
 }
 
+; Fold a gep offset into a sign-extending load.
+
+; CHECK-LABEL: load_i8_s_with_folded_gep_offset:
+; CHECK: i32.load8_s $push0=, 24($0){{$}}
+define i32 @load_i8_s_with_folded_gep_offset(i8* %p) {
+  %s = getelementptr inbounds i8, i8* %p, i32 24
+  %t = load i8, i8* %s
+  %u = sext i8 %t to i32
+  ret i32 %u
+}
+
 ; Fold an offset into a zero-extending load.
 
 ; CHECK-LABEL: load_i8_u_with_folded_offset:
 ; CHECK: i32.load8_u $push0=, 24($0){{$}}
 define i32 @load_i8_u_with_folded_offset(i8* %p) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
   %t = load i8, i8* %s
   %u = zext i8 %t to i32
   ret i32 %u
 }
 
+; Fold a gep offset into a zero-extending load.
+
+; CHECK-LABEL: load_i8_u_with_folded_gep_offset:
+; CHECK: i32.load8_u $push0=, 24($0){{$}}
+define i32 @load_i8_u_with_folded_gep_offset(i8* %p) {
+  %s = getelementptr inbounds i8, i8* %p, i32 24
+  %t = load i8, i8* %s
+  %u = zext i8 %t to i32
+  ret i32 %u
+}
+
 ; Fold an offset into a truncating store.
 
 ; CHECK-LABEL: store_i8_with_folded_offset:
 ; CHECK: i32.store8 $discard=, 24($0), $pop0{{$}}
 define void @store_i8_with_folded_offset(i8* %p) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to i8*
   store i8 0, i8* %s
   ret void
 }
+
+; Fold a gep offset into a truncating store.
+
+; CHECK-LABEL: store_i8_with_folded_gep_offset:
+; CHECK: i32.store8 $discard=, 24($0), $pop0{{$}}
+define void @store_i8_with_folded_gep_offset(i8* %p) {
+  %s = getelementptr inbounds i8, i8* %p, i32 24
+  store i8 0, i8* %s
+  ret void
+}
+
+; Fold the offsets when lowering aggregate loads and stores.
+
+; CHECK-LABEL: aggregate_load_store:
+; CHECK: i32.load  $2=, 0($0){{$}}
+; CHECK: i32.load  $3=, 4($0){{$}}
+; CHECK: i32.load  $4=, 8($0){{$}}
+; CHECK: i32.load  $push0=, 12($0){{$}}
+; CHECK: i32.store $discard=, 12($1), $pop0{{$}}
+; CHECK: i32.store $discard=, 8($1), $4{{$}}
+; CHECK: i32.store $discard=, 4($1), $3{{$}}
+; CHECK: i32.store $discard=, 0($1), $2{{$}}
+define void @aggregate_load_store({i32,i32,i32,i32}* %p, {i32,i32,i32,i32}* %q) {
+  ; volatile so that things stay in order for the tests above
+  %t = load volatile {i32,i32,i32,i32}, {i32, i32,i32,i32}* %p
+  store volatile {i32,i32,i32,i32} %t, {i32, i32,i32,i32}* %q
+  ret void
+}
+
+; Fold the offsets when lowering aggregate return values.
+
+; CHECK-LABEL: aggregate_return:
+; CHECK: i32.const   $push0=, 0{{$}}
+; CHECK: i32.store   $push1=, 12($0), $pop0{{$}}
+; CHECK: i32.store   $push2=, 8($0), $pop1{{$}}
+; CHECK: i32.store   $push3=, 4($0), $pop2{{$}}
+; CHECK: i32.store   $discard=, 0($0), $pop3{{$}}
+define {i32,i32,i32,i32} @aggregate_return() {
+  ret {i32,i32,i32,i32} zeroinitializer
+}