[WebAssembly] Stackify code emitted by eliminateFrameIndex and SP writeback

[dschuff]

Summary

MRI::eliminateFrameIndex can emit several instructions to do address
calculations; these can usually be stackified. Because instructions with
FI operands can have subsequent operands which may be expression trees,
find the top of the leftmost tree and insert the code before it, to keep
the LIFO property.

Also use stackified registers when writing back the SP value to memory
in the epilog; it's unnecessary because SP will not be used after the
epilog, and it results in better code.

[dschuff]

I still need to add some lit tests for the new FI placement behavior, but I've ensured no torture regressions, and you can take a look at the approach.

To find the insertion point we could have used def/use information but a lot of it is unavailable when this pass runs, and this ended up much simpler.

Also the extra epilog stackification turns out to be important because s2wasm can reorder things (I'll post more on that tomorrow but see https://pastebin.mozilla.org/8863952 for example).

[sunfish]

I'll review this more closely when I get more time, but at a first readthrough it looks good.

lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp

64 ↗

(On Diff #50898)

Would it make sense to name this canStackifyOperand?

[dschuff]

• rename canStackify

llvm/trunk/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp

 bool WebAssemblyFrameLowering::needsSPWriteback(
     const MachineFunction &MF, const MachineFrameInfo &MFI) const {
   return MFI.getStackSize() > RedZoneSize || MFI.hasCalls() ||
          MF.getFunction()->hasFnAttribute(Attribute::NoRedZone);
 }
 
 static void writeSPToMemory(unsigned SrcReg, MachineFunction &MF,
                             MachineBasicBlock &MBB,
-                            MachineBasicBlock::iterator &InsertPt,
+                            MachineBasicBlock::iterator &InsertAddr,
+                            MachineBasicBlock::iterator &InsertStore,
                             DebugLoc DL) {
   auto *SPSymbol = MF.createExternalSymbolName("__stack_pointer");
   unsigned SPAddr =
       MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
 
-  BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::CONST_I32), SPAddr)
+  BuildMI(MBB, InsertAddr, DL, TII->get(WebAssembly::CONST_I32), SPAddr)
       .addExternalSymbol(SPSymbol);
   auto *MMO = new MachineMemOperand(MachinePointerInfo(),
                                     MachineMemOperand::MOStore, 4, 4);
-  BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::STORE_I32),
-          WebAssembly::SP32)
+  BuildMI(MBB, InsertStore, DL, TII->get(WebAssembly::STORE_I32),
+          SrcReg)
       .addImm(0)
       .addReg(SPAddr)
       .addImm(2)  // p2align
       .addReg(SrcReg)
       .addMemOperand(MMO);
   MF.getInfo<WebAssemblyFunctionInfo>()->stackifyVReg(SPAddr);
 }
 
 void WebAssemblyFrameLowering::eliminateCallFramePseudoInstr(
     MachineFunction &MF, MachineBasicBlock &MBB,
     MachineBasicBlock::iterator I) const {
   assert(!I->getOperand(0).getImm() && hasFP(MF) &&
          "Call frame pseudos should only be used for dynamic stack adjustment");
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
   if (I->getOpcode() == TII->getCallFrameDestroyOpcode() &&
       needsSPWriteback(MF, *MF.getFrameInfo())) {
     DebugLoc DL = I->getDebugLoc();
-    writeSPToMemory(WebAssembly::SP32, MF, MBB, I, DL);
+    writeSPToMemory(WebAssembly::SP32, MF, MBB, I, I, DL);
   }
   MBB.erase(I);
 }
 
 void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
                                             MachineBasicBlock &MBB) const {
   // TODO: Do ".setMIFlag(MachineInstr::FrameSetup)" on emitted instructions
   auto *MFI = MF.getFrameInfo();
if (hasFP(MF)) {
     // Unlike most conventional targets (where FP points to the saved FP),
     // FP points to the bottom of the fixed-size locals, so we can use positive
     // offsets in load/store instructions.
     BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::COPY_LOCAL_I32),
             WebAssembly::FP32)
         .addReg(WebAssembly::SP32);
   }
   if (StackSize && needsSPWriteback(MF, *MFI)) {
-    writeSPToMemory(WebAssembly::SP32, MF, MBB, InsertPt, DL);
+    writeSPToMemory(WebAssembly::SP32, MF, MBB, InsertPt, InsertPt, DL);
   }
 }
 
 void WebAssemblyFrameLowering::emitEpilogue(MachineFunction &MF,
                                             MachineBasicBlock &MBB) const {
   auto *MFI = MF.getFrameInfo();
   uint64_t StackSize = MFI->getStackSize();
   if (!needsSP(MF, *MFI) || !needsSPWriteback(MF, *MFI)) return;
   auto *WFI = MF.getInfo<WebAssemblyFunctionInfo>();
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
   auto &MRI = MF.getRegInfo();
   auto InsertPt = MBB.getFirstTerminator();
   DebugLoc DL;
 
   if (InsertPt != MBB.end()) {
     DL = InsertPt->getDebugLoc();
   }
 
   // Restore the stack pointer. If we had fixed-size locals, add the offset
   // subtracted in the prolog.
+  unsigned SPReg = 0;
+  MachineBasicBlock::iterator InsertAddr = InsertPt;
   if (StackSize) {
     unsigned OffsetReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+    InsertAddr =
         BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::CONST_I32), OffsetReg)
             .addImm(StackSize);
-    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::ADD_I32),
-            WebAssembly::SP32)
+    // In the epilog we don't need to write the result back to the SP32 physreg
+    // because it won't be used again. We can use a stackified register instead.
+    SPReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+    BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::ADD_I32), SPReg)
         .addReg(hasFP(MF) ? WebAssembly::FP32 : WebAssembly::SP32)
         .addReg(OffsetReg);
     WFI->stackifyVReg(OffsetReg);
+    WFI->stackifyVReg(SPReg);
+  } else {
+    SPReg = hasFP(MF) ? WebAssembly::FP32 : WebAssembly::SP32;
   }
 
-  writeSPToMemory(
-      (!StackSize && hasFP(MF)) ? WebAssembly::FP32 : WebAssembly::SP32, MF,
-      MBB, InsertPt, DL);
+  writeSPToMemory(SPReg, MF, MBB, InsertAddr, InsertPt, DL);
 }

llvm/trunk/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp

 WebAssemblyRegisterInfo::getReservedRegs(const MachineFunction & /*MF*/) const {
   BitVector Reserved(getNumRegs());
   for (auto Reg : {WebAssembly::SP32, WebAssembly::SP64, WebAssembly::FP32,
                    WebAssembly::FP64})
     Reserved.set(Reg);
   return Reserved;
 }
 
+static bool isStackifiedVReg(const WebAssemblyFunctionInfo *WFI,
+                             const MachineOperand& Op) {
+  if (Op.isReg()) {
+    unsigned Reg = Op.getReg();
+    return TargetRegisterInfo::isVirtualRegister(Reg) &&
+        WFI->isVRegStackified(Reg);
+  }
+  return false;
+}
+
+static bool canStackifyOperand(const MachineInstr& Inst) {
+  unsigned Op = Inst.getOpcode();
+  return Op != TargetOpcode::PHI &&
+      Op != TargetOpcode::INLINEASM &&
+      Op != TargetOpcode::DBG_VALUE;
+}
+
+// Determine if the FI sequence can be stackified, and if so, where the code can
+// be inserted. If stackification is possible, returns true and ajusts II to
+// point to the insertion point.
+bool findInsertPt(const WebAssemblyFunctionInfo *WFI, MachineBasicBlock &MBB,
+                  unsigned OperandNum, MachineBasicBlock::iterator &II) {
+  if (!canStackifyOperand(*II)) return false;
+
+  MachineBasicBlock::iterator InsertPt(II);
+  int StackCount = 0;
+  // Operands are popped in reverse order, so any operands after FIOperand
+  // impose a constraint
+  for (unsigned i = OperandNum; i < II->getNumOperands(); i++) {
+    if (isStackifiedVReg(WFI, II->getOperand(i))) ++StackCount;
+  }
+  // Walk backwards, tracking stack depth. When it reaches 0 we have reached the
+  // top of the subtree.
+  while (StackCount) {
+    if (InsertPt == MBB.begin()) return false;
+    --InsertPt;
+    for (const auto &def : InsertPt->defs())
+      if (isStackifiedVReg(WFI, def)) --StackCount;
+    for (const auto &use : InsertPt->explicit_uses())
+      if (isStackifiedVReg(WFI, use)) ++StackCount;
+  }
+  II = InsertPt;
+  return true;
+}
+
 void WebAssemblyRegisterInfo::eliminateFrameIndex(
     MachineBasicBlock::iterator II, int SPAdj, unsigned FIOperandNum,
     RegScavenger * /*RS*/) const {
   assert(SPAdj == 0);
   MachineInstr &MI = *II;
 
   MachineBasicBlock &MBB = *MI.getParent();
   MachineFunction &MF = *MBB.getParent();
if (static_cast<uint64_t>(Offset) > std::numeric_limits<uint32_t>::max()) {
       // If this happens the program is invalid, but better to error here than
       // generate broken code.
       report_fatal_error("Memory offset field overflow");
     }
     MI.getOperand(FIOperandNum - 1).setImm(Offset);
     MI.getOperand(FIOperandNum)
         .ChangeToRegister(WebAssembly::SP32, /*IsDef=*/false);
   } else {
-    // Otherwise create an i32.add SP, offset and make it the operand.
+    // Otherwise calculate the address
     auto &MRI = MF.getRegInfo();
     const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
 
     unsigned FIRegOperand = WebAssembly::SP32;
     if (FrameOffset) {
-      FIRegOperand = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
-      BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(WebAssembly::CONST_I32),
-              FIRegOperand)
+      // Create i32.add SP, offset and make it the operand. We want to stackify
+      // this sequence, but we need to preserve the LIFO expr stack ordering
+      // (i.e. we can't insert our code in between MI and any operands it
+      // pops before FIOperand).
+      auto *WFI = MF.getInfo<WebAssemblyFunctionInfo>();
+      bool CanStackifyFI = findInsertPt(WFI, MBB, FIOperandNum, II);
+
+      unsigned OffsetOp = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+      BuildMI(MBB, *II, II->getDebugLoc(), TII->get(WebAssembly::CONST_I32),
+              OffsetOp)
           .addImm(FrameOffset);
-      BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(WebAssembly::ADD_I32),
+      if (CanStackifyFI) {
+        WFI->stackifyVReg(OffsetOp);
+        FIRegOperand = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+        WFI->stackifyVReg(FIRegOperand);
+      } else {
+        FIRegOperand = OffsetOp;
+      }
+      BuildMI(MBB, *II, II->getDebugLoc(), TII->get(WebAssembly::ADD_I32),
               FIRegOperand)
           .addReg(WebAssembly::SP32)
-          .addReg(FIRegOperand);
+          .addReg(OffsetOp);
     }
     MI.getOperand(FIOperandNum).ChangeToRegister(FIRegOperand, /*IsDef=*/false);
   }
 }
 
 unsigned
 WebAssemblyRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   static const unsigned Regs[2][2] = {

llvm/trunk/test/CodeGen/WebAssembly/byval.ll

  ; CHECK-NEXT: i32.sub [[SP:.+]]=, $pop[[L2]], $pop[[L3]]
  ; Ensure SP is stored back before the call
  ; CHECK-NEXT: i32.const $push[[L4:.+]]=, __stack_pointer
  ; CHECK-NEXT: i32.store {{.*}}=, 0($pop[[L4]]), [[SP]]
  ; Copy the SmallStruct argument to the stack (SP+12, original SP-4)
  ; CHECK-NEXT: i32.load $push[[L4:.+]]=, 0($0)
  ; CHECK-NEXT: i32.store {{.*}}=, 12([[SP]]), $pop[[L4]]
  ; Pass a pointer to the stack slot to the function
- ; CHECK-NEXT: i32.const [[L5:.+]]=, 12
- ; CHECK-NEXT: i32.add [[ARG:.+]]=, [[SP]], [[L5]]
- ; CHECK-NEXT: call ext_byval_func@FUNCTION, [[L5]]
+ ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 12
+ ; CHECK-NEXT: i32.add $push[[ARG:.+]]=, [[SP]], $pop[[L5]]
+ ; CHECK-NEXT: call ext_byval_func@FUNCTION, $pop[[ARG]]
  call void @ext_byval_func(%SmallStruct* byval %ptr)
  ; Restore the stack
- ; CHECK-NEXT: i32.const $push[[L6:.+]]=, 16
- ; CHECK-NEXT: i32.add [[SP]]=, [[SP]], $pop[[L6]]
  ; CHECK-NEXT: i32.const $push[[L7:.+]]=, __stack_pointer
- ; CHECK-NEXT: i32.store {{.*}}=, 0($pop[[L7]]), [[SP]]
+ ; CHECK-NEXT: i32.const $push[[L6:.+]]=, 16
+ ; CHECK-NEXT: i32.add $push[[L8:.+]]=, [[SP]], $pop[[L6]]
+ ; CHECK-NEXT: i32.store {{.*}}=, 0($pop[[L7]]), $pop[[L8]]
  ; CHECK-NEXT: return
  ret void
 }
 
 ; CHECK-LABEL: byval_arg_align8
 define void @byval_arg_align8(%SmallStruct* %ptr) {
  ; CHECK: .param i32
  ; Don't check the entire SP sequence, just enough to get the alignment.
  ; CHECK: i32.const $push[[L1:.+]]=, 16
  ; CHECK-NEXT: i32.sub [[SP:.+]]=, {{.+}}, $pop[[L1]]
  ; Copy the SmallStruct argument to the stack (SP+8, original SP-8)
  ; CHECK: i32.load $push[[L4:.+]]=, 0($0):p2align=3
  ; CHECK-NEXT: i32.store {{.*}}=, 8([[SP]]):p2align=3, $pop[[L4]]
  ; Pass a pointer to the stack slot to the function
- ; CHECK-NEXT: i32.const [[L5:.+]]=, 8
- ; CHECK-NEXT: i32.add [[ARG:.+]]=, [[SP]], [[L5]]
- ; CHECK-NEXT: call ext_byval_func_align8@FUNCTION, [[L5]]
+ ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 8
+ ; CHECK-NEXT: i32.add $push[[ARG:.+]]=, [[SP]], $pop[[L5]]
+ ; CHECK-NEXT: call ext_byval_func_align8@FUNCTION, $pop[[ARG]]
  call void @ext_byval_func_align8(%SmallStruct* byval align 8 %ptr)
  ret void
 }
 
 ; CHECK-LABEL: byval_arg_double
 define void @byval_arg_double(%AlignedStruct* %ptr) {
  ; CHECK: .param i32
  ; Subtract 16 from SP (SP is 16-byte aligned)

llvm/trunk/test/CodeGen/WebAssembly/mem-intrinsics.ll

 ; CHECK-NEXT: return{{$}}
 define void @set_no(i8* %dst, i8 %src, i32 %len) {
   call void @llvm.memset.p0i8.i32(i8* %dst, i8 %src, i32 %len, i32 1, i1 false)
   ret void
 }
 
 
 ; CHECK-LABEL: frame_index:
-; CHECK: i32.call $discard=, memset@FUNCTION, $0, $pop1, $pop0{{$}}
-; CHECK: i32.call $discard=, memset@FUNCTION, $1, $pop3, $pop2{{$}}
+; CHECK: i32.call $discard=, memset@FUNCTION, $pop12, $pop1, $pop0{{$}}
+; CHECK: i32.call $discard=, memset@FUNCTION, $0, $pop3, $pop2{{$}}
 ; CHECK: return{{$}}
 define void @frame_index() {
 entry:
   %a = alloca [2048 x i8], align 16
   %b = alloca [2048 x i8], align 16
   %0 = getelementptr inbounds [2048 x i8], [2048 x i8]* %a, i32 0, i32 0
   %1 = getelementptr inbounds [2048 x i8], [2048 x i8]* %b, i32 0, i32 0
   call void @llvm.memset.p0i8.i32(i8* %0, i8 256, i32 1024, i32 16, i1 false)

llvm/trunk/test/CodeGen/WebAssembly/userstack.ll

  ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 16
  ; CHECK-NEXT: i32.sub [[SP:.+]]=, $pop[[L2]], $pop[[L3]]
  ; CHECK-NEXT: i32.const $push[[L4:.+]]=, __stack_pointer{{$}}
  ; CHECK-NEXT: i32.store $discard=, 0($pop[[L4]]), [[SP]]
  %retval = alloca i32
  ; CHECK: i32.const $push[[L0:.+]]=, 0
  ; CHECK: i32.store {{.*}}=, 12([[SP]]), $pop[[L0]]
  store i32 0, i32* %retval
- ; CHECK: i32.const $push[[L5:.+]]=, 16
- ; CHECK-NEXT: i32.add [[SP]]=, [[SP]], $pop[[L5]]
- ; CHECK-NEXT: i32.const $push[[L6:.+]]=, __stack_pointer
- ; CHECK-NEXT: i32.store $discard=, 0($pop[[L6]]), [[SP]]
+ ; CHECK: i32.const $push[[L6:.+]]=, __stack_pointer
+ ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 16
+ ; CHECK-NEXT: i32.add $push[[L7:.+]]=, [[SP]], $pop[[L5]]
+ ; CHECK-NEXT: i32.store $discard=, 0($pop[[L6]]), $pop[[L7]]
  ret void
 }
 
 ; CHECK-LABEL: alloca3264:
 ; CHECK: .local i32{{$}}
 define void @alloca3264() {
  ; CHECK: i32.const $push[[L1:.+]]=, __stack_pointer
  ; CHECK-NEXT: i32.load $push[[L2:.+]]=, 0($pop[[L1]])
  ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 16
  ; CHECK-NEXT: i32.sub [[SP:.+]]=, $pop[[L2]], $pop[[L3]]
  %r1 = alloca i32
  %r2 = alloca double
  ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 0
  ; CHECK-NEXT: i32.store {{.*}}=, 12([[SP]]), $pop[[L3]]
  store i32 0, i32* %r1
  ; CHECK-NEXT: i64.const $push[[L0:.+]]=, 0
  ; CHECK-NEXT: i64.store {{.*}}=, 0([[SP]]), $pop[[L0]]
  store double 0.0, double* %r2
  ; CHECK-NEXT: return
  ret void
 }
 
 ; CHECK-LABEL: allocarray:
-; CHECK: .local i32, i32{{$}}
+; CHECK: .local i32{{$}}
 define void @allocarray() {
  ; CHECK: i32.const $push[[L1:.+]]=, __stack_pointer
  ; CHECK-NEXT: i32.load $push[[L2:.+]]=, 0($pop[[L1]])
  ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 144{{$}}
  ; CHECK-NEXT: i32.sub [[SP:.+]]=, $pop[[L2]], $pop[[L3]]
  ; CHECK-NEXT: i32.const $push[[L4:.+]]=, __stack_pointer{{$}}
  ; CHECK-NEXT: i32.store $discard=, 0($pop[[L4]]), [[SP]]
  %r = alloca [33 x i32]
 
+ ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 12
+ ; CHECK-NEXT: i32.add $push[[L7:.+]]=, [[SP]], $pop[[L5]]
  ; CHECK-NEXT: i32.const $push[[L4:.+]]=, 12
- ; CHECK-NEXT: i32.const [[L5:.+]]=, 12
- ; CHECK-NEXT: i32.add [[L5]]=, [[SP]], [[L5]]
- ; CHECK-NEXT: i32.add $push[[L6:.+]]=, [[L5]], $pop[[L4]]
+ ; CHECK-NEXT: i32.add $push[[L6:.+]]=, $pop[[L7]], $pop[[L4]]
  ; CHECK-NEXT: i32.const $push[[L9:.+]]=, 1{{$}}
  ; CHECK-NEXT: i32.store $push[[L10:.+]]=, 12([[SP]]), $pop[[L9]]{{$}}
  ; CHECK-NEXT: i32.store $discard=, 0($pop3), $pop[[L10]]{{$}}
  %p = getelementptr [33 x i32], [33 x i32]* %r, i32 0, i32 0
  store i32 1, i32* %p
  %p2 = getelementptr [33 x i32], [33 x i32]* %r, i32 0, i32 3
  store i32 1, i32* %p2
 
- ; CHECK: i32.const $push[[L11:.+]]=, 144
- ; CHECK-NEXT: i32.add [[SP]]=, [[SP]], $pop[[L11]]
- ; CHECK-NEXT: i32.const $push[[L12:.+]]=, __stack_pointer
- ; CHECK-NEXT: i32.store $discard=, 0($pop[[L12]]), [[SP]]
+ ; CHECK: i32.const $push[[L12:.+]]=, __stack_pointer
+ ; CHECK-NEXT: i32.const $push[[L11:.+]]=, 144
+ ; CHECK-NEXT: i32.add $push[[L13:.+]]=, [[SP]], $pop[[L11]]
+ ; CHECK-NEXT: i32.store $discard=, 0($pop[[L12]]), $pop[[L13]]
  ret void
 }
 
 ; CHECK-LABEL: non_mem_use
 define void @non_mem_use(i8** %addr) {
  ; CHECK: i32.const $push[[L1:.+]]=, 48
  ; CHECK-NEXT: i32.sub [[SP:.+]]=, {{.+}}, $pop[[L1]]
  %buf = alloca [27 x i8], align 16
  %r = alloca i64
  %r2 = alloca i64
  ; %r is at SP+8
- ; CHECK: i32.const [[OFF:.+]]=, 8
- ; CHECK-NEXT: i32.add [[ARG1:.+]]=, [[SP]], [[OFF]]
- ; CHECK-NEXT: call ext_func@FUNCTION, [[ARG1]]
+ ; CHECK: i32.const $push[[OFF:.+]]=, 8
+ ; CHECK-NEXT: i32.add $push[[ARG1:.+]]=, [[SP]], $pop[[OFF]]
+ ; CHECK-NEXT: call ext_func@FUNCTION, $pop[[ARG1]]
  call void @ext_func(i64* %r)
  ; %r2 is at SP+0, no add needed
  ; CHECK-NEXT: call ext_func@FUNCTION, [[SP]]
  call void @ext_func(i64* %r2)
  ; Use as a value, but in a store
  ; %buf is at SP+16
- ; CHECK: i32.const [[OFF:.+]]=, 16
- ; CHECK-NEXT: i32.add [[VAL:.+]]=, [[SP]], [[OFF]]
- ; CHECK-NEXT: i32.store {{.*}}=, 0($0), [[VAL]]
+ ; CHECK: i32.const $push[[OFF:.+]]=, 16
+ ; CHECK-NEXT: i32.add $push[[VAL:.+]]=, [[SP]], $pop[[OFF]]
+ ; CHECK-NEXT: i32.store {{.*}}=, 0($0), $pop[[VAL]]
  %gep = getelementptr inbounds [27 x i8], [27 x i8]* %buf, i32 0, i32 0
  store i8* %gep, i8** %addr
  ret void
 }
 
 ; CHECK-LABEL: allocarray_inbounds:
 ; CHECK: .local i32{{$}}
 define void @allocarray_inbounds() {
  ; CHECK: i32.const $push[[L1:.+]]=, __stack_pointer
  ; CHECK-NEXT: i32.load $push[[L2:.+]]=, 0($pop[[L1]])
  ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 32{{$}}
  ; CHECK-NEXT: i32.sub [[SP:.+]]=, $pop[[L2]], $pop[[L3]]
  %r = alloca [5 x i32]
  ; CHECK: i32.const $push[[L3:.+]]=, 1
  ; CHECK: i32.store {{.*}}=, 12([[SP]]), $pop[[L3]]
  %p = getelementptr inbounds [5 x i32], [5 x i32]* %r, i32 0, i32 0
  store i32 1, i32* %p
  ; This store should have both the GEP and the FI folded into it.
  ; CHECK-NEXT: i32.store {{.*}}=, 24([[SP]]), $pop
  %p2 = getelementptr inbounds [5 x i32], [5 x i32]* %r, i32 0, i32 3
  store i32 1, i32* %p2
  call void @ext_func(i64* null);
- ; CHECK: i32.const $push[[L5:.+]]=, 32
- ; CHECK-NEXT: i32.add [[SP]]=, [[SP]], $pop[[L5]]
- ; CHECK-NEXT: i32.const $push[[L6:.+]]=, __stack_pointer
- ; CHECK-NEXT: i32.store $discard=, 0($pop[[L6]]), [[SP]]
+ ; CHECK: i32.const $push[[L6:.+]]=, __stack_pointer
+ ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 32
+ ; CHECK-NEXT: i32.add $push[[L7:.+]]=, [[SP]], $pop[[L5]]
+ ; CHECK-NEXT: i32.store $discard=, 0($pop[[L6]]), $pop[[L7]]
  ret void
 }
 
 ; CHECK-LABEL: dynamic_alloca:
 define void @dynamic_alloca(i32 %alloc) {
  ; CHECK: i32.const $push[[L1:.+]]=, __stack_pointer
  ; CHECK-NEXT: i32.load [[SP:.+]]=, 0($pop[[L1]])
  ; CHECK-NEXT: copy_local [[FP:.+]]=, [[SP]]
  ; Target independent codegen bumps the stack pointer.
  ; CHECK: i32.sub
  ; CHECK-NEXT: copy_local [[SP]]=,
  ; Check that SP is written back to memory after decrement
  ; CHECK-NEXT: i32.const $push[[L4:.+]]=, __stack_pointer{{$}}
  ; CHECK-NEXT: i32.store $discard=, 0($pop[[L4]]), [[SP]]
  %r = alloca i32, i32 %alloc
  ; Target-independent codegen also calculates the store addr
  ; CHECK: call ext_func_i32@FUNCTION
  call void @ext_func_i32(i32* %r)
  ; CHECK: i32.const $push[[L3:.+]]=, __stack_pointer
- ; CHECK-NEXT: i32.store [[SP]]=, 0($pop[[L3]]), [[FP]]
+ ; CHECK-NEXT: i32.store $discard=, 0($pop[[L3]]), [[FP]]
  ret void
 }
 
 ; CHECK-LABEL: dynamic_alloca_redzone:
 define void @dynamic_alloca_redzone(i32 %alloc) {
  ; CHECK: i32.const $push[[L1:.+]]=, __stack_pointer
  ; CHECK-NEXT: i32.load [[SP:.+]]=, 0($pop[[L1]])
  ; CHECK-NEXT: copy_local [[FP:.+]]=, [[SP]]

  ; CHECK: i32.sub
  ; CHECK: copy_local [[SP]]=,
  ; Writeback to memory.
  ; CHECK-NEXT: i32.const $push[[L4:.+]]=, __stack_pointer
  ; CHECK-NEXT: i32.store {{.*}}=, 0($pop[[L4]]), [[SP]]
  %r1 = alloca i32
  %r = alloca i32, i32 %alloc
  store i32 0, i32* %r
- ; CHECK: i32.const $push[[L5:.+]]=, 16
- ; CHECK-NEXT: i32.add [[SP]]=, [[FP]], $pop[[L5]]
- ; CHECK-NEXT: i32.const $push[[L6:.+]]=, __stack_pointer
- ; CHECK-NEXT: i32.store $discard=, 0($pop[[L6]]), [[SP]]
+ ; CHECK: i32.const $push[[L6:.+]]=, __stack_pointer
+ ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 16
+ ; CHECK-NEXT: i32.add $push[[L7:.+]]=, [[FP]], $pop[[L5]]
+ ; CHECK-NEXT: i32.store $discard=, 0($pop[[L6]]), $pop[[L7]]
  ret void
 }
 
 ; The use of the alloca in a phi causes a CopyToReg DAG node to be generated,
 ; which has to have special handling because CopyToReg can't have a FI operand
 ; CHECK-LABEL: copytoreg_fi:
 define void @copytoreg_fi(i1 %cond, i32* %b) {
 entry:
  ; CHECK: i32.const $push[[L1:.+]]=, 16
  ; CHECK-NEXT: i32.sub [[SP:.+]]=, {{.+}}, $pop[[L1]]
  %addr = alloca i32
- ; CHECK: i32.const [[OFF:.+]]=, 12
- ; CHECK-NEXT: i32.add [[ADDR:.+]]=, [[SP]], [[OFF]]
- ; CHECK-NEXT: copy_local [[COPY:.+]]=, [[ADDR]]
+ ; CHECK: i32.const $push[[OFF:.+]]=, 12
+ ; CHECK-NEXT: i32.add $push[[ADDR:.+]]=, [[SP]], $pop[[OFF]]
+ ; CHECK-NEXT: copy_local [[COPY:.+]]=, $pop[[ADDR]]
  br label %body
 body:
  %a = phi i32* [%addr, %entry], [%b, %body]
  store i32 1, i32* %a
  ; CHECK: i32.store {{.*}}, 0([[COPY]]),
  br i1 %cond, label %body, label %exit
 exit:
  ret void