[FastISel] Reuse register for bitcast that does not change MVT

[nikic]

Summary

The current FastISel code reuses the register for a bitcast that doesn't change the IR type, but uses a reg-to-reg copy if it changes the MVT. However, we can simply reuse the register in that case as well.

In particular, this avoids unnecessary reg-to-reg copies for pointer bitcasts. This was found while inspecting O0 codegen differences between typed and opaque pointers.

[RKSimon]

LGTM

[ab]

Hi @nikic, I may have hit an issue with this change: when the noop-bitcast result is already used in other (potentially already selected) blocks, we already assigned a vreg for the value, and it may be too late to change it when selecting this bitcast. Concretely, this repro fails for x86_64 with -verify-machineinstrs, because the second use of %tmp1 uses a never-defined vreg:

define void @repro(i8** %a0, i1 %a1) {
  %tmp0 = load i8*, i8** %a0
  %tmp1 = bitcast i8* %tmp0 to void ()*
  call void %tmp1()
  br i1 %a1, label %bb1, label %bb2

bb1:
  call void %tmp1()
  ret void

bb2:
  ret void
}

My fast-isel is way too rusty to suggest an obvious fix, so can you please take a look? ;) Thanks!

[nikic]

In D119432#3507087, @ab wrote:

Hi @nikic, I may have hit an issue with this change: when the noop-bitcast result is already used in other (potentially already selected) blocks, we already assigned a vreg for the value, and it may be too late to change it when selecting this bitcast. Concretely, this repro fails for x86_64 with -verify-machineinstrs, because the second use of %tmp1 uses a never-defined vreg:

define void @repro(i8** %a0, i1 %a1) {
  %tmp0 = load i8*, i8** %a0
  %tmp1 = bitcast i8* %tmp0 to void ()*
  call void %tmp1()
  br i1 %a1, label %bb1, label %bb2

bb1:
  call void %tmp1()
  ret void

bb2:
  ret void
}

My fast-isel is way too rusty to suggest an obvious fix, so can you please take a look? ;) Thanks!

Thanks for the report! I've put up https://reviews.llvm.org/D125459 to fix this issue.

llvm/lib/CodeGen/SelectionDAG/FastISel.cpp

bool FastISel::selectCast(const User *I, unsigned Opcode) {
   if (!ResultReg)
     return false;
 
   updateValueMap(I, ResultReg);
   return true;
 }
 
 bool FastISel::selectBitCast(const User *I) {
-  // If the bitcast doesn't change the type, just use the operand value.
-  if (I->getType() == I->getOperand(0)->getType()) {
-    Register Reg = getRegForValue(I->getOperand(0));
-    if (!Reg)
-      return false;
-    updateValueMap(I, Reg);
-    return true;
-  }
-
-  // Bitcasts of other values become reg-reg copies or BITCAST operators.
   EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
   EVT DstEVT = TLI.getValueType(DL, I->getType());
   if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
       !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
     // Unhandled type. Halt "fast" selection and bail.
     return false;
 
   MVT SrcVT = SrcEVT.getSimpleVT();
   MVT DstVT = DstEVT.getSimpleVT();
   Register Op0 = getRegForValue(I->getOperand(0));
   if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
     return false;
 
-  // First, try to perform the bitcast by inserting a reg-reg copy.
-  Register ResultReg;
+  // If the bitcast doesn't change the type, just use the operand value.
   if (SrcVT == DstVT) {
-    ResultReg = createResultReg(TLI.getRegClassFor(DstVT));
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-            TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
+    updateValueMap(I, Op0);
+    return true;
   }
 
-  // If the reg-reg copy failed, select a BITCAST opcode.
-  if (!ResultReg)
-    ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0);
-
+  // Otherwise, select a BITCAST opcode.
+  Register ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0);
   if (!ResultReg)
     return false;
 
   updateValueMap(I, ResultReg);
   return true;
 }
 
 bool FastISel::selectFreeze(const User *I) {

llvm/test/CodeGen/AArch64/swifterror.ll

 ; CHECK-O0-AARCH64-NEXT:    stp x29, x30, [sp, #48] ; 16-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:    add x29, sp, #48
 ; CHECK-O0-AARCH64-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w30, -8
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w29, -16
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w21, -24
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w22, -32
 ; CHECK-O0-AARCH64-NEXT:    ; implicit-def: $x1
-; CHECK-O0-AARCH64-NEXT:    str x0, [sp] ; 8-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #8] ; 8-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:    mov x21, xzr
 ; CHECK-O0-AARCH64-NEXT:    bl _foo
-; CHECK-O0-AARCH64-NEXT:    str x21, [sp, #8] ; 8-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    mov x0, x21
-; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str x21, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:    cbnz x21, LBB1_2
 ; CHECK-O0-AARCH64-NEXT:  ; %bb.1: ; %cont
-; CHECK-O0-AARCH64-NEXT:    ldr x9, [sp] ; 8-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    ldr x8, [sp, #8] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x9, [sp, #8] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x8, [sp, #16] ; 8-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    ldrb w8, [x8, #8]
 ; CHECK-O0-AARCH64-NEXT:    strb w8, [x9]
 ; CHECK-O0-AARCH64-NEXT:  LBB1_2: ; %handler
 ; CHECK-O0-AARCH64-NEXT:    ldr x0, [sp, #16] ; 8-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    bl _free
 ; CHECK-O0-AARCH64-NEXT:    fmov s0, #1.00000000
 ; CHECK-O0-AARCH64-NEXT:    ldp x29, x30, [sp, #48] ; 16-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload

 ; CHECK-O0-ARM64_32-NEXT:    stp x22, x21, [sp, #32] ; 16-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    stp x29, x30, [sp, #48] ; 16-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w30, -8
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w29, -16
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w21, -24
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w22, -32
 ; CHECK-O0-ARM64_32-NEXT:    ; implicit-def: $x1
-; CHECK-O0-ARM64_32-NEXT:    str x0, [sp] ; 8-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #8] ; 8-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    mov x21, xzr
 ; CHECK-O0-ARM64_32-NEXT:    bl _foo
-; CHECK-O0-ARM64_32-NEXT:    str x21, [sp, #8] ; 8-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    mov x0, x21
-; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str x21, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    cmp x21, #0
 ; CHECK-O0-ARM64_32-NEXT:    b.ne LBB1_2
 ; CHECK-O0-ARM64_32-NEXT:  ; %bb.1: ; %cont
-; CHECK-O0-ARM64_32-NEXT:    ldr x9, [sp] ; 8-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    ldr x8, [sp, #8] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x9, [sp, #8] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x8, [sp, #16] ; 8-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    ldrb w8, [x8, #8]
 ; CHECK-O0-ARM64_32-NEXT:    ; kill: def $w0 killed $w8
 ; CHECK-O0-ARM64_32-NEXT:    strb w8, [x9]
 ; CHECK-O0-ARM64_32-NEXT:  LBB1_2: ; %handler
 ; CHECK-O0-ARM64_32-NEXT:    ldr x0, [sp, #16] ; 8-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    bl _free
 ; CHECK-O0-ARM64_32-NEXT:    fmov s0, #1.00000000
 ; CHECK-O0-ARM64_32-NEXT:    ldp x29, x30, [sp, #48] ; 16-byte Folded Reload

 ; CHECK-APPLE-AARCH64-NEXT:    ldp x20, x19, [sp, #48] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    ldp d9, d8, [sp, #16] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    add sp, sp, #80
 ; CHECK-APPLE-AARCH64-NEXT:    ret
 ;
 ; CHECK-O0-AARCH64-LABEL: caller2:
 ; CHECK-O0-AARCH64:       ; %bb.0: ; %entry
-; CHECK-O0-AARCH64-NEXT:    sub sp, sp, #80
-; CHECK-O0-AARCH64-NEXT:    stp x22, x21, [sp, #48] ; 16-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    add x29, sp, #64
+; CHECK-O0-AARCH64-NEXT:    sub sp, sp, #64
+; CHECK-O0-AARCH64-NEXT:    stp x22, x21, [sp, #32] ; 16-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    stp x29, x30, [sp, #48] ; 16-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    add x29, sp, #48
 ; CHECK-O0-AARCH64-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w30, -8
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w29, -16
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w21, -24
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w22, -32
 ; CHECK-O0-AARCH64-NEXT:    ; implicit-def: $x1
-; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #32] ; 8-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:  LBB2_1: ; %bb_loop
 ; CHECK-O0-AARCH64-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-O0-AARCH64-NEXT:    mov x21, xzr
 ; CHECK-O0-AARCH64-NEXT:    bl _foo
-; CHECK-O0-AARCH64-NEXT:    str s0, [sp, #12] ; 4-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    str x21, [sp, #16] ; 8-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    mov x0, x21
-; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #24] ; 8-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str s0, [sp, #4] ; 4-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str x21, [sp, #8] ; 8-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:    cbnz x21, LBB2_4
 ; CHECK-O0-AARCH64-NEXT:  ; %bb.2: ; %cont
 ; CHECK-O0-AARCH64-NEXT:    ; in Loop: Header=BB2_1 Depth=1
-; CHECK-O0-AARCH64-NEXT:    ldr s0, [sp, #12] ; 4-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr s0, [sp, #4] ; 4-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    fmov s1, #1.00000000
 ; CHECK-O0-AARCH64-NEXT:    fcmp s0, s1
 ; CHECK-O0-AARCH64-NEXT:    b.le LBB2_1
 ; CHECK-O0-AARCH64-NEXT:  ; %bb.3: ; %bb_end
-; CHECK-O0-AARCH64-NEXT:    ldr x9, [sp, #32] ; 8-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    ldr x8, [sp, #16] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x9, [sp, #16] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x8, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    ldrb w8, [x8, #8]
 ; CHECK-O0-AARCH64-NEXT:    strb w8, [x9]
 ; CHECK-O0-AARCH64-NEXT:  LBB2_4: ; %handler
-; CHECK-O0-AARCH64-NEXT:    ldr x0, [sp, #24] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x0, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    bl _free
 ; CHECK-O0-AARCH64-NEXT:    fmov s0, #1.00000000
-; CHECK-O0-AARCH64-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    ldp x22, x21, [sp, #48] ; 16-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    add sp, sp, #80
+; CHECK-O0-AARCH64-NEXT:    ldp x29, x30, [sp, #48] ; 16-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    add sp, sp, #64
 ; CHECK-O0-AARCH64-NEXT:    ret
 ;
 ; CHECK-APPLE-ARM64_32-LABEL: caller2:
 ; CHECK-APPLE-ARM64_32:       ; %bb.0: ; %entry
 ; CHECK-APPLE-ARM64_32-NEXT:    sub sp, sp, #80
 ; CHECK-APPLE-ARM64_32-NEXT:    stp d9, d8, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-ARM64_32-NEXT:    stp x22, x21, [sp, #32] ; 16-byte Folded Spill
 ; CHECK-APPLE-ARM64_32-NEXT:    stp x20, x19, [sp, #48] ; 16-byte Folded Spill

 ; CHECK-APPLE-ARM64_32-NEXT:    ldp x20, x19, [sp, #48] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    ldp d9, d8, [sp, #16] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    add sp, sp, #80
 ; CHECK-APPLE-ARM64_32-NEXT:    ret
 ;
 ; CHECK-O0-ARM64_32-LABEL: caller2:
 ; CHECK-O0-ARM64_32:       ; %bb.0: ; %entry
-; CHECK-O0-ARM64_32-NEXT:    sub sp, sp, #80
-; CHECK-O0-ARM64_32-NEXT:    stp x22, x21, [sp, #48] ; 16-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-O0-ARM64_32-NEXT:    sub sp, sp, #64
+; CHECK-O0-ARM64_32-NEXT:    stp x22, x21, [sp, #32] ; 16-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    stp x29, x30, [sp, #48] ; 16-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w30, -8
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w29, -16
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w21, -24
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w22, -32
 ; CHECK-O0-ARM64_32-NEXT:    ; implicit-def: $x1
-; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #32] ; 8-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:  LBB2_1: ; %bb_loop
 ; CHECK-O0-ARM64_32-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-O0-ARM64_32-NEXT:    mov x21, xzr
 ; CHECK-O0-ARM64_32-NEXT:    bl _foo
-; CHECK-O0-ARM64_32-NEXT:    str s0, [sp, #12] ; 4-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    str x21, [sp, #16] ; 8-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    mov x0, x21
-; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #24] ; 8-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str s0, [sp, #4] ; 4-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str x21, [sp, #8] ; 8-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    cmp x21, #0
 ; CHECK-O0-ARM64_32-NEXT:    b.ne LBB2_4
 ; CHECK-O0-ARM64_32-NEXT:  ; %bb.2: ; %cont
 ; CHECK-O0-ARM64_32-NEXT:    ; in Loop: Header=BB2_1 Depth=1
-; CHECK-O0-ARM64_32-NEXT:    ldr s0, [sp, #12] ; 4-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr s0, [sp, #4] ; 4-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    fmov s1, #1.00000000
 ; CHECK-O0-ARM64_32-NEXT:    fcmp s0, s1
 ; CHECK-O0-ARM64_32-NEXT:    b.le LBB2_1
 ; CHECK-O0-ARM64_32-NEXT:  ; %bb.3: ; %bb_end
-; CHECK-O0-ARM64_32-NEXT:    ldr x9, [sp, #32] ; 8-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    ldr x8, [sp, #16] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x9, [sp, #16] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x8, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    ldrb w8, [x8, #8]
 ; CHECK-O0-ARM64_32-NEXT:    ; kill: def $w0 killed $w8
 ; CHECK-O0-ARM64_32-NEXT:    strb w8, [x9]
 ; CHECK-O0-ARM64_32-NEXT:  LBB2_4: ; %handler
-; CHECK-O0-ARM64_32-NEXT:    ldr x0, [sp, #24] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x0, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    bl _free
 ; CHECK-O0-ARM64_32-NEXT:    fmov s0, #1.00000000
-; CHECK-O0-ARM64_32-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    ldp x22, x21, [sp, #48] ; 16-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    add sp, sp, #80
+; CHECK-O0-ARM64_32-NEXT:    ldp x29, x30, [sp, #48] ; 16-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    add sp, sp, #64
 ; CHECK-O0-ARM64_32-NEXT:    ret
 ; Access part of the error object and save it to error_ref
 
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   br label %bb_loop
 bb_loop:
   store %swift_error* null, %swift_error** %error_ptr_ref

 ; CHECK-APPLE-AARCH64-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    ldp x20, x19, [sp, #48] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    add sp, sp, #80
 ; CHECK-APPLE-AARCH64-NEXT:    ret
 ;
 ; CHECK-O0-AARCH64-LABEL: caller3:
 ; CHECK-O0-AARCH64:       ; %bb.0: ; %entry
-; CHECK-O0-AARCH64-NEXT:    sub sp, sp, #96
-; CHECK-O0-AARCH64-NEXT:    stp x22, x21, [sp, #64] ; 16-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    stp x29, x30, [sp, #80] ; 16-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    add x29, sp, #80
+; CHECK-O0-AARCH64-NEXT:    sub sp, sp, #80
+; CHECK-O0-AARCH64-NEXT:    stp x22, x21, [sp, #48] ; 16-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    add x29, sp, #64
 ; CHECK-O0-AARCH64-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w30, -8
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w29, -16
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w21, -24
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w22, -32
 ; CHECK-O0-AARCH64-NEXT:    ; implicit-def: $x1
-; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #8] ; 8-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str x0, [sp] ; 8-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:    mov x21, xzr
-; CHECK-O0-AARCH64-NEXT:    add x8, sp, #40
+; CHECK-O0-AARCH64-NEXT:    add x8, sp, #24
 ; CHECK-O0-AARCH64-NEXT:    mov w0, #1
 ; CHECK-O0-AARCH64-NEXT:    bl _foo_sret
-; CHECK-O0-AARCH64-NEXT:    str x21, [sp, #16] ; 8-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    mov x0, x21
-; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #24] ; 8-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str x21, [sp, #8] ; 8-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:    cbnz x21, LBB6_2
 ; CHECK-O0-AARCH64-NEXT:  ; %bb.1: ; %cont
-; CHECK-O0-AARCH64-NEXT:    ldr x9, [sp, #8] ; 8-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    ldr x8, [sp, #16] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x9, [sp] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x8, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    ldrb w8, [x8, #8]
 ; CHECK-O0-AARCH64-NEXT:    strb w8, [x9]
 ; CHECK-O0-AARCH64-NEXT:  LBB6_2: ; %handler
-; CHECK-O0-AARCH64-NEXT:    ldr x0, [sp, #24] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x0, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    bl _free
 ; CHECK-O0-AARCH64-NEXT:    fmov s0, #1.00000000
-; CHECK-O0-AARCH64-NEXT:    ldp x29, x30, [sp, #80] ; 16-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    ldp x22, x21, [sp, #64] ; 16-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    add sp, sp, #96
+; CHECK-O0-AARCH64-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldp x22, x21, [sp, #48] ; 16-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    add sp, sp, #80
 ; CHECK-O0-AARCH64-NEXT:    ret
 ;
 ; CHECK-APPLE-ARM64_32-LABEL: caller3:
 ; CHECK-APPLE-ARM64_32:       ; %bb.0: ; %entry
 ; CHECK-APPLE-ARM64_32-NEXT:    sub sp, sp, #80
 ; CHECK-APPLE-ARM64_32-NEXT:    stp x22, x21, [sp, #32] ; 16-byte Folded Spill
 ; CHECK-APPLE-ARM64_32-NEXT:    stp x20, x19, [sp, #48] ; 16-byte Folded Spill
 ; CHECK-APPLE-ARM64_32-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill

 ; CHECK-APPLE-ARM64_32-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    ldp x20, x19, [sp, #48] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    add sp, sp, #80
 ; CHECK-APPLE-ARM64_32-NEXT:    ret
 ;
 ; CHECK-O0-ARM64_32-LABEL: caller3:
 ; CHECK-O0-ARM64_32:       ; %bb.0: ; %entry
-; CHECK-O0-ARM64_32-NEXT:    sub sp, sp, #96
-; CHECK-O0-ARM64_32-NEXT:    stp x22, x21, [sp, #64] ; 16-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    stp x29, x30, [sp, #80] ; 16-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-O0-ARM64_32-NEXT:    sub sp, sp, #80
+; CHECK-O0-ARM64_32-NEXT:    stp x22, x21, [sp, #48] ; 16-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w30, -8
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w29, -16
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w21, -24
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w22, -32
 ; CHECK-O0-ARM64_32-NEXT:    ; implicit-def: $x1
-; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #8] ; 8-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str x0, [sp] ; 8-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    mov x21, xzr
-; CHECK-O0-ARM64_32-NEXT:    add x8, sp, #40
+; CHECK-O0-ARM64_32-NEXT:    add x8, sp, #24
 ; CHECK-O0-ARM64_32-NEXT:    mov w0, #1
 ; CHECK-O0-ARM64_32-NEXT:    bl _foo_sret
-; CHECK-O0-ARM64_32-NEXT:    str x21, [sp, #16] ; 8-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    mov x0, x21
-; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #24] ; 8-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str x21, [sp, #8] ; 8-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    cmp x21, #0
 ; CHECK-O0-ARM64_32-NEXT:    b.ne LBB6_2
 ; CHECK-O0-ARM64_32-NEXT:  ; %bb.1: ; %cont
-; CHECK-O0-ARM64_32-NEXT:    ldr x9, [sp, #8] ; 8-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    ldr x8, [sp, #16] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x9, [sp] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x8, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    ldrb w8, [x8, #8]
 ; CHECK-O0-ARM64_32-NEXT:    ; kill: def $w0 killed $w8
 ; CHECK-O0-ARM64_32-NEXT:    strb w8, [x9]
 ; CHECK-O0-ARM64_32-NEXT:  LBB6_2: ; %handler
-; CHECK-O0-ARM64_32-NEXT:    ldr x0, [sp, #24] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x0, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    bl _free
 ; CHECK-O0-ARM64_32-NEXT:    fmov s0, #1.00000000
-; CHECK-O0-ARM64_32-NEXT:    ldp x29, x30, [sp, #80] ; 16-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    ldp x22, x21, [sp, #64] ; 16-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    add sp, sp, #96
+; CHECK-O0-ARM64_32-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldp x22, x21, [sp, #48] ; 16-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    add sp, sp, #80
 ; CHECK-O0-ARM64_32-NEXT:    ret
 ; Access part of the error object and save it to error_ref
 
 ; spill x0
 ; Access part of the error object and save it to error_ref
 ; reload from stack
 entry:
   %s = alloca %struct.S, align 8

 ; CHECK-APPLE-AARCH64-NEXT:    ldp x29, x30, [sp, #80] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    ldp x20, x19, [sp, #64] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    ldp x22, x21, [sp, #48] ; 16-byte Folded Reload
 ; CHECK-APPLE-AARCH64-NEXT:    add sp, sp, #96
 ; CHECK-APPLE-AARCH64-NEXT:    ret
 ;
 ; CHECK-O0-AARCH64-LABEL: caller4:
 ; CHECK-O0-AARCH64:       ; %bb.0: ; %entry
-; CHECK-O0-AARCH64-NEXT:    sub sp, sp, #112
-; CHECK-O0-AARCH64-NEXT:    stp x22, x21, [sp, #80] ; 16-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    stp x29, x30, [sp, #96] ; 16-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    add x29, sp, #96
+; CHECK-O0-AARCH64-NEXT:    sub sp, sp, #96
+; CHECK-O0-AARCH64-NEXT:    stp x22, x21, [sp, #64] ; 16-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    stp x29, x30, [sp, #80] ; 16-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    add x29, sp, #80
 ; CHECK-O0-AARCH64-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w30, -8
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w29, -16
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w21, -24
 ; CHECK-O0-AARCH64-NEXT:    .cfi_offset w22, -32
 ; CHECK-O0-AARCH64-NEXT:    ; implicit-def: $x1
-; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #32] ; 8-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #24] ; 8-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:    mov x21, xzr
 ; CHECK-O0-AARCH64-NEXT:    mov w8, #10
 ; CHECK-O0-AARCH64-NEXT:    stur w8, [x29, #-28]
 ; CHECK-O0-AARCH64-NEXT:    mov w8, #11
 ; CHECK-O0-AARCH64-NEXT:    stur w8, [x29, #-32]
 ; CHECK-O0-AARCH64-NEXT:    mov w8, #12
 ; CHECK-O0-AARCH64-NEXT:    stur w8, [x29, #-36]
 ; CHECK-O0-AARCH64-NEXT:    ldur w8, [x29, #-28]
 ; CHECK-O0-AARCH64-NEXT:    ; kill: def $x8 killed $w8
 ; CHECK-O0-AARCH64-NEXT:    ldur w9, [x29, #-32]
 ; CHECK-O0-AARCH64-NEXT:    mov w10, w9
 ; CHECK-O0-AARCH64-NEXT:    ldur w9, [x29, #-36]
 ; CHECK-O0-AARCH64-NEXT:    mov w11, w9
 ; CHECK-O0-AARCH64-NEXT:    mov x9, sp
 ; CHECK-O0-AARCH64-NEXT:    str x11, [x9, #16]
 ; CHECK-O0-AARCH64-NEXT:    str x10, [x9, #8]
 ; CHECK-O0-AARCH64-NEXT:    str x8, [x9]
 ; CHECK-O0-AARCH64-NEXT:    bl _foo_vararg
-; CHECK-O0-AARCH64-NEXT:    str x21, [sp, #40] ; 8-byte Folded Spill
-; CHECK-O0-AARCH64-NEXT:    mov x0, x21
-; CHECK-O0-AARCH64-NEXT:    str x0, [sp, #48] ; 8-byte Folded Spill
+; CHECK-O0-AARCH64-NEXT:    str x21, [sp, #32] ; 8-byte Folded Spill
 ; CHECK-O0-AARCH64-NEXT:    cbnz x21, LBB8_2
 ; CHECK-O0-AARCH64-NEXT:  ; %bb.1: ; %cont
-; CHECK-O0-AARCH64-NEXT:    ldr x9, [sp, #32] ; 8-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    ldr x8, [sp, #40] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x9, [sp, #24] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x8, [sp, #32] ; 8-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    ldrb w8, [x8, #8]
 ; CHECK-O0-AARCH64-NEXT:    strb w8, [x9]
 ; CHECK-O0-AARCH64-NEXT:  LBB8_2: ; %handler
-; CHECK-O0-AARCH64-NEXT:    ldr x0, [sp, #48] ; 8-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldr x0, [sp, #32] ; 8-byte Folded Reload
 ; CHECK-O0-AARCH64-NEXT:    bl _free
 ; CHECK-O0-AARCH64-NEXT:    fmov s0, #1.00000000
-; CHECK-O0-AARCH64-NEXT:    ldp x29, x30, [sp, #96] ; 16-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    ldp x22, x21, [sp, #80] ; 16-byte Folded Reload
-; CHECK-O0-AARCH64-NEXT:    add sp, sp, #112
+; CHECK-O0-AARCH64-NEXT:    ldp x29, x30, [sp, #80] ; 16-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    ldp x22, x21, [sp, #64] ; 16-byte Folded Reload
+; CHECK-O0-AARCH64-NEXT:    add sp, sp, #96
 ; CHECK-O0-AARCH64-NEXT:    ret
 ;
 ; CHECK-APPLE-ARM64_32-LABEL: caller4:
 ; CHECK-APPLE-ARM64_32:       ; %bb.0: ; %entry
 ; CHECK-APPLE-ARM64_32-NEXT:    sub sp, sp, #80
 ; CHECK-APPLE-ARM64_32-NEXT:    stp x22, x21, [sp, #32] ; 16-byte Folded Spill
 ; CHECK-APPLE-ARM64_32-NEXT:    stp x20, x19, [sp, #48] ; 16-byte Folded Spill
 ; CHECK-APPLE-ARM64_32-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill

 ; CHECK-APPLE-ARM64_32-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    ldp x20, x19, [sp, #48] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload
 ; CHECK-APPLE-ARM64_32-NEXT:    add sp, sp, #80
 ; CHECK-APPLE-ARM64_32-NEXT:    ret
 ;
 ; CHECK-O0-ARM64_32-LABEL: caller4:
 ; CHECK-O0-ARM64_32:       ; %bb.0: ; %entry
-; CHECK-O0-ARM64_32-NEXT:    sub sp, sp, #96
-; CHECK-O0-ARM64_32-NEXT:    stp x22, x21, [sp, #64] ; 16-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    stp x29, x30, [sp, #80] ; 16-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-O0-ARM64_32-NEXT:    sub sp, sp, #80
+; CHECK-O0-ARM64_32-NEXT:    stp x22, x21, [sp, #48] ; 16-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    stp x29, x30, [sp, #64] ; 16-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w30, -8
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w29, -16
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w21, -24
 ; CHECK-O0-ARM64_32-NEXT:    .cfi_offset w22, -32
 ; CHECK-O0-ARM64_32-NEXT:    ; implicit-def: $x1
-; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #24] ; 8-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    mov x21, xzr
 ; CHECK-O0-ARM64_32-NEXT:    mov w8, #10
-; CHECK-O0-ARM64_32-NEXT:    str w8, [sp, #56]
+; CHECK-O0-ARM64_32-NEXT:    str w8, [sp, #40]
 ; CHECK-O0-ARM64_32-NEXT:    mov w8, #11
-; CHECK-O0-ARM64_32-NEXT:    str w8, [sp, #52]
+; CHECK-O0-ARM64_32-NEXT:    str w8, [sp, #36]
 ; CHECK-O0-ARM64_32-NEXT:    mov w8, #12
-; CHECK-O0-ARM64_32-NEXT:    str w8, [sp, #48]
-; CHECK-O0-ARM64_32-NEXT:    ldr w8, [sp, #56]
-; CHECK-O0-ARM64_32-NEXT:    ldr w10, [sp, #52]
-; CHECK-O0-ARM64_32-NEXT:    ldr w11, [sp, #48]
+; CHECK-O0-ARM64_32-NEXT:    str w8, [sp, #32]
+; CHECK-O0-ARM64_32-NEXT:    ldr w8, [sp, #40]
+; CHECK-O0-ARM64_32-NEXT:    ldr w10, [sp, #36]
+; CHECK-O0-ARM64_32-NEXT:    ldr w11, [sp, #32]
 ; CHECK-O0-ARM64_32-NEXT:    mov x9, sp
 ; CHECK-O0-ARM64_32-NEXT:    str w11, [x9, #8]
 ; CHECK-O0-ARM64_32-NEXT:    str w10, [x9, #4]
 ; CHECK-O0-ARM64_32-NEXT:    str w8, [x9]
 ; CHECK-O0-ARM64_32-NEXT:    bl _foo_vararg
-; CHECK-O0-ARM64_32-NEXT:    str x21, [sp, #32] ; 8-byte Folded Spill
-; CHECK-O0-ARM64_32-NEXT:    mov x0, x21
-; CHECK-O0-ARM64_32-NEXT:    str x0, [sp, #40] ; 8-byte Folded Spill
+; CHECK-O0-ARM64_32-NEXT:    str x21, [sp, #24] ; 8-byte Folded Spill
 ; CHECK-O0-ARM64_32-NEXT:    cmp x21, #0
 ; CHECK-O0-ARM64_32-NEXT:    b.ne LBB8_2
 ; CHECK-O0-ARM64_32-NEXT:  ; %bb.1: ; %cont
-; CHECK-O0-ARM64_32-NEXT:    ldr x9, [sp, #24] ; 8-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    ldr x8, [sp, #32] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x9, [sp, #16] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x8, [sp, #24] ; 8-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    ldrb w8, [x8, #8]
 ; CHECK-O0-ARM64_32-NEXT:    ; kill: def $w0 killed $w8
 ; CHECK-O0-ARM64_32-NEXT:    strb w8, [x9]
 ; CHECK-O0-ARM64_32-NEXT:  LBB8_2: ; %handler
-; CHECK-O0-ARM64_32-NEXT:    ldr x0, [sp, #40] ; 8-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldr x0, [sp, #24] ; 8-byte Folded Reload
 ; CHECK-O0-ARM64_32-NEXT:    bl _free
 ; CHECK-O0-ARM64_32-NEXT:    fmov s0, #1.00000000
-; CHECK-O0-ARM64_32-NEXT:    ldp x29, x30, [sp, #80] ; 16-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    ldp x22, x21, [sp, #64] ; 16-byte Folded Reload
-; CHECK-O0-ARM64_32-NEXT:    add sp, sp, #96
+; CHECK-O0-ARM64_32-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    ldp x22, x21, [sp, #48] ; 16-byte Folded Reload
+; CHECK-O0-ARM64_32-NEXT:    add sp, sp, #80
 ; CHECK-O0-ARM64_32-NEXT:    ret
 
 
 ; Access part of the error object and save it to error_ref
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   store %swift_error* null, %swift_error** %error_ptr_ref
 

llvm/test/CodeGen/ARM/swifterror.ll

 ; CHECK-APPLE-NEXT:    add sp, sp, #4
 ; CHECK-APPLE-NEXT:    pop {r4, r8, pc}
 ;
 ; CHECK-O0-LABEL: caller:
 ; CHECK-O0:       @ %bb.0: @ %entry
 ; CHECK-O0-NEXT:    push {r7, lr}
 ; CHECK-O0-NEXT:    mov r7, sp
 ; CHECK-O0-NEXT:    push {r8}
-; CHECK-O0-NEXT:    sub sp, sp, #16
+; CHECK-O0-NEXT:    sub sp, sp, #12
 ; CHECK-O0-NEXT:    @ implicit-def: $r1
 ; CHECK-O0-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-O0-NEXT:    mov r8, #0
 ; CHECK-O0-NEXT:    bl _foo
 ; CHECK-O0-NEXT:    str r8, [sp, #4] @ 4-byte Spill
-; CHECK-O0-NEXT:    mov r0, r8
-; CHECK-O0-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-O0-NEXT:    movw r0, #0
 ; CHECK-O0-NEXT:    cmp r8, r0
 ; CHECK-O0-NEXT:    bne LBB1_2
 ; CHECK-O0-NEXT:  @ %bb.1: @ %cont
 ; CHECK-O0-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; CHECK-O0-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-O0-NEXT:    ldrb r0, [r0, #8]
 ; CHECK-O0-NEXT:    strb r0, [r1]
 ; CHECK-O0-NEXT:  LBB1_2: @ %handler
-; CHECK-O0-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-O0-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-O0-NEXT:    bl _free
 ; CHECK-O0-NEXT:    mov r0, #1065353216
 ; CHECK-O0-NEXT:    sub sp, r7, #4
 ; CHECK-O0-NEXT:    pop {r8}
 ; CHECK-O0-NEXT:    pop {r7, pc}
 ;
 ; CHECK-ANDROID-LABEL: caller:
 ; CHECK-ANDROID:       @ %bb.0: @ %entry

 ; CHECK-APPLE-NEXT:    vpop {d8}
 ; CHECK-APPLE-NEXT:    pop {r4, r8, pc}
 ;
 ; CHECK-O0-LABEL: caller2:
 ; CHECK-O0:       @ %bb.0: @ %entry
 ; CHECK-O0-NEXT:    push {r7, lr}
 ; CHECK-O0-NEXT:    mov r7, sp
 ; CHECK-O0-NEXT:    push {r8}
-; CHECK-O0-NEXT:    sub sp, sp, #20
+; CHECK-O0-NEXT:    sub sp, sp, #16
 ; CHECK-O0-NEXT:    @ implicit-def: $r1
-; CHECK-O0-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-O0-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-O0-NEXT:  LBB2_1: @ %bb_loop
 ; CHECK-O0-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-O0-NEXT:    mov r8, #0
 ; CHECK-O0-NEXT:    bl _foo
 ; CHECK-O0-NEXT:    vmov s0, r0
 ; CHECK-O0-NEXT:    vstr s0, [sp] @ 4-byte Spill
 ; CHECK-O0-NEXT:    str r8, [sp, #4] @ 4-byte Spill
-; CHECK-O0-NEXT:    mov r0, r8
-; CHECK-O0-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-O0-NEXT:    movw r0, #0
 ; CHECK-O0-NEXT:    cmp r8, r0
 ; CHECK-O0-NEXT:    bne LBB2_4
 ; CHECK-O0-NEXT:  @ %bb.2: @ %cont
 ; CHECK-O0-NEXT:    @ in Loop: Header=BB2_1 Depth=1
 ; CHECK-O0-NEXT:    vldr s0, [sp] @ 4-byte Reload
 ; CHECK-O0-NEXT:    vmov.f32 s2, #1.000000e+00
 ; CHECK-O0-NEXT:    vcmp.f32 s0, s2
 ; CHECK-O0-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-O0-NEXT:    ble LBB2_1
 ; CHECK-O0-NEXT:  @ %bb.3: @ %bb_end
-; CHECK-O0-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-O0-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; CHECK-O0-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-O0-NEXT:    ldrb r0, [r0, #8]
 ; CHECK-O0-NEXT:    strb r0, [r1]
 ; CHECK-O0-NEXT:  LBB2_4: @ %handler
-; CHECK-O0-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-O0-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-O0-NEXT:    bl _free
 ; CHECK-O0-NEXT:    mov r0, #1065353216
 ; CHECK-O0-NEXT:    sub sp, r7, #4
 ; CHECK-O0-NEXT:    pop {r8}
 ; CHECK-O0-NEXT:    pop {r7, pc}
 ;
 ; CHECK-ANDROID-LABEL: caller2:
 ; CHECK-ANDROID:       @ %bb.0: @ %entry

 ; CHECK-O0-LABEL: caller3:
 ; CHECK-O0:       @ %bb.0: @ %entry
 ; CHECK-O0-NEXT:    push {r7, lr}
 ; CHECK-O0-NEXT:    mov r7, sp
 ; CHECK-O0-NEXT:    push {r8}
 ; CHECK-O0-NEXT:    sub sp, sp, #44
 ; CHECK-O0-NEXT:    bfc sp, #0, #3
 ; CHECK-O0-NEXT:    @ implicit-def: $r1
-; CHECK-O0-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-O0-NEXT:    str r0, [sp, #4] @ 4-byte Spill
 ; CHECK-O0-NEXT:    mov r8, #0
 ; CHECK-O0-NEXT:    add r0, sp, #16
 ; CHECK-O0-NEXT:    mov r1, #1
 ; CHECK-O0-NEXT:    bl _foo_sret
-; CHECK-O0-NEXT:    str r8, [sp, #4] @ 4-byte Spill
-; CHECK-O0-NEXT:    mov r0, r8
-; CHECK-O0-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; CHECK-O0-NEXT:    str r8, [sp, #8] @ 4-byte Spill
 ; CHECK-O0-NEXT:    movw r0, #0
 ; CHECK-O0-NEXT:    cmp r8, r0
 ; CHECK-O0-NEXT:    bne LBB6_2
 ; CHECK-O0-NEXT:  @ %bb.1: @ %cont
-; CHECK-O0-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; CHECK-O0-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-O0-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-O0-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; CHECK-O0-NEXT:    ldrb r0, [r0, #8]
 ; CHECK-O0-NEXT:    strb r0, [r1]
 ; CHECK-O0-NEXT:  LBB6_2: @ %handler
 ; CHECK-O0-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; CHECK-O0-NEXT:    bl _free
 ; CHECK-O0-NEXT:    mov r0, #1065353216
 ; CHECK-O0-NEXT:    sub sp, r7, #4
 ; CHECK-O0-NEXT:    pop {r8}

 ; CHECK-APPLE-NEXT:    add sp, sp, #16
 ; CHECK-APPLE-NEXT:    pop {r4, r8, pc}
 ;
 ; CHECK-O0-LABEL: caller4:
 ; CHECK-O0:       @ %bb.0: @ %entry
 ; CHECK-O0-NEXT:    push {r7, lr}
 ; CHECK-O0-NEXT:    mov r7, sp
 ; CHECK-O0-NEXT:    push {r8}
-; CHECK-O0-NEXT:    sub sp, sp, #28
+; CHECK-O0-NEXT:    sub sp, sp, #24
 ; CHECK-O0-NEXT:    @ implicit-def: $r1
 ; CHECK-O0-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-O0-NEXT:    mov r8, #0
 ; CHECK-O0-NEXT:    mov r0, #10
 ; CHECK-O0-NEXT:    str r0, [r7, #-12]
 ; CHECK-O0-NEXT:    mov r0, #11
-; CHECK-O0-NEXT:    str r0, [sp, #16]
-; CHECK-O0-NEXT:    mov r0, #12
 ; CHECK-O0-NEXT:    str r0, [sp, #12]
+; CHECK-O0-NEXT:    mov r0, #12
+; CHECK-O0-NEXT:    str r0, [sp, #8]
 ; CHECK-O0-NEXT:    ldr r0, [r7, #-12]
-; CHECK-O0-NEXT:    ldr r1, [sp, #16]
-; CHECK-O0-NEXT:    ldr r2, [sp, #12]
+; CHECK-O0-NEXT:    ldr r1, [sp, #12]
+; CHECK-O0-NEXT:    ldr r2, [sp, #8]
 ; CHECK-O0-NEXT:    bl _foo_vararg
 ; CHECK-O0-NEXT:    str r8, [sp, #4] @ 4-byte Spill
-; CHECK-O0-NEXT:    mov r0, r8
-; CHECK-O0-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-O0-NEXT:    movw r0, #0
 ; CHECK-O0-NEXT:    cmp r8, r0
 ; CHECK-O0-NEXT:    bne LBB8_2
 ; CHECK-O0-NEXT:  @ %bb.1: @ %cont
 ; CHECK-O0-NEXT:    ldr r1, [sp] @ 4-byte Reload
 ; CHECK-O0-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-O0-NEXT:    ldrb r0, [r0, #8]
 ; CHECK-O0-NEXT:    strb r0, [r1]
 ; CHECK-O0-NEXT:  LBB8_2: @ %handler
-; CHECK-O0-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-O0-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-O0-NEXT:    bl _free
 ; CHECK-O0-NEXT:    mov r0, #1065353216
 ; CHECK-O0-NEXT:    sub sp, r7, #4
 ; CHECK-O0-NEXT:    pop {r8}
 ; CHECK-O0-NEXT:    pop {r7, pc}
 ;
 ; CHECK-ANDROID-LABEL: caller4:
 ; CHECK-ANDROID:       @ %bb.0: @ %entry

llvm/test/CodeGen/X86/atomic64.ll

 ; X64-NEXT:    movl $5, %eax
 ; X64-NEXT:    lock xaddq %rax, sc64(%rip)
 ; X64-NEXT:    lock addq %rax, sc64(%rip)
 ; X64-NEXT:    retq
 ;
 ; I486-LABEL: atomic_fetch_add64:
 ; I486:       # %bb.0: # %entry
 ; I486-NEXT:    subl $16, %esp
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $1, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_add_8@PLT
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $3, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_add_8@PLT
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $5, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_add_8@PLT
 ; I486-NEXT:    movl %eax, %ecx
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_add_8@PLT
 ; I486-NEXT:    addl $16, %esp
 ; I486-NEXT:    retl

 ; X64-NEXT:    movq $-5, %rax
 ; X64-NEXT:    lock xaddq %rax, sc64(%rip)
 ; X64-NEXT:    lock subq %rax, sc64(%rip)
 ; X64-NEXT:    retq
 ;
 ; I486-LABEL: atomic_fetch_sub64:
 ; I486:       # %bb.0:
 ; I486-NEXT:    subl $16, %esp
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $1, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_sub_8@PLT
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $3, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_sub_8@PLT
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $5, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_sub_8@PLT
 ; I486-NEXT:    movl %eax, %ecx
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_sub_8@PLT
 ; I486-NEXT:    addl $16, %esp
 ; I486-NEXT:    retl

 ; X64-NEXT:  .LBB2_2: # %atomicrmw.end
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; X64-NEXT:    lock andq %rax, sc64(%rip)
 ; X64-NEXT:    retq
 ;
 ; I486-LABEL: atomic_fetch_and64:
 ; I486:       # %bb.0:
 ; I486-NEXT:    subl $16, %esp
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $3, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_and_8@PLT
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $5, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_and_8@PLT
 ; I486-NEXT:    movl %eax, %ecx
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_and_8@PLT
 ; I486-NEXT:    addl $16, %esp
 ; I486-NEXT:    retl

 ; X64-NEXT:  .LBB3_2: # %atomicrmw.end
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; X64-NEXT:    lock orq %rax, sc64(%rip)
 ; X64-NEXT:    retq
 ;
 ; I486-LABEL: atomic_fetch_or64:
 ; I486:       # %bb.0:
 ; I486-NEXT:    subl $16, %esp
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $3, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_or_8@PLT
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $5, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_or_8@PLT
 ; I486-NEXT:    movl %eax, %ecx
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_or_8@PLT
 ; I486-NEXT:    addl $16, %esp
 ; I486-NEXT:    retl

 ; X64-NEXT:  .LBB4_2: # %atomicrmw.end
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; X64-NEXT:    lock xorq %rax, sc64(%rip)
 ; X64-NEXT:    retq
 ;
 ; I486-LABEL: atomic_fetch_xor64:
 ; I486:       # %bb.0:
 ; I486-NEXT:    subl $16, %esp
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $3, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_xor_8@PLT
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $0, 8(%eax)
 ; I486-NEXT:    movl $5, 4(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_xor_8@PLT
 ; I486-NEXT:    movl %eax, %ecx
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_xor_8@PLT
 ; I486-NEXT:    addl $16, %esp
 ; I486-NEXT:    retl

 ; X64-NEXT:  .LBB5_2: # %atomicrmw.end
 ; X64-NEXT:    retq
 ;
 ; I486-LABEL: atomic_fetch_nand64:
 ; I486:       # %bb.0:
 ; I486-NEXT:    subl $16, %esp
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_fetch_nand_8@PLT
 ; I486-NEXT:    addl $16, %esp
 ; I486-NEXT:    retl

 ; I486:       # %bb.0:
 ; I486-NEXT:    pushl %ebp
 ; I486-NEXT:    movl %esp, %ebp
 ; I486-NEXT:    andl $-8, %esp
 ; I486-NEXT:    subl $32, %esp
 ; I486-NEXT:    leal {{[0-9]+}}(%esp), %ecx
 ; I486-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; I486-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 20(%eax)
 ; I486-NEXT:    movl $2, 16(%eax)
 ; I486-NEXT:    movl $0, 12(%eax)
 ; I486-NEXT:    movl $1, 8(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_compare_exchange_8@PLT

 ; X64-NEXT:    movq %rdi, sc64(%rip)
 ; X64-NEXT:    retq
 ;
 ; I486-LABEL: atomic_fetch_store64:
 ; I486:       # %bb.0:
 ; I486-NEXT:    subl $16, %esp
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $3, 12(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_store_8@PLT
 ; I486-NEXT:    addl $16, %esp
 ; I486-NEXT:    retl
   store atomic i64 %x, i64* @sc64 release, align 8
   ret void
 }
 
 define void @atomic_fetch_swap64(i64 %x) nounwind {
 ; X64-LABEL: atomic_fetch_swap64:
 ; X64:       # %bb.0:
 ; X64-NEXT:    xchgq %rdi, sc64(%rip)
 ; X64-NEXT:    retq
 ;
 ; I486-LABEL: atomic_fetch_swap64:
 ; I486:       # %bb.0:
 ; I486-NEXT:    subl $16, %esp
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $sc64, (%eax)
 ; I486-NEXT:    calll __atomic_exchange_8@PLT
 ; I486-NEXT:    addl $16, %esp
 ; I486-NEXT:    retl

 ; I486-NEXT:    pushl %ebp
 ; I486-NEXT:    movl %esp, %ebp
 ; I486-NEXT:    andl $-8, %esp
 ; I486-NEXT:    subl $24, %esp
 ; I486-NEXT:    fldl 8(%ebp)
 ; I486-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; I486-NEXT:    leal fsc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 12(%eax)
 ; I486-NEXT:    movl $fsc64, (%eax)
 ; I486-NEXT:    calll __atomic_exchange_8@PLT
 ; I486-NEXT:    movl %ebp, %esp
 ; I486-NEXT:    popl %ebp
 ; I486-NEXT:    retl
   %t1 = atomicrmw xchg double* @fsc64, double %x acquire
   ret void
 }

llvm/test/CodeGen/X86/swifterror.ll

 ; CHECK-O0-LABEL: caller:
 ; CHECK-O0:       ## %bb.0: ## %entry
 ; CHECK-O0-NEXT:    pushq %r12
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-O0-NEXT:    subq $32, %rsp
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-O0-NEXT:    .cfi_offset %r12, -16
 ; CHECK-O0-NEXT:    ## implicit-def: $rax
-; CHECK-O0-NEXT:    movq %rdi, (%rsp) ## 8-byte Spill
+; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:    xorl %eax, %eax
 ; CHECK-O0-NEXT:    movl %eax, %r12d
 ; CHECK-O0-NEXT:    callq _foo
 ; CHECK-O0-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; CHECK-O0-NEXT:    movq %r12, %rax
-; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:    cmpq $0, %r12
 ; CHECK-O0-NEXT:    jne LBB1_2
 ; CHECK-O0-NEXT:  ## %bb.1: ## %cont
-; CHECK-O0-NEXT:    movq (%rsp), %rax ## 8-byte Reload
+; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax ## 8-byte Reload
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx ## 8-byte Reload
 ; CHECK-O0-NEXT:    movb 8(%rcx), %cl
 ; CHECK-O0-NEXT:    movb %cl, (%rax)
 ; CHECK-O0-NEXT:  LBB1_2: ## %handler
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi ## 8-byte Reload
 ; CHECK-O0-NEXT:    callq _free
 ; CHECK-O0-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-O0-NEXT:    addq $32, %rsp

 ; CHECK-APPLE-NEXT:    popq %rbx
 ; CHECK-APPLE-NEXT:    popq %r12
 ; CHECK-APPLE-NEXT:    retq
 ;
 ; CHECK-O0-LABEL: caller2:
 ; CHECK-O0:       ## %bb.0: ## %entry
 ; CHECK-O0-NEXT:    pushq %r12
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-O0-NEXT:    subq $48, %rsp
-; CHECK-O0-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-O0-NEXT:    subq $32, %rsp
+; CHECK-O0-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-O0-NEXT:    .cfi_offset %r12, -16
 ; CHECK-O0-NEXT:    ## implicit-def: $rax
 ; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:  LBB2_1: ## %bb_loop
 ; CHECK-O0-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-O0-NEXT:    xorl %eax, %eax
 ; CHECK-O0-NEXT:    movl %eax, %r12d
 ; CHECK-O0-NEXT:    callq _foo
 ; CHECK-O0-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; CHECK-O0-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; CHECK-O0-NEXT:    movq %r12, %rax
-; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:    cmpq $0, %r12
 ; CHECK-O0-NEXT:    jne LBB2_4
 ; CHECK-O0-NEXT:  ## %bb.2: ## %cont
 ; CHECK-O0-NEXT:    ## in Loop: Header=BB2_1 Depth=1
 ; CHECK-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 4-byte Reload
 ; CHECK-O0-NEXT:    ## xmm0 = mem[0],zero,zero,zero
 ; CHECK-O0-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; CHECK-O0-NEXT:    ucomiss %xmm1, %xmm0
 ; CHECK-O0-NEXT:    jbe LBB2_1
 ; CHECK-O0-NEXT:  ## %bb.3: ## %bb_end
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax ## 8-byte Reload
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx ## 8-byte Reload
 ; CHECK-O0-NEXT:    movb 8(%rcx), %cl
 ; CHECK-O0-NEXT:    movb %cl, (%rax)
 ; CHECK-O0-NEXT:  LBB2_4: ## %handler
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi ## 8-byte Reload
 ; CHECK-O0-NEXT:    callq _free
 ; CHECK-O0-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-O0-NEXT:    addq $48, %rsp
+; CHECK-O0-NEXT:    addq $32, %rsp
 ; CHECK-O0-NEXT:    popq %r12
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-i386-LABEL: caller2:
 ; CHECK-i386:       ## %bb.0: ## %entry
 ; CHECK-i386-NEXT:    pushl %edi
 ; CHECK-i386-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-i386-NEXT:    pushl %esi

 ; CHECK-APPLE-NEXT:    popq %rbx
 ; CHECK-APPLE-NEXT:    popq %r12
 ; CHECK-APPLE-NEXT:    retq
 ;
 ; CHECK-O0-LABEL: caller3:
 ; CHECK-O0:       ## %bb.0: ## %entry
 ; CHECK-O0-NEXT:    pushq %r12
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-O0-NEXT:    subq $64, %rsp
-; CHECK-O0-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-O0-NEXT:    subq $48, %rsp
+; CHECK-O0-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-O0-NEXT:    .cfi_offset %r12, -16
 ; CHECK-O0-NEXT:    ## implicit-def: $rax
-; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-O0-NEXT:    movq %rdi, (%rsp) ## 8-byte Spill
 ; CHECK-O0-NEXT:    xorl %eax, %eax
 ; CHECK-O0-NEXT:    movl %eax, %r12d
-; CHECK-O0-NEXT:    leaq 40(%rsp), %rdi
+; CHECK-O0-NEXT:    leaq 24(%rsp), %rdi
 ; CHECK-O0-NEXT:    movl $1, %esi
 ; CHECK-O0-NEXT:    callq _foo_sret
 ; CHECK-O0-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; CHECK-O0-NEXT:    movq %r12, %rax
-; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:    cmpq $0, %r12
 ; CHECK-O0-NEXT:    jne LBB6_2
 ; CHECK-O0-NEXT:  ## %bb.1: ## %cont
-; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax ## 8-byte Reload
+; CHECK-O0-NEXT:    movq (%rsp), %rax ## 8-byte Reload
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx ## 8-byte Reload
 ; CHECK-O0-NEXT:    movb 8(%rcx), %cl
 ; CHECK-O0-NEXT:    movb %cl, (%rax)
 ; CHECK-O0-NEXT:  LBB6_2: ## %handler
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi ## 8-byte Reload
 ; CHECK-O0-NEXT:    callq _free
 ; CHECK-O0-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-O0-NEXT:    addq $64, %rsp
+; CHECK-O0-NEXT:    addq $48, %rsp
 ; CHECK-O0-NEXT:    popq %r12
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-i386-LABEL: caller3:
 ; CHECK-i386:       ## %bb.0: ## %entry
 ; CHECK-i386-NEXT:    subl $44, %esp
 ; CHECK-i386-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-i386-NEXT:    movl $0, 12(%esp)

 ; CHECK-O0-LABEL: caller_with_multiple_swifterror_values:
 ; CHECK-O0:       ## %bb.0: ## %entry
 ; CHECK-O0-NEXT:    pushq %rbp
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-O0-NEXT:    .cfi_offset %rbp, -16
 ; CHECK-O0-NEXT:    movq %rsp, %rbp
 ; CHECK-O0-NEXT:    .cfi_def_cfa_register %rbp
 ; CHECK-O0-NEXT:    pushq %r12
-; CHECK-O0-NEXT:    subq $56, %rsp
+; CHECK-O0-NEXT:    subq $40, %rsp
 ; CHECK-O0-NEXT:    .cfi_offset %r12, -24
 ; CHECK-O0-NEXT:    ## implicit-def: $rax
 ; CHECK-O0-NEXT:    ## implicit-def: $rax
 ; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:    xorl %eax, %eax
 ; CHECK-O0-NEXT:    movl %eax, %r12d
 ; CHECK-O0-NEXT:    callq _foo
 ; CHECK-O0-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; CHECK-O0-NEXT:    movq %r12, %rax
-; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:    cmpq $0, %r12
 ; CHECK-O0-NEXT:    jne LBB7_2
 ; CHECK-O0-NEXT:  ## %bb.1: ## %cont
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax ## 8-byte Reload
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx ## 8-byte Reload
 ; CHECK-O0-NEXT:    movb 8(%rcx), %cl
 ; CHECK-O0-NEXT:    movb %cl, (%rax)
 ; CHECK-O0-NEXT:  LBB7_2: ## %handler
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi ## 8-byte Reload
 ; CHECK-O0-NEXT:    callq _free
 ; CHECK-O0-NEXT:    movq %rsp, %rax
 ; CHECK-O0-NEXT:    addq $-16, %rax
 ; CHECK-O0-NEXT:    movq %rax, %rsp
 ; CHECK-O0-NEXT:    xorl %eax, %eax
 ; CHECK-O0-NEXT:    movl %eax, %r12d
 ; CHECK-O0-NEXT:    callq _foo
 ; CHECK-O0-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; CHECK-O0-NEXT:    movq %r12, %rax
-; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-O0-NEXT:    cmpq $0, %r12
 ; CHECK-O0-NEXT:    jne LBB7_4
 ; CHECK-O0-NEXT:  ## %bb.3: ## %cont2
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax ## 8-byte Reload
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx ## 8-byte Reload
 ; CHECK-O0-NEXT:    movb 8(%rcx), %cl
 ; CHECK-O0-NEXT:    movb %cl, (%rax)
 ; CHECK-O0-NEXT:  LBB7_4: ## %handler2