[X86] Enable shrink-wrapping for frameless non-nounwind functions on platforms not using compact unwind

[MaskRay]

Summary

The current compact unwind scheme does not work when the prologue is not at the
start (the instructions before the prologue cannot be described). (Technically
this is fixable, but it requires multiple compact unwind descriptors for one
function.)

rL255175 chose to not perform shrink-wrapping for frameless functions not
marked as nounwind to work around PR25614. This is overly limited, as platforms
not supporting compact unwind (all non-Darwin) does not need the workaround.
This patch restricts the limitation to compact unwind platforms.

[MaskRay]

Ping:)

llvm/lib/Target/X86/X86FrameLowering.cpp

 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/WinEHFuncInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
 #include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Target/TargetOptions.h"
 #include <cstdlib>
 
 #define DEBUG_TYPE "x86-fl"
 
 STATISTIC(NumFrameLoopProbe, "Number of loop stack probes used in prologue");
bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
   // otherwise, conservatively assume this is not
   // safe to insert the epilogue here.
   return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
 }
 
 bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
   // If we may need to emit frameless compact unwind information, give
   // up as this is currently broken: PR25614.
-  return (MF.getFunction().hasFnAttribute(Attribute::NoUnwind) || hasFP(MF)) &&
-         // The lowering of segmented stack and HiPE only support entry blocks
-         // as prologue blocks: PR26107.
-         // This limitation may be lifted if we fix:
+  bool CompactUnwind =
+      MF.getMMI().getContext().getObjectFileInfo()->getCompactUnwindSection() !=
+      nullptr;
+  return (MF.getFunction().hasFnAttribute(Attribute::NoUnwind) || hasFP(MF) ||
+          !CompactUnwind) &&
+         // The lowering of segmented stack and HiPE only support entry
+         // blocks as prologue blocks: PR26107. This limitation may be
+         // lifted if we fix:
          // - adjustForSegmentedStacks
          // - adjustForHiPEPrologue
          MF.getFunction().getCallingConv() != CallingConv::HiPE &&
          !MF.shouldSplitStack();
 }
 
 MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

llvm/test/CodeGen/X86/absolute-cmp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -show-mc-encoding | FileCheck %s --check-prefixes=CHECK,NOPIC
 ; RUN: llc -relocation-model=pic -show-mc-encoding < %s | FileCheck %s --check-prefixes=CHECK,PIC
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @cmp8 = external hidden global i8, !absolute_symbol !0
 @cmp32 = external hidden global i8, !absolute_symbol !1
 
 declare void @f()
 
 define void @foo8(i64 %val) {
 ; NOPIC-LABEL: foo8:
 ; NOPIC:       # %bb.0:
-; NOPIC-NEXT:    pushq %rax # encoding: [0x50]
-; NOPIC-NEXT:    .cfi_def_cfa_offset 16
 ; NOPIC-NEXT:    cmpq $cmp8@ABS8, %rdi # encoding: [0x48,0x83,0xff,A]
 ; NOPIC-NEXT:    # fixup A - offset: 3, value: cmp8@ABS8, kind: FK_Data_1
 ; NOPIC-NEXT:    ja .LBB0_2 # encoding: [0x77,A]
 ; NOPIC-NEXT:    # fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 ; NOPIC-NEXT:  # %bb.1: # %t
+; NOPIC-NEXT:    pushq %rax # encoding: [0x50]
+; NOPIC-NEXT:    .cfi_def_cfa_offset 16
 ; NOPIC-NEXT:    callq f # encoding: [0xe8,A,A,A,A]
 ; NOPIC-NEXT:    # fixup A - offset: 1, value: f-4, kind: reloc_branch_4byte_pcrel
-; NOPIC-NEXT:  .LBB0_2: # %f
 ; NOPIC-NEXT:    popq %rax # encoding: [0x58]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 8
+; NOPIC-NEXT:  .LBB0_2: # %f
 ; NOPIC-NEXT:    retq # encoding: [0xc3]
 ;
 ; PIC-LABEL: foo8:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    pushq %rax # encoding: [0x50]
-; PIC-NEXT:    .cfi_def_cfa_offset 16
 ; PIC-NEXT:    cmpq $cmp8@ABS8, %rdi # encoding: [0x48,0x83,0xff,A]
 ; PIC-NEXT:    # fixup A - offset: 3, value: cmp8@ABS8, kind: FK_Data_1
 ; PIC-NEXT:    ja .LBB0_2 # encoding: [0x77,A]
 ; PIC-NEXT:    # fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 ; PIC-NEXT:  # %bb.1: # %t
+; PIC-NEXT:    pushq %rax # encoding: [0x50]
+; PIC-NEXT:    .cfi_def_cfa_offset 16
 ; PIC-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
 ; PIC-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
-; PIC-NEXT:  .LBB0_2: # %f
 ; PIC-NEXT:    popq %rax # encoding: [0x58]
 ; PIC-NEXT:    .cfi_def_cfa_offset 8
+; PIC-NEXT:  .LBB0_2: # %f
 ; PIC-NEXT:    retq # encoding: [0xc3]
   %cmp = icmp ule i64 %val, ptrtoint (i8* @cmp8 to i64)
   br i1 %cmp, label %t, label %f
 
 t:
   call void @f()
   ret void
 
 f:
   ret void
 }
 
 define void @foo32(i64 %val) {
 ; NOPIC-LABEL: foo32:
 ; NOPIC:       # %bb.0:
-; NOPIC-NEXT:    pushq %rax # encoding: [0x50]
-; NOPIC-NEXT:    .cfi_def_cfa_offset 16
 ; NOPIC-NEXT:    cmpq $cmp32, %rdi # encoding: [0x48,0x81,0xff,A,A,A,A]
 ; NOPIC-NEXT:    # fixup A - offset: 3, value: cmp32, kind: reloc_signed_4byte
 ; NOPIC-NEXT:    ja .LBB1_2 # encoding: [0x77,A]
 ; NOPIC-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 ; NOPIC-NEXT:  # %bb.1: # %t
+; NOPIC-NEXT:    pushq %rax # encoding: [0x50]
+; NOPIC-NEXT:    .cfi_def_cfa_offset 16
 ; NOPIC-NEXT:    callq f # encoding: [0xe8,A,A,A,A]
 ; NOPIC-NEXT:    # fixup A - offset: 1, value: f-4, kind: reloc_branch_4byte_pcrel
-; NOPIC-NEXT:  .LBB1_2: # %f
 ; NOPIC-NEXT:    popq %rax # encoding: [0x58]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 8
+; NOPIC-NEXT:  .LBB1_2: # %f
 ; NOPIC-NEXT:    retq # encoding: [0xc3]
 ;
 ; PIC-LABEL: foo32:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    pushq %rax # encoding: [0x50]
-; PIC-NEXT:    .cfi_def_cfa_offset 16
 ; PIC-NEXT:    cmpq $cmp32, %rdi # encoding: [0x48,0x81,0xff,A,A,A,A]
 ; PIC-NEXT:    # fixup A - offset: 3, value: cmp32, kind: reloc_signed_4byte
 ; PIC-NEXT:    ja .LBB1_2 # encoding: [0x77,A]
 ; PIC-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 ; PIC-NEXT:  # %bb.1: # %t
+; PIC-NEXT:    pushq %rax # encoding: [0x50]
+; PIC-NEXT:    .cfi_def_cfa_offset 16
 ; PIC-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
 ; PIC-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
-; PIC-NEXT:  .LBB1_2: # %f
 ; PIC-NEXT:    popq %rax # encoding: [0x58]
 ; PIC-NEXT:    .cfi_def_cfa_offset 8
+; PIC-NEXT:  .LBB1_2: # %f
 ; PIC-NEXT:    retq # encoding: [0xc3]
   %cmp = icmp ule i64 %val, ptrtoint (i8* @cmp32 to i64)
   br i1 %cmp, label %t, label %f
 
 t:
   call void @f()
   ret void
 
 f:
   ret void
 }
 
 !0 = !{i64 0, i64 128}
 !1 = !{i64 0, i64 2147483648}

llvm/test/CodeGen/X86/avx512-mask-op.ll

; X86-NEXT:    retl
   %ret = bitcast <8 x i1> %m2 to i8
   ret i8 %ret
 }
 
 ; Make sure we don't emit a ktest for signed comparisons.
 define void @ktest_signed(<16 x i32> %x, <16 x i32> %y) {
 ; KNL-LABEL: ktest_signed:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    pushq %rax
-; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; KNL-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    testw %ax, %ax
 ; KNL-NEXT:    jle LBB66_1
 ; KNL-NEXT:  ## %bb.2: ## %bb.2
-; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ; KNL-NEXT:  LBB66_1: ## %bb.1
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
-; KNL-NEXT:    popq %rax
+; KNL-NEXT:    addq $8, %rsp
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: ktest_signed:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    pushq %rax
-; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    testw %ax, %ax
 ; SKX-NEXT:    jle LBB66_1
 ; SKX-NEXT:  ## %bb.2: ## %bb.2
-; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ; SKX-NEXT:  LBB66_1: ## %bb.1
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
-; SKX-NEXT:    popq %rax
+; SKX-NEXT:    addq $8, %rsp
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: ktest_signed:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    pushq %rax
-; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testw %ax, %ax
 ; AVX512BW-NEXT:    jle LBB66_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %bb.2
-; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ; AVX512BW-NEXT:  LBB66_1: ## %bb.1
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
-; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    addq $8, %rsp
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: ktest_signed:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    pushq %rax
-; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %eax
 ; AVX512DQ-NEXT:    testw %ax, %ax
 ; AVX512DQ-NEXT:    jle LBB66_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %bb.2
-; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ; AVX512DQ-NEXT:  LBB66_1: ## %bb.1
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
-; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    addq $8, %rsp
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: ktest_signed:
 ; X86:       ## %bb.0:
-; X86-NEXT:    subl $12, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; X86-NEXT:    kmovd %k0, %eax
 ; X86-NEXT:    testw %ax, %ax
 ; X86-NEXT:    jle LBB66_1
 ; X86-NEXT:  ## %bb.2: ## %bb.2
-; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ; X86-NEXT:  LBB66_1: ## %bb.1
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    retl
   %a = icmp eq <16 x i32> %x, zeroinitializer
   %b = icmp eq <16 x i32> %y, zeroinitializer
   %c = and <16 x i1> %a, %b
   %d = bitcast <16 x i1> %c to i16
   %e = icmp sgt i16 %d, 0
   br i1 %e, label %bb.2, label %bb.1
 bb.1:
   call void @foo()
   br label %bb.2
 bb.2:
   ret void
 }
 declare void @foo()
 
 ; Make sure we can use the C flag from kortest to check for all ones.
 define void @ktest_allones(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-LABEL: ktest_allones:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; CHECK-NEXT:    kortestw %k0, %k0
 ; CHECK-NEXT:    jb LBB67_2
 ; CHECK-NEXT:  ## %bb.1: ## %bb.1
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    callq _foo
+; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:  LBB67_2: ## %bb.2
-; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
 ;
 ; X86-LABEL: ktest_allones:
 ; X86:       ## %bb.0:
-; X86-NEXT:    subl $12, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; X86-NEXT:    kortestw %k0, %k0
 ; X86-NEXT:    jb LBB67_2
 ; X86-NEXT:  ## %bb.1: ## %bb.1
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
-; X86-NEXT:  LBB67_2: ## %bb.2
 ; X86-NEXT:    addl $12, %esp
+; X86-NEXT:  LBB67_2: ## %bb.2
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
   %a = icmp eq <16 x i32> %x, zeroinitializer
   %b = icmp eq <16 x i32> %y, zeroinitializer
   %c = and <16 x i1> %a, %b
   %d = bitcast <16 x i1> %c to i16
   %e = icmp eq i16 %d, -1
   br i1 %e, label %bb.2, label %bb.1
; X86-NEXT:    retl
   tail call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> %16, <16 x i32>* %17, i32 1, <16 x i1> %18) #2
   ret void
 }
 declare void @llvm.masked.store.v16i32.p0v16i32(<16 x i32>, <16 x i32>*, i32, <16 x i1>)
 
 define void @ktest_3(<8 x i32> %w, <8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
 ; KNL-LABEL: ktest_3:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    pushq %rax
-; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    ## kill: def $ymm3 killed $ymm3 def $zmm3
 ; KNL-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
 ; KNL-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
 ; KNL-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    vptestnmd %zmm1, %zmm1, %k1
 ; KNL-NEXT:    vptestnmd %zmm2, %zmm2, %k2
 ; KNL-NEXT:    vptestnmd %zmm3, %zmm3, %k3
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    korw %k3, %k2, %k1
 ; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    testb %al, %al
 ; KNL-NEXT:    je LBB74_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
-; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ; KNL-NEXT:  LBB74_1: ## %bar
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
-; KNL-NEXT:    popq %rax
+; KNL-NEXT:    addq $8, %rsp
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: ktest_3:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    pushq %rax
-; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k0
 ; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1
 ; SKX-NEXT:    korb %k1, %k0, %k0
 ; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1
 ; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k2
 ; SKX-NEXT:    korb %k2, %k1, %k1
 ; SKX-NEXT:    ktestb %k1, %k0
 ; SKX-NEXT:    je LBB74_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
-; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ; SKX-NEXT:  LBB74_1: ## %bar
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
-; SKX-NEXT:    popq %rax
+; SKX-NEXT:    addq $8, %rsp
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: ktest_3:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    pushq %rax
-; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    ## kill: def $ymm3 killed $ymm3 def $zmm3
 ; AVX512BW-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
 ; AVX512BW-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512BW-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    vptestnmd %zmm1, %zmm1, %k1
 ; AVX512BW-NEXT:    vptestnmd %zmm2, %zmm2, %k2
 ; AVX512BW-NEXT:    vptestnmd %zmm3, %zmm3, %k3
 ; AVX512BW-NEXT:    korw %k1, %k0, %k0
 ; AVX512BW-NEXT:    korw %k3, %k2, %k1
 ; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    je LBB74_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
-; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ; AVX512BW-NEXT:  LBB74_1: ## %bar
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
-; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    addq $8, %rsp
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: ktest_3:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    pushq %rax
-; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    ## kill: def $ymm3 killed $ymm3 def $zmm3
 ; AVX512DQ-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
 ; AVX512DQ-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512DQ-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512DQ-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512DQ-NEXT:    vptestnmd %zmm1, %zmm1, %k1
 ; AVX512DQ-NEXT:    vptestnmd %zmm2, %zmm2, %k2
 ; AVX512DQ-NEXT:    vptestnmd %zmm3, %zmm3, %k3
 ; AVX512DQ-NEXT:    korb %k1, %k0, %k0
 ; AVX512DQ-NEXT:    korb %k3, %k2, %k1
 ; AVX512DQ-NEXT:    ktestb %k1, %k0
 ; AVX512DQ-NEXT:    je LBB74_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
-; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ; AVX512DQ-NEXT:  LBB74_1: ## %bar
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
-; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    addq $8, %rsp
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: ktest_3:
 ; X86:       ## %bb.0:
-; X86-NEXT:    subl $12, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vptestnmd %ymm0, %ymm0, %k0
 ; X86-NEXT:    vptestnmd %ymm1, %ymm1, %k1
 ; X86-NEXT:    korb %k1, %k0, %k0
 ; X86-NEXT:    vptestnmd %ymm2, %ymm2, %k1
 ; X86-NEXT:    vptestnmd %ymm3, %ymm3, %k2
 ; X86-NEXT:    korb %k2, %k1, %k1
 ; X86-NEXT:    ktestb %k1, %k0
 ; X86-NEXT:    je LBB74_1
 ; X86-NEXT:  ## %bb.2: ## %exit
-; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ; X86-NEXT:  LBB74_1: ## %bar
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    retl
   %a = icmp eq <8 x i32> %w, zeroinitializer
   %b = icmp eq <8 x i32> %x, zeroinitializer
   %c = icmp eq <8 x i32> %y, zeroinitializer
   %d = icmp eq <8 x i32> %z, zeroinitializer

 
 exit:
   ret void
 }
 
 define void @ktest_4(<8 x i64> %w, <8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
 ; KNL-LABEL: ktest_4:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    pushq %rax
-; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
 ; KNL-NEXT:    vptestnmq %zmm2, %zmm2, %k2
 ; KNL-NEXT:    vptestnmq %zmm3, %zmm3, %k3
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    korw %k3, %k2, %k1
 ; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    testb %al, %al
 ; KNL-NEXT:    je LBB75_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
-; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ; KNL-NEXT:  LBB75_1: ## %bar
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
-; KNL-NEXT:    popq %rax
+; KNL-NEXT:    addq $8, %rsp
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: ktest_4:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    pushq %rax
-; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1
 ; SKX-NEXT:    korb %k1, %k0, %k0
 ; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1
 ; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k2
 ; SKX-NEXT:    korb %k2, %k1, %k1
 ; SKX-NEXT:    ktestb %k1, %k0
 ; SKX-NEXT:    je LBB75_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
-; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ; SKX-NEXT:  LBB75_1: ## %bar
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
-; SKX-NEXT:    popq %rax
+; SKX-NEXT:    addq $8, %rsp
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: ktest_4:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    pushq %rax
-; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    vptestnmq %zmm1, %zmm1, %k1
 ; AVX512BW-NEXT:    vptestnmq %zmm2, %zmm2, %k2
 ; AVX512BW-NEXT:    vptestnmq %zmm3, %zmm3, %k3
 ; AVX512BW-NEXT:    korw %k1, %k0, %k0
 ; AVX512BW-NEXT:    korw %k3, %k2, %k1
 ; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    je LBB75_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
-; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ; AVX512BW-NEXT:  LBB75_1: ## %bar
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
-; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    addq $8, %rsp
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: ktest_4:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    pushq %rax
-; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512DQ-NEXT:    vptestnmq %zmm1, %zmm1, %k1
 ; AVX512DQ-NEXT:    korb %k1, %k0, %k0
 ; AVX512DQ-NEXT:    vptestnmq %zmm2, %zmm2, %k1
 ; AVX512DQ-NEXT:    vptestnmq %zmm3, %zmm3, %k2
 ; AVX512DQ-NEXT:    korb %k2, %k1, %k1
 ; AVX512DQ-NEXT:    ktestb %k1, %k0
 ; AVX512DQ-NEXT:    je LBB75_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
-; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ; AVX512DQ-NEXT:  LBB75_1: ## %bar
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
-; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    addq $8, %rsp
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: ktest_4:
 ; X86:       ## %bb.0:
-; X86-NEXT:    subl $12, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; X86-NEXT:    vptestnmq %zmm1, %zmm1, %k1
 ; X86-NEXT:    korb %k1, %k0, %k0
 ; X86-NEXT:    vptestnmq %zmm2, %zmm2, %k1
 ; X86-NEXT:    vptestnmq %zmm3, %zmm3, %k2
 ; X86-NEXT:    korb %k2, %k1, %k1
 ; X86-NEXT:    ktestb %k1, %k0
 ; X86-NEXT:    je LBB75_1
 ; X86-NEXT:  ## %bb.2: ## %exit
-; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ; X86-NEXT:  LBB75_1: ## %bar
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    retl
   %a = icmp eq <8 x i64> %w, zeroinitializer
   %b = icmp eq <8 x i64> %x, zeroinitializer
   %c = icmp eq <8 x i64> %y, zeroinitializer
   %d = icmp eq <8 x i64> %z, zeroinitializer

 
 exit:
   ret void
 }
 
 define void @ktest_5(<16 x i32> %w, <16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
 ; KNL-LABEL: ktest_5:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    pushq %rax
-; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    vptestnmd %zmm1, %zmm1, %k1
 ; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; KNL-NEXT:    vptestnmd %zmm3, %zmm3, %k2
 ; KNL-NEXT:    korw %k2, %k1, %k1
 ; KNL-NEXT:    kandw %k1, %k0, %k0
 ; KNL-NEXT:    kortestw %k0, %k0
 ; KNL-NEXT:    je LBB76_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
-; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ; KNL-NEXT:  LBB76_1: ## %bar
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
-; KNL-NEXT:    popq %rax
+; KNL-NEXT:    addq $8, %rsp
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: ktest_5:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    pushq %rax
-; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1
 ; SKX-NEXT:    korw %k1, %k0, %k0
 ; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k2
 ; SKX-NEXT:    korw %k2, %k1, %k1
 ; SKX-NEXT:    ktestw %k1, %k0
 ; SKX-NEXT:    je LBB76_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
-; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ; SKX-NEXT:  LBB76_1: ## %bar
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
-; SKX-NEXT:    popq %rax
+; SKX-NEXT:    addq $8, %rsp
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: ktest_5:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    pushq %rax
-; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    vptestnmd %zmm1, %zmm1, %k1
 ; AVX512BW-NEXT:    korw %k1, %k0, %k0
 ; AVX512BW-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vptestnmd %zmm3, %zmm3, %k2
 ; AVX512BW-NEXT:    korw %k2, %k1, %k1
 ; AVX512BW-NEXT:    kandw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kortestw %k0, %k0
 ; AVX512BW-NEXT:    je LBB76_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
-; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ; AVX512BW-NEXT:  LBB76_1: ## %bar
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
-; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    addq $8, %rsp
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: ktest_5:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    pushq %rax
-; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512DQ-NEXT:    vptestnmd %zmm1, %zmm1, %k1
 ; AVX512DQ-NEXT:    korw %k1, %k0, %k0
 ; AVX512DQ-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512DQ-NEXT:    vptestnmd %zmm3, %zmm3, %k2
 ; AVX512DQ-NEXT:    korw %k2, %k1, %k1
 ; AVX512DQ-NEXT:    ktestw %k1, %k0
 ; AVX512DQ-NEXT:    je LBB76_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
-; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ; AVX512DQ-NEXT:  LBB76_1: ## %bar
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
-; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    addq $8, %rsp
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: ktest_5:
 ; X86:       ## %bb.0:
-; X86-NEXT:    subl $12, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; X86-NEXT:    vptestnmd %zmm1, %zmm1, %k1
 ; X86-NEXT:    korw %k1, %k0, %k0
 ; X86-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; X86-NEXT:    vptestnmd %zmm3, %zmm3, %k2
 ; X86-NEXT:    korw %k2, %k1, %k1
 ; X86-NEXT:    ktestw %k1, %k0
 ; X86-NEXT:    je LBB76_1
 ; X86-NEXT:  ## %bb.2: ## %exit
-; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ; X86-NEXT:  LBB76_1: ## %bar
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    retl
   %a = icmp eq <16 x i32> %w, zeroinitializer
   %b = icmp eq <16 x i32> %x, zeroinitializer
   %c = icmp eq <16 x i32> %y, zeroinitializer
   %d = icmp eq <16 x i32> %z, zeroinitializer

 
 exit:
   ret void
 }
 
 define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z) {
 ; KNL-LABEL: ktest_6:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    pushq %rax
-; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
 ; KNL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
 ; KNL-NEXT:    vpcmpeqw %ymm5, %ymm4, %ymm4
 ; KNL-NEXT:    vpcmpeqw %ymm5, %ymm0, %ymm0
 ; KNL-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
 ; KNL-NEXT:    vpcmpeqw %ymm5, %ymm4, %ymm4
 ; KNL-NEXT:    vpcmpeqw %ymm5, %ymm1, %ymm1

 ; KNL-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
 ; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kmovw %k0, %ecx
 ; KNL-NEXT:    shll $16, %ecx
 ; KNL-NEXT:    orl %eax, %ecx
 ; KNL-NEXT:    je LBB77_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
-; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ; KNL-NEXT:  LBB77_1: ## %bar
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
-; KNL-NEXT:    popq %rax
+; KNL-NEXT:    addq $8, %rsp
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: ktest_6:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    pushq %rax
-; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k0
 ; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1
 ; SKX-NEXT:    kord %k1, %k0, %k0
 ; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; SKX-NEXT:    vptestnmw %zmm3, %zmm3, %k2
 ; SKX-NEXT:    kord %k2, %k1, %k1
 ; SKX-NEXT:    ktestd %k1, %k0
 ; SKX-NEXT:    je LBB77_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
-; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ; SKX-NEXT:  LBB77_1: ## %bar
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
-; SKX-NEXT:    popq %rax
+; SKX-NEXT:    addq $8, %rsp
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: ktest_6:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    pushq %rax
-; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vptestnmw %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm1, %k1
 ; AVX512BW-NEXT:    kord %k1, %k0, %k0
 ; AVX512BW-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vptestnmw %zmm3, %zmm3, %k2
 ; AVX512BW-NEXT:    kord %k2, %k1, %k1
 ; AVX512BW-NEXT:    ktestd %k1, %k0
 ; AVX512BW-NEXT:    je LBB77_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
-; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ; AVX512BW-NEXT:  LBB77_1: ## %bar
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
-; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    addq $8, %rsp
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: ktest_6:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    pushq %rax
-; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
 ; AVX512DQ-NEXT:    vpxor %xmm5, %xmm5, %xmm5
 ; AVX512DQ-NEXT:    vpcmpeqw %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpcmpeqw %ymm5, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
 ; AVX512DQ-NEXT:    vpcmpeqw %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpcmpeqw %ymm5, %ymm1, %ymm1

 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
 ; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %ecx
 ; AVX512DQ-NEXT:    shll $16, %ecx
 ; AVX512DQ-NEXT:    orl %eax, %ecx
 ; AVX512DQ-NEXT:    je LBB77_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
-; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ; AVX512DQ-NEXT:  LBB77_1: ## %bar
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
-; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    addq $8, %rsp
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: ktest_6:
 ; X86:       ## %bb.0:
-; X86-NEXT:    subl $12, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vptestnmw %zmm0, %zmm0, %k0
 ; X86-NEXT:    vptestnmw %zmm1, %zmm1, %k1
 ; X86-NEXT:    kord %k1, %k0, %k0
 ; X86-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; X86-NEXT:    vptestnmw %zmm3, %zmm3, %k2
 ; X86-NEXT:    kord %k2, %k1, %k1
 ; X86-NEXT:    ktestd %k1, %k0
 ; X86-NEXT:    je LBB77_1
 ; X86-NEXT:  ## %bb.2: ## %exit
-; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ; X86-NEXT:  LBB77_1: ## %bar
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    retl
   %a = icmp eq <32 x i16> %w, zeroinitializer
   %b = icmp eq <32 x i16> %x, zeroinitializer
   %c = icmp eq <32 x i16> %y, zeroinitializer
   %d = icmp eq <32 x i16> %z, zeroinitializer

 
 exit:
   ret void
 }
 
 define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
 ; KNL-LABEL: ktest_7:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    pushq %rax
-; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
 ; KNL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
 ; KNL-NEXT:    vpcmpeqb %ymm5, %ymm4, %ymm4
 ; KNL-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
 ; KNL-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
 ; KNL-NEXT:    vpcmpeqb %ymm5, %ymm4, %ymm4
 ; KNL-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm1

 ; KNL-NEXT:    vpternlogq $200, %zmm1, %zmm0, %zmm2
 ; KNL-NEXT:    vpmovmskb %ymm2, %eax
 ; KNL-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
 ; KNL-NEXT:    vpmovmskb %ymm0, %ecx
 ; KNL-NEXT:    shlq $32, %rcx
 ; KNL-NEXT:    orq %rax, %rcx
 ; KNL-NEXT:    je LBB78_1
 ; KNL-NEXT:  ## %bb.2: ## %exit
-; KNL-NEXT:    popq %rax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ; KNL-NEXT:  LBB78_1: ## %bar
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    callq _foo
-; KNL-NEXT:    popq %rax
+; KNL-NEXT:    addq $8, %rsp
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: ktest_7:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    pushq %rax
-; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vptestnmb %zmm0, %zmm0, %k0
 ; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1
 ; SKX-NEXT:    korq %k1, %k0, %k0
 ; SKX-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; SKX-NEXT:    vptestnmb %zmm3, %zmm3, %k2
 ; SKX-NEXT:    korq %k2, %k1, %k1
 ; SKX-NEXT:    ktestq %k1, %k0
 ; SKX-NEXT:    je LBB78_1
 ; SKX-NEXT:  ## %bb.2: ## %exit
-; SKX-NEXT:    popq %rax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ; SKX-NEXT:  LBB78_1: ## %bar
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    callq _foo
-; SKX-NEXT:    popq %rax
+; SKX-NEXT:    addq $8, %rsp
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: ktest_7:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    pushq %rax
-; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    vptestnmb %zmm1, %zmm1, %k1
 ; AVX512BW-NEXT:    korq %k1, %k0, %k0
 ; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vptestnmb %zmm3, %zmm3, %k2
 ; AVX512BW-NEXT:    korq %k2, %k1, %k1
 ; AVX512BW-NEXT:    ktestq %k1, %k0
 ; AVX512BW-NEXT:    je LBB78_1
 ; AVX512BW-NEXT:  ## %bb.2: ## %exit
-; AVX512BW-NEXT:    popq %rax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ; AVX512BW-NEXT:  LBB78_1: ## %bar
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    callq _foo
-; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    addq $8, %rsp
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: ktest_7:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    pushq %rax
-; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
 ; AVX512DQ-NEXT:    vpxor %xmm5, %xmm5, %xmm5
 ; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
 ; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm4, %ymm4
 ; AVX512DQ-NEXT:    vpcmpeqb %ymm5, %ymm1, %ymm1

 ; AVX512DQ-NEXT:    vpternlogq $200, %zmm1, %zmm0, %zmm2
 ; AVX512DQ-NEXT:    vpmovmskb %ymm2, %eax
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm0
 ; AVX512DQ-NEXT:    vpmovmskb %ymm0, %ecx
 ; AVX512DQ-NEXT:    shlq $32, %rcx
 ; AVX512DQ-NEXT:    orq %rax, %rcx
 ; AVX512DQ-NEXT:    je LBB78_1
 ; AVX512DQ-NEXT:  ## %bb.2: ## %exit
-; AVX512DQ-NEXT:    popq %rax
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ; AVX512DQ-NEXT:  LBB78_1: ## %bar
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    callq _foo
-; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    addq $8, %rsp
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: ktest_7:
 ; X86:       ## %bb.0:
-; X86-NEXT:    subl $12, %esp
-; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vptestnmb %zmm0, %zmm0, %k0
 ; X86-NEXT:    vptestnmb %zmm1, %zmm1, %k1
 ; X86-NEXT:    korq %k1, %k0, %k0
 ; X86-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; X86-NEXT:    vptestnmb %zmm3, %zmm3, %k2
 ; X86-NEXT:    korq %k2, %k1, %k1
 ; X86-NEXT:    kandq %k1, %k0, %k0
 ; X86-NEXT:    kshiftrq $32, %k0, %k1
 ; X86-NEXT:    kortestd %k1, %k0
 ; X86-NEXT:    je LBB78_1
 ; X86-NEXT:  ## %bb.2: ## %exit
-; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ; X86-NEXT:  LBB78_1: ## %bar
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    calll _foo
 ; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    retl
   %a = icmp eq <64 x i8> %w, zeroinitializer
   %b = icmp eq <64 x i8> %x, zeroinitializer
   %c = icmp eq <64 x i8> %y, zeroinitializer
   %d = icmp eq <64 x i8> %z, zeroinitializer

llvm/test/CodeGen/X86/live-out-reg-info.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
 ; Make sure dagcombine doesn't eliminate the comparison due
 ; to an off-by-one bug with computeKnownBits information.
 
 declare void @qux()
 
 define void @foo(i32 %a) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    shrl $23, %edi
 ; CHECK-NEXT:    testl $256, %edi # imm = 0x100
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %true
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq qux
-; CHECK-NEXT:  .LBB0_2: # %false
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB0_2: # %false
 ; CHECK-NEXT:    retq
   %t0 = lshr i32 %a, 23
   br label %next
 next:
   %t1 = and i32 %t0, 256
   %t2 = icmp eq i32 %t1, 0
   br i1 %t2, label %true, label %false
 true:
   call void @qux()
   ret void
 false:
   ret void
 }
 

llvm/test/CodeGen/X86/swifterror.ll

 
 entry:
   %call = call swiftcc float @moo(%swift_error** swifterror %error_ptr_ref)
   ret float %call
 }
 
 define swiftcc float @conditionally_forward_swifterror(%swift_error** swifterror %error_ptr_ref, i32 %cc) {
 ; CHECK-APPLE-LABEL: conditionally_forward_swifterror:
-; CHECK-APPLE:  pushq %rax
 ; CHECK-APPLE:  testl %edi, %edi
 ; CHECK-APPLE:  je
 
+; CHECK-APPLE:  pushq %rax
 ; CHECK-APPLE:  callq _moo
 ; CHECK-APPLE:  popq %rax
 ; CHECK-APPLE:  retq
 
 ; CHECK-APPLE:  xorps %xmm0, %xmm0
-; CHECK-APPLE:  popq %rax
 ; CHECK-APPLE:  retq
 
 ; CHECK-O0-LABEL: conditionally_forward_swifterror:
 ; CHECK-O0: pushq [[REG1:%[a-z0-9]+]]
 ; CHECK-O0-DAG:  movq %r12, (%rsp)
 ; CHECK-O0:  cmpl $0, %edi
 ; CHECK-O0:  je
 

 ; CHECK-APPLE:  retq
 define swiftcc {i32, i32, i32} @empty_swiftcc({i32, i32, i32} , %swift_error** swifterror %error_ptr_ref) {
 entry:
   ret {i32, i32, i32} %0
 }
 
 ; Make sure we can handle the case when isel generates new machine basic blocks.
 ; CHECK-APPLE-LABEL: dont_crash_on_new_isel_blocks:
-; CHECK-APPLE: pushq   %rax
 ; CHECK-APPLE: xorl    %eax, %eax
 ; CHECK-APPLE: testb   %al, %al
 ; CHECK-APPLE: jne
-; CHECK-APPLE: callq   *%rax
-; CHECK-APPLE: popq    %rax
-; CHECK-APPLE: ret
+; CHECK-APPLE:         pushq   %rax
+; CHECK-APPLE-NEXT:  .cfi_def_cfa_offset 16
+; CHECK-APPLE-NEXT:    callq   *%rax
+; CHECK-APPLE-NEXT:    popq    %rax
+; CHECK-APPLE-NEXT:    ret
 
 define swiftcc void @dont_crash_on_new_isel_blocks(%swift_error** nocapture swifterror, i1, i8**) {
 entry:
   %3 = or i1 false, %1
   br i1 %3, label %cont, label %falsebb
 
 falsebb:
   %4 = load i8*, i8** %2, align 8

llvm/test/CodeGen/X86/test-vs-bittest.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
 define void @test64(i64 inreg %x) {
 ; CHECK-LABEL: test64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB0_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB0_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 2048
   %s = icmp eq i64 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_optsize(i64 inreg %x) optsize {
 ; CHECK-LABEL: test64_optsize:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jb .LBB1_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB1_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB1_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 2048
   %s = icmp eq i64 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_pgso(i64 inreg %x) !prof !14 {
 ; CHECK-LABEL: test64_pgso:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jb .LBB2_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB2_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB2_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 2048
   %s = icmp eq i64 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 ; This test is identical to test64 above with only the destination of the br
 ; reversed. This somehow causes the two functions to get slightly different
 ; initial IR. One has an extra invert of the setcc. This previous caused one
 ; the functions to use a BT while the other used a TEST due to another DAG
 ; combine messing with an expected canonical form.
 define void @test64_2(i64 inreg %x) {
 ; CHECK-LABEL: test64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
 ; CHECK-NEXT:    je .LBB3_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB3_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB3_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 2048
   %s = icmp eq i64 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_optsize_2(i64 inreg %x) optsize {
 ; CHECK-LABEL: test64_optsize_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jae .LBB4_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB4_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB4_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 2048
   %s = icmp eq i64 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_pgso_2(i64 inreg %x) !prof !14 {
 ; CHECK-LABEL: test64_pgso_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jae .LBB5_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB5_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB5_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 2048
   %s = icmp eq i64 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_3(i64 inreg %x) {
 ; CHECK-LABEL: test64_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
 ; CHECK-NEXT:    jb .LBB6_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB6_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB6_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 4294967296
   %s = icmp eq i64 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_optsize_3(i64 inreg %x) optsize {
 ; CHECK-LABEL: test64_optsize_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
 ; CHECK-NEXT:    jb .LBB7_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB7_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB7_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 4294967296
   %s = icmp eq i64 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_pgso_3(i64 inreg %x) !prof !14 {
 ; CHECK-LABEL: test64_pgso_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
 ; CHECK-NEXT:    jb .LBB8_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB8_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB8_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 4294967296
   %s = icmp eq i64 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_4(i64 inreg %x) {
 ; CHECK-LABEL: test64_4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
 ; CHECK-NEXT:    jae .LBB9_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB9_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB9_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 4294967296
   %s = icmp eq i64 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_optsize_4(i64 inreg %x) optsize {
 ; CHECK-LABEL: test64_optsize_4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
 ; CHECK-NEXT:    jae .LBB10_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB10_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB10_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 4294967296
   %s = icmp eq i64 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test64_pgso_4(i64 inreg %x) !prof !14 {
 ; CHECK-LABEL: test64_pgso_4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btq $32, %rdi
 ; CHECK-NEXT:    jae .LBB11_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB11_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB11_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i64 %x, 4294967296
   %s = icmp eq i64 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test32(i32 inreg %x) {
 ; CHECK-LABEL: test32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
 ; CHECK-NEXT:    jne .LBB12_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB12_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB12_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i32 %x, 2048
   %s = icmp eq i32 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test32_optsize(i32 inreg %x) optsize {
 ; CHECK-LABEL: test32_optsize:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jb .LBB13_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB13_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB13_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i32 %x, 2048
   %s = icmp eq i32 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test32_2(i32 inreg %x) {
 ; CHECK-LABEL: test32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
 ; CHECK-NEXT:    je .LBB14_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB14_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB14_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i32 %x, 2048
   %s = icmp eq i32 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test32_optsize_2(i32 inreg %x) optsize {
 ; CHECK-LABEL: test32_optsize_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jae .LBB15_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB15_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB15_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i32 %x, 2048
   %s = icmp eq i32 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test32_pgso_2(i32 inreg %x) !prof !14 {
 ; CHECK-LABEL: test32_pgso_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jae .LBB16_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB16_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB16_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i32 %x, 2048
   %s = icmp eq i32 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test16(i16 inreg %x) {
 ; CHECK-LABEL: test16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
 ; CHECK-NEXT:    jne .LBB17_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB17_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB17_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i16 %x, 2048
   %s = icmp eq i16 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test16_optsize(i16 inreg %x) optsize {
 ; CHECK-LABEL: test16_optsize:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jb .LBB18_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB18_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB18_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i16 %x, 2048
   %s = icmp eq i16 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test16_pgso(i16 inreg %x) !prof !14 {
 ; CHECK-LABEL: test16_pgso:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jb .LBB19_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB19_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB19_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i16 %x, 2048
   %s = icmp eq i16 %t, 0
   br i1 %s, label %yes, label %no
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test16_2(i16 inreg %x) {
 ; CHECK-LABEL: test16_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    testl $2048, %edi # imm = 0x800
 ; CHECK-NEXT:    je .LBB20_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB20_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB20_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i16 %x, 2048
   %s = icmp eq i16 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test16_optsize_2(i16 inreg %x) optsize {
 ; CHECK-LABEL: test16_optsize_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jae .LBB21_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB21_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB21_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i16 %x, 2048
   %s = icmp eq i16 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void
 no:
   ret void
 }
 
 define void @test16_pgso_2(i16 inreg %x) !prof !14 {
 ; CHECK-LABEL: test16_pgso_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    btl $11, %edi
 ; CHECK-NEXT:    jae .LBB22_2
 ; CHECK-NEXT:  # %bb.1: # %yes
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq bar
-; CHECK-NEXT:  .LBB22_2: # %no
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB22_2: # %no
 ; CHECK-NEXT:    retq
   %t = and i16 %x, 2048
   %s = icmp eq i16 %t, 0
   br i1 %s, label %no, label %yes
 
 yes:
   call void @bar()
   ret void

llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll

-; RUN: llc %s -o - | FileCheck %s
+; RUN: llc -mtriple=x86_64-apple-darwin10.6 < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-linux < %s | FileCheck %s --check-prefix=NOCOMPACTUNWIND
 ;
 ; Note: This test cannot be merged with the shrink-wrapping tests
 ; because the booleans set on the command line take precedence on
 ; the target logic that disable shrink-wrapping.
-target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-target triple = "x86_64-apple-macosx"
 
-
-; This test checks that we do not use shrink-wrapping when
-; the function does not have any frame pointer and may unwind.
-; This is a workaround for a limitation in the emission of
-; the CFI directives, that are not correct in such case.
-; PR25614
+; The current compact unwind scheme does not work when the prologue is not at
+; the start (the instructions before the prologue cannot be described).
+; Currently we choose to not perform shrink-wrapping for functions without FP
+; not marked as nounwind. PR25614
 ;
 ; No shrink-wrapping should occur here, until the CFI information are fixed.
 ; CHECK-LABEL: framelessUnwind:
 ;
 ; Prologue code.
 ; (What we push does not matter. It should be some random sratch register.)
 ; CHECK: pushq
 ;

 ;
 ; CHECK: [[EXIT_LABEL]]:
 ;
 ; Without shrink-wrapping, epilogue is in the exit block.
 ; Epilogue code. (What we pop does not matter.)
 ; CHECK-NEXT: popq
 ;
 ; CHECK-NEXT: retq
+
+; On a platform which does not support compact unwind, shrink wrapping is enabled.
+; NOCOMPACTUNWIND-LABEL: framelessUnwind:
+; NOCOMPACTUNWIND-NOT:     pushq
+; NOCOMPACTUNWIND:       # %bb.1:
+; NOCOMPACTUNWIND-NEXT:    pushq %rax
 define i32 @framelessUnwind(i32 %a, i32 %b) #0 {
   %tmp = alloca i32, align 4
   %tmp2 = icmp slt i32 %a, %b
   br i1 %tmp2, label %true, label %false
 
 true:
   store i32 %a, i32* %tmp, align 4
   %tmp4 = call i32 @doSomething(i32 0, i32* %tmp)