[X86] Bring back the MOV64r0 pseudo instruction

[craig.topper]

Summary

This patch brings back the MOV64r0 pseudo instruction for zeroing a 64-bit register. This replaces the SUBREG_TO_REG MOV32r0 sequence we use today. Post register allocation we will rewrite the MOV64r0 to a 32-bit xor with an implicit def of the 64-bit register similar to what we do for the various XMM/YMM/ZMM zeroing pseudos.

My main motivation is to enable the spill optimization in foldMemoryOperandImpl. As we were seeing some code that repeatedly did "xor eax, eax; store eax;" to spill several registers with a new xor for each store. With this optimization enabled we get a store of a 0 immediate instead of an xor. Though I admit the ideal solution would be one xor where there are multiple spills. I don't believe we have a test case that shows this optimization in here. I'll see if I can try to reduce one from the code were looking at.

There's definitely some other machine CSE(and maybe other passes) behavior changes exposed by this patch. So it seems like there might be some other deficiencies in SUBREG_TO_REG handling.

[craig.topper]

Rebase and add spill-zero-x86_64.ll to cover the stack spill case that originally prompted me to look at this. I've reduced this from a complex function that has something like 20 spills of 0 in it. This is the best I was able to reduce it. We don't seem to have any directed tests for spill a 32-bit zero either. It seems to happen in a half dozen or so larger tests.

[craig.topper]

Update the fast-isel code in MaterializeInt as well

[RKSimon]

LGTM

llvm/trunk/lib/Target/X86/X86FastISel.cpp

bool IsOpSigned;          // Whether to use signed or unsigned form.
         { X86::IDIV32r, X86::CDQ,     Copy,            X86::EDX, S }, // SRem
         { X86::DIV32r,  X86::MOV32r0, Copy,            X86::EAX, U }, // UDiv
         { X86::DIV32r,  X86::MOV32r0, Copy,            X86::EDX, U }, // URem
       }
     }, // i32
     { &X86::GR64RegClass, X86::RAX, X86::RDX, {
         { X86::IDIV64r, X86::CQO,     Copy,            X86::RAX, S }, // SDiv
         { X86::IDIV64r, X86::CQO,     Copy,            X86::RDX, S }, // SRem
-        { X86::DIV64r,  X86::MOV32r0, Copy,            X86::RAX, U }, // UDiv
-        { X86::DIV64r,  X86::MOV32r0, Copy,            X86::RDX, U }, // URem
+        { X86::DIV64r,  X86::MOV64r0, Copy,            X86::RAX, U }, // UDiv
+        { X86::DIV64r,  X86::MOV64r0, Copy,            X86::RDX, U }, // URem
       }
     }, // i64
   };
 
   MVT VT;
   if (!isTypeLegal(I->getType(), VT))
     return false;
 
bool X86FastISel::X86SelectDivRem(const Instruction *I) {
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(OpEntry.OpCopy), TypeEntry.LowInReg).addReg(Op0Reg);
   // Zero-extend or sign-extend into high-order input register.
   if (OpEntry.OpSignExtend) {
     if (OpEntry.IsOpSigned)
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
               TII.get(OpEntry.OpSignExtend));
     else {
-      unsigned Zero32 = createResultReg(&X86::GR32RegClass);
+      unsigned ZeroReg = createResultReg(VT == MVT::i64 ? &X86::GR64RegClass
+                                                        : &X86::GR32RegClass);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-              TII.get(X86::MOV32r0), Zero32);
+              TII.get(OpEntry.OpSignExtend), ZeroReg);
 
       // Copy the zero into the appropriate sub/super/identical physical
       // register. Unfortunately the operations needed are not uniform enough
       // to fit neatly into the table above.
-      if (VT == MVT::i16) {
+      if (VT == MVT::i16)
         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
                 TII.get(Copy), TypeEntry.HighInReg)
-          .addReg(Zero32, 0, X86::sub_16bit);
-      } else if (VT == MVT::i32) {
+          .addReg(ZeroReg, 0, X86::sub_16bit);
+      else
         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
                 TII.get(Copy), TypeEntry.HighInReg)
-            .addReg(Zero32);
-      } else if (VT == MVT::i64) {
-        BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-                TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
-            .addImm(0).addReg(Zero32).addImm(X86::sub_32bit);
-      }
+            .addReg(ZeroReg);
     }
   }
   // Generate the DIV/IDIV instruction.
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
           TII.get(OpEntry.OpDivRem)).addReg(Op1Reg);
   // For i8 remainder, we can't reference ah directly, as we'll end
   // up with bogus copies like %r9b = COPY %ah. Reference ax
   // instead to prevent ah references in a rex instruction.

 }
 
 unsigned X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
   if (VT > MVT::i64)
     return 0;
 
   uint64_t Imm = CI->getZExtValue();
   if (Imm == 0) {
+    if (VT.SimpleTy == MVT::i64)
+      return fastEmitInst_(X86::MOV64r0, &X86::GR64RegClass);
+
     unsigned SrcReg = fastEmitInst_(X86::MOV32r0, &X86::GR32RegClass);
     switch (VT.SimpleTy) {
     default: llvm_unreachable("Unexpected value type");
     case MVT::i1:
     case MVT::i8:
       return fastEmitInst_extractsubreg(MVT::i8, SrcReg, /*Kill=*/true,
                                         X86::sub_8bit);
     case MVT::i16:
       return fastEmitInst_extractsubreg(MVT::i16, SrcReg, /*Kill=*/true,
                                         X86::sub_16bit);
     case MVT::i32:
       return SrcReg;
-    case MVT::i64: {
-      unsigned ResultReg = createResultReg(&X86::GR64RegClass);
-      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-              TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)
-        .addImm(0).addReg(SrcReg).addImm(X86::sub_32bit);
-      return ResultReg;
-    }
     }
   }
 
   unsigned Opc = 0;
   switch (VT.SimpleTy) {
   default: llvm_unreachable("Unexpected value type");
   case MVT::i1:
     VT = MVT::i8;

llvm/trunk/lib/Target/X86/X86ISelDAGToDAG.cpp

if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) {
         CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
                              LoReg, N0, SDValue()).getValue(1);
       if (isSigned && !signBitIsZero) {
         // Sign extend the low part into the high part.
         InFlag =
           SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InFlag),0);
       } else {
         // Zero out the high part, effectively zero extending the input.
-        SDValue ClrNode = SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, NVT), 0);
+        unsigned ClrOpc = NVT.SimpleTy == MVT::i64 ? X86::MOV64r0
+                                                   : X86::MOV32r0;
+        MVT ClrVT = NVT.SimpleTy == MVT::i64 ? MVT::i64 : MVT::i32;
+        SDValue ClrNode = SDValue(CurDAG->getMachineNode(ClrOpc, dl, ClrVT), 0);
         switch (NVT.SimpleTy) {
         case MVT::i16:
           ClrNode =
               SDValue(CurDAG->getMachineNode(
                           TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
                           CurDAG->getTargetConstant(X86::sub_16bit, dl,
                                                     MVT::i32)),
                       0);
           break;
         case MVT::i32:
-          break;
         case MVT::i64:
-          ClrNode =
-              SDValue(CurDAG->getMachineNode(
-                          TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
-                          CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
-                          CurDAG->getTargetConstant(X86::sub_32bit, dl,
-                                                    MVT::i32)),
-                      0);
           break;
         default:
           llvm_unreachable("Unexpected division source");
         }
 
         InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
                                       ClrNode, InFlag).getValue(1);
       }

llvm/trunk/lib/Target/X86/X86InstrCompiler.td

 
 //===----------------------------------------------------------------------===//
 // Alias Instructions
 //===----------------------------------------------------------------------===//
 
 // Alias instruction mapping movr0 to xor.
 // FIXME: remove when we can teach regalloc that xor reg, reg is ok.
 let Defs = [EFLAGS], isReMaterializable = 1, isAsCheapAsAMove = 1,
-    isPseudo = 1, AddedComplexity = 10 in
+    isPseudo = 1, AddedComplexity = 10 in {
 def MOV32r0  : I<0, Pseudo, (outs GR32:$dst), (ins), "",
                  [(set GR32:$dst, 0)]>, Sched<[WriteZero]>;
+def MOV64r0  : I<0, Pseudo, (outs GR64:$dst), (ins), "",
+                 [(set GR64:$dst, 0)]>, Sched<[WriteZero]>;
+}
 
 // Other widths can also make use of the 32-bit xor, which may have a smaller
 // encoding and avoid partial register updates.
 let AddedComplexity = 10 in {
 def : Pat<(i8 0), (EXTRACT_SUBREG (MOV32r0), sub_8bit)>;
 def : Pat<(i16 0), (EXTRACT_SUBREG (MOV32r0), sub_16bit)>;
-def : Pat<(i64 0), (SUBREG_TO_REG (i64 0), (MOV32r0), sub_32bit)>;
 }
 
 let Predicates = [OptForSize, Not64BitMode],
     AddedComplexity = 10 in {
   let SchedRW = [WriteALU] in {
   // Pseudo instructions for materializing 1 and -1 using XOR+INC/DEC,
   // which only require 3 bytes compared to MOV32ri which requires 5.
   let Defs = [EFLAGS], isReMaterializable = 1, isPseudo = 1 in {

llvm/trunk/lib/Target/X86/X86InstrInfo.cpp

if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS) {
       ClobbersEFLAGS = true;
       break;
     }
   }
 
   if (ClobbersEFLAGS && !isSafeToClobberEFLAGS(MBB, I)) {
     // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side
     // effects.
+    unsigned NewOpc = X86::MOV32ri;
     int Value;
     switch (Orig.getOpcode()) {
+    case X86::MOV64r0:  NewOpc = X86::MOV32ri64; Value = 0; break;
     case X86::MOV32r0:  Value = 0; break;
     case X86::MOV32r1:  Value = 1; break;
     case X86::MOV32r_1: Value = -1; break;
     default:
       llvm_unreachable("Unexpected instruction!");
     }
 
     const DebugLoc &DL = Orig.getDebugLoc();
-    BuildMI(MBB, I, DL, get(X86::MOV32ri))
+    BuildMI(MBB, I, DL, get(NewOpc))
         .add(Orig.getOperand(0))
         .addImm(Value);
   } else {
     MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
     MBB.insert(I, MI);
   }
 
   MachineInstr &NewMI = *std::prev(I);
for (; RI != RE; ++RI) {
 
     if (Instr.modifiesRegister(X86::EFLAGS, TRI) ||
         Instr.readsRegister(X86::EFLAGS, TRI)) {
       // This instruction modifies or uses EFLAGS.
 
       // MOV32r0 etc. are implemented with xor which clobbers condition code.
       // They are safe to move up, if the definition to EFLAGS is dead and
       // earlier instructions do not read or write EFLAGS.
-      if (!Movr0Inst && Instr.getOpcode() == X86::MOV32r0 &&
+      if (!Movr0Inst &&
+          (Instr.getOpcode() == X86::MOV32r0 ||
+           Instr.getOpcode() == X86::MOV64r0) &&
           Instr.registerDefIsDead(X86::EFLAGS, TRI)) {
         Movr0Inst = &Instr;
         continue;
       }
 
       // We can't remove CmpInstr.
       return false;
     }
static bool expandNOVLXStore(MachineInstrBuilder &MIB,
   return true;
 }
 bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   bool HasAVX = Subtarget.hasAVX();
   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
   switch (MI.getOpcode()) {
   case X86::MOV32r0:
     return Expand2AddrUndef(MIB, get(X86::XOR32rr));
+  case X86::MOV64r0: {
+    const TargetRegisterInfo *TRI = &getRegisterInfo();
+    unsigned Reg = MIB->getOperand(0).getReg();
+    unsigned Reg32 = TRI->getSubReg(Reg, X86::sub_32bit);
+    MIB->getOperand(0).setReg(Reg32);
+    Expand2AddrUndef(MIB, get(X86::XOR32rr));
+    MIB.addReg(Reg, RegState::ImplicitDefine);
+    return true;
+  }
   case X86::MOV32r1:
     return expandMOV32r1(MIB, *this, /*MinusOne=*/ false);
   case X86::MOV32r_1:
     return expandMOV32r1(MIB, *this, /*MinusOne=*/ true);
   case X86::MOV32ImmSExti8:
   case X86::MOV64ImmSExti8:
     return ExpandMOVImmSExti8(MIB, *this, Subtarget);
   case X86::SETB_C8r:
MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
   // replacing the *two* registers with the memory location.
   if (isTwoAddr && NumOps >= 2 && OpNum < 2 && MI.getOperand(0).isReg() &&
       MI.getOperand(1).isReg() &&
       MI.getOperand(0).getReg() == MI.getOperand(1).getReg()) {
     I = lookupTwoAddrFoldTable(MI.getOpcode());
     isTwoAddrFold = true;
   } else {
     if (OpNum == 0) {
-      if (MI.getOpcode() == X86::MOV32r0) {
-        NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, InsertPt, MI);
+      if (MI.getOpcode() == X86::MOV32r0 || MI.getOpcode() == X86::MOV64r0) {
+        unsigned NewOpc = MI.getOpcode() == X86::MOV64r0 ? X86::MOV64mi32
+                                                         : X86::MOV32mi;
+        NewMI = MakeM0Inst(*this, NewOpc, MOs, InsertPt, MI);
         if (NewMI)
           return NewMI;
       }
     }
 
     I = lookupFoldTable(MI.getOpcode(), OpNum);
   }
 

llvm/trunk/lib/Target/X86/X86SpeculativeLoadHardening.cpp

bool X86SpeculativeLoadHardeningPass::runOnMachineFunction(
   if (HardenInterprocedurally && !FenceCallAndRet) {
     // Set up the predicate state by extracting it from the incoming stack
     // pointer so we pick up any misspeculation in our caller.
     PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc);
   } else {
     // Otherwise, just build the predicate state itself by zeroing a register
     // as we don't need any initial state.
     PS->InitialReg = MRI->createVirtualRegister(PS->RC);
-    unsigned PredStateSubReg = MRI->createVirtualRegister(&X86::GR32RegClass);
-    auto ZeroI = BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV32r0),
-                         PredStateSubReg);
+    auto ZeroI = BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV64r0),
+                         PS->InitialReg);
     ++NumInstsInserted;
     MachineOperand *ZeroEFLAGSDefOp =
         ZeroI->findRegisterDefOperand(X86::EFLAGS);
     assert(ZeroEFLAGSDefOp && ZeroEFLAGSDefOp->isImplicit() &&
            "Must have an implicit def of EFLAGS!");
     ZeroEFLAGSDefOp->setIsDead(true);
-    BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::SUBREG_TO_REG),
-            PS->InitialReg)
-        .addImm(0)
-        .addReg(PredStateSubReg)
-        .addImm(X86::sub_32bit);
   }
 
   // We're going to need to trace predicate state throughout the function's
   // CFG. Prepare for this by setting up our initial state of PHIs with unique
   // predecessor entries and all the initial predicate state.
   canonicalizePHIOperands(MF);
 
   // Track the updated values in an SSA updater to rewrite into SSA form at the

llvm/trunk/test/CodeGen/X86/GlobalISel/constant.ll

 ; ALL-NEXT:    movq $-1, %rax
 ; ALL-NEXT:    retq
   ret i64 -1
 }
 
 define void @main(i32 ** %data) {
 ; ALL-LABEL: main:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    movq $0, %rax
+; ALL-NEXT:    xorl %eax, %eax
 ; ALL-NEXT:    movq %rax, (%rdi)
 ; ALL-NEXT:    retq
   store i32* null, i32** %data, align 8
   ret void
 }
 

llvm/trunk/test/CodeGen/X86/avg.ll

 ; AVX1-LABEL: not_avg_v16i8_wide_constants:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    pushq %r15
 ; AVX1-NEXT:    pushq %r14
 ; AVX1-NEXT:    pushq %r13
 ; AVX1-NEXT:    pushq %r12
 ; AVX1-NEXT:    pushq %rbx
-; AVX1-NEXT:    subq $24, %rsp
+; AVX1-NEXT:    subq $16, %rsp
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
 ; AVX1-NEXT:    vpextrq $1, %xmm5, %rbx
 ; AVX1-NEXT:    vmovq %xmm5, %rbp
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
-; AVX1-NEXT:    vpextrq $1, %xmm4, %rsi
-; AVX1-NEXT:    vmovq %xmm4, %rcx
+; AVX1-NEXT:    vpextrq $1, %xmm4, %rcx
+; AVX1-NEXT:    vmovq %xmm4, %rsi
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpextrq $1, %xmm4, %r8
+; AVX1-NEXT:    vpextrq $1, %xmm4, %r10
 ; AVX1-NEXT:    vmovq %xmm4, %r11
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
 ; AVX1-NEXT:    vpextrq $1, %xmm3, %r13
 ; AVX1-NEXT:    vmovq %xmm3, %r12
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero
 ; AVX1-NEXT:    vpextrq $1, %xmm4, %r15
-; AVX1-NEXT:    vmovq %xmm4, %rdi
+; AVX1-NEXT:    vmovq %xmm4, %rdx
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
 ; AVX1-NEXT:    vpextrq $1, %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX1-NEXT:    vmovq %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
 ; AVX1-NEXT:    vpextrq $1, %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX1-NEXT:    vmovq %xmm3, %r10
+; AVX1-NEXT:    vmovq %xmm3, %r9
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpextrq $1, %xmm4, %rdx
-; AVX1-NEXT:    addq %rbx, %rdx
-; AVX1-NEXT:    vmovq %xmm4, %r9
-; AVX1-NEXT:    addq %rbp, %r9
+; AVX1-NEXT:    vpextrq $1, %xmm4, %r14
+; AVX1-NEXT:    addq %rbx, %r14
+; AVX1-NEXT:    vmovq %xmm4, %r8
+; AVX1-NEXT:    addq %rbp, %r8
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX1-NEXT:    vpextrq $1, %xmm3, %rdi
+; AVX1-NEXT:    addq %rcx, %rdi
+; AVX1-NEXT:    vmovq %xmm3, %rax
 ; AVX1-NEXT:    addq %rsi, %rax
-; AVX1-NEXT:    movq %rax, %r14
-; AVX1-NEXT:    vmovq %xmm3, %rbp
-; AVX1-NEXT:    addq %rcx, %rbp
+; AVX1-NEXT:    movq %rax, %rsi
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
-; AVX1-NEXT:    vpextrq $1, %xmm3, %rsi
-; AVX1-NEXT:    addq %r8, %rsi
+; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
+; AVX1-NEXT:    addq %r10, %rax
+; AVX1-NEXT:    movq %rax, %r10
 ; AVX1-NEXT:    vmovq %xmm3, %rax
 ; AVX1-NEXT:    addq %r11, %rax
 ; AVX1-NEXT:    movq %rax, %r11
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
 ; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
 ; AVX1-NEXT:    addq %r13, %rax
-; AVX1-NEXT:    movq %rax, %rcx
+; AVX1-NEXT:    movq %rax, %rbx
 ; AVX1-NEXT:    vmovq %xmm2, %rax
 ; AVX1-NEXT:    addq %r12, %rax
-; AVX1-NEXT:    movq %rax, %r8
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX1-NEXT:    vpextrq $1, %xmm3, %rax
 ; AVX1-NEXT:    addq %r15, %rax
-; AVX1-NEXT:    movq %rax, %rbx
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    vmovq %xmm3, %rax
-; AVX1-NEXT:    addq %rdi, %rax
+; AVX1-NEXT:    addq %rdx, %rax
 ; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
 ; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
 ; AVX1-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
 ; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    vmovq %xmm2, %rax
 ; AVX1-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
 ; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX1-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
-; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    vmovq %xmm2, %r12
-; AVX1-NEXT:    addq %r10, %r12
+; AVX1-NEXT:    vpextrq $1, %xmm2, %rbp
+; AVX1-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
+; AVX1-NEXT:    vmovq %xmm2, %r15
+; AVX1-NEXT:    addq %r9, %r15
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT:    vpextrq $1, %xmm0, %r10
-; AVX1-NEXT:    addq %rax, %r10
-; AVX1-NEXT:    vmovq %xmm1, %rax
-; AVX1-NEXT:    vmovq %xmm0, %rdi
-; AVX1-NEXT:    addq %rax, %rdi
-; AVX1-NEXT:    addq $-1, %rdx
-; AVX1-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    vpextrq $1, %xmm0, %r9
+; AVX1-NEXT:    addq %rax, %r9
+; AVX1-NEXT:    vmovq %xmm1, %rcx
+; AVX1-NEXT:    vmovq %xmm0, %rdx
+; AVX1-NEXT:    addq %rcx, %rdx
+; AVX1-NEXT:    addq $-1, %r14
+; AVX1-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    movl $0, %eax
 ; AVX1-NEXT:    adcq $-1, %rax
 ; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    addq $-1, %r9
-; AVX1-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    addq $-1, %r8
+; AVX1-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    movl $0, %eax
 ; AVX1-NEXT:    adcq $-1, %rax
-; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    addq $-1, %r14
-; AVX1-NEXT:    movq %r14, (%rsp) # 8-byte Spill
+; AVX1-NEXT:    movq %rax, (%rsp) # 8-byte Spill
+; AVX1-NEXT:    addq $-1, %rdi
+; AVX1-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    movl $0, %eax
 ; AVX1-NEXT:    adcq $-1, %rax
 ; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    addq $-1, %rbp
-; AVX1-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    addq $-1, %rsi
+; AVX1-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    movl $0, %eax
 ; AVX1-NEXT:    adcq $-1, %rax
 ; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    addq $-1, %rsi
-; AVX1-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    addq $-1, %r10
+; AVX1-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    movl $0, %eax
 ; AVX1-NEXT:    adcq $-1, %rax
 ; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    addq $-1, %r11
 ; AVX1-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    movl $0, %eax
 ; AVX1-NEXT:    adcq $-1, %rax
 ; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    addq $-1, %rcx
-; AVX1-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    movl $0, %ebp
-; AVX1-NEXT:    adcq $-1, %rbp
-; AVX1-NEXT:    addq $-1, %r8
-; AVX1-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    movl $0, %r15d
-; AVX1-NEXT:    adcq $-1, %r15
 ; AVX1-NEXT:    addq $-1, %rbx
 ; AVX1-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX1-NEXT:    movl $0, %eax
 ; AVX1-NEXT:    adcq $-1, %rax
 ; AVX1-NEXT:    movq %rax, %rsi
 ; AVX1-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX1-NEXT:    movl $0, %r12d
+; AVX1-NEXT:    adcq $-1, %r12
+; AVX1-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX1-NEXT:    movl $0, %r13d
 ; AVX1-NEXT:    adcq $-1, %r13
 ; AVX1-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX1-NEXT:    movl $0, %r14d
 ; AVX1-NEXT:    adcq $-1, %r14
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; AVX1-NEXT:    addq $-1, %rdx
-; AVX1-NEXT:    movl $0, %r11d
-; AVX1-NEXT:    adcq $-1, %r11
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX1-NEXT:    addq $-1, %rax
+; AVX1-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX1-NEXT:    movl $0, %ebx
 ; AVX1-NEXT:    adcq $-1, %rbx
-; AVX1-NEXT:    addq $-1, %r12
-; AVX1-NEXT:    movl $0, %r9d
-; AVX1-NEXT:    adcq $-1, %r9
-; AVX1-NEXT:    addq $-1, %r10
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX1-NEXT:    addq $-1, %rcx
+; AVX1-NEXT:    movl $0, %r11d
+; AVX1-NEXT:    adcq $-1, %r11
+; AVX1-NEXT:    addq $-1, %rbp
+; AVX1-NEXT:    movl $0, %r10d
+; AVX1-NEXT:    adcq $-1, %r10
+; AVX1-NEXT:    addq $-1, %r15
 ; AVX1-NEXT:    movl $0, %r8d
 ; AVX1-NEXT:    adcq $-1, %r8
-; AVX1-NEXT:    addq $-1, %rdi
-; AVX1-NEXT:    movl $0, %ecx
-; AVX1-NEXT:    adcq $-1, %rcx
-; AVX1-NEXT:    shldq $63, %rdi, %rcx
-; AVX1-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    shldq $63, %r10, %r8
-; AVX1-NEXT:    shldq $63, %r12, %r9
-; AVX1-NEXT:    shldq $63, %rax, %rbx
-; AVX1-NEXT:    shldq $63, %rdx, %r11
+; AVX1-NEXT:    addq $-1, %r9
+; AVX1-NEXT:    movl $0, %edi
+; AVX1-NEXT:    adcq $-1, %rdi
+; AVX1-NEXT:    addq $-1, %rdx
+; AVX1-NEXT:    movl $0, %eax
+; AVX1-NEXT:    adcq $-1, %rax
+; AVX1-NEXT:    shldq $63, %rdx, %rax
+; AVX1-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT:    shldq $63, %r9, %rdi
+; AVX1-NEXT:    shldq $63, %r15, %r8
+; AVX1-NEXT:    shldq $63, %rbp, %r10
+; AVX1-NEXT:    shldq $63, %rcx, %r11
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX1-NEXT:    shldq $63, %rdx, %rbx
 ; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
 ; AVX1-NEXT:    shldq $63, %rdx, %r14
 ; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
 ; AVX1-NEXT:    shldq $63, %rdx, %r13
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX1-NEXT:    shldq $63, %rdx, %r12
 ; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX1-NEXT:    shldq $63, %rax, %rsi
 ; AVX1-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX1-NEXT:    shldq $63, %rax, %r15
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX1-NEXT:    shldq $63, %rax, %rbp
 ; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
 ; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX1-NEXT:    shldq $63, %rax, %rsi
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX1-NEXT:    shldq $63, %rax, %rcx
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
 ; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX1-NEXT:    shldq $63, %rax, %rdi
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; AVX1-NEXT:    movq (%rsp), %rax # 8-byte Reload
-; AVX1-NEXT:    shldq $63, %rax, %r12
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX1-NEXT:    shldq $63, %rax, %r15
 ; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX1-NEXT:    shldq $63, %rax, %r10
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX1-NEXT:    shldq $63, %rcx, %rax
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX1-NEXT:    shldq $63, %rcx, %r9
+; AVX1-NEXT:    movq (%rsp), %rcx # 8-byte Reload
 ; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX1-NEXT:    shldq $63, %rdx, %rax
-; AVX1-NEXT:    vmovq %rax, %xmm8
-; AVX1-NEXT:    vmovq %r10, %xmm0
-; AVX1-NEXT:    vmovq %r12, %xmm1
-; AVX1-NEXT:    vmovq %rdi, %xmm11
-; AVX1-NEXT:    vmovq %rcx, %xmm2
+; AVX1-NEXT:    shldq $63, %rdx, %rcx
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX1-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; AVX1-NEXT:    shldq $63, %rbp, %rdx
+; AVX1-NEXT:    vmovq %rdx, %xmm8
+; AVX1-NEXT:    vmovq %rcx, %xmm0
+; AVX1-NEXT:    vmovq %r9, %xmm1
+; AVX1-NEXT:    vmovq %rax, %xmm11
+; AVX1-NEXT:    vmovq %r15, %xmm2
 ; AVX1-NEXT:    vmovq %rsi, %xmm13
-; AVX1-NEXT:    vmovq %rbp, %xmm14
-; AVX1-NEXT:    vmovq %r15, %xmm15
-; AVX1-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 8-byte Folded Reload
-; AVX1-NEXT:    # xmm9 = mem[0],zero
-; AVX1-NEXT:    vmovq %r13, %xmm10
-; AVX1-NEXT:    vmovq %r14, %xmm12
+; AVX1-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 8-byte Folded Reload
+; AVX1-NEXT:    # xmm14 = mem[0],zero
+; AVX1-NEXT:    vmovq %r12, %xmm15
+; AVX1-NEXT:    vmovq %r13, %xmm9
+; AVX1-NEXT:    vmovq %r14, %xmm10
+; AVX1-NEXT:    vmovq %rbx, %xmm12
 ; AVX1-NEXT:    vmovq %r11, %xmm3
-; AVX1-NEXT:    vmovq %rbx, %xmm4
-; AVX1-NEXT:    vmovq %r9, %xmm5
-; AVX1-NEXT:    vmovq %r8, %xmm6
+; AVX1-NEXT:    vmovq %r10, %xmm4
+; AVX1-NEXT:    vmovq %r8, %xmm5
+; AVX1-NEXT:    vmovq %rdi, %xmm6
 ; AVX1-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 8-byte Folded Reload
 ; AVX1-NEXT:    # xmm7 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm0[0],xmm8[0]
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm11[0],xmm1[0]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm8 = xmm8[0,2],xmm0[0,2]
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm13[0],xmm2[0]
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm15[0],xmm14[0]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm11 = xmm0[0,2],xmm1[0,2]

 ; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm7[0],xmm6[0]
 ; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,2],xmm5[0,2]
 ; AVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX1-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX1-NEXT:    vmovdqu %xmm0, (%rax)
-; AVX1-NEXT:    addq $24, %rsp
+; AVX1-NEXT:    addq $16, %rsp
 ; AVX1-NEXT:    popq %rbx
 ; AVX1-NEXT:    popq %r12
 ; AVX1-NEXT:    popq %r13
 ; AVX1-NEXT:    popq %r14
 ; AVX1-NEXT:    popq %r15
 ; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: not_avg_v16i8_wide_constants:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    pushq %rbp
 ; AVX2-NEXT:    pushq %r15
 ; AVX2-NEXT:    pushq %r14
 ; AVX2-NEXT:    pushq %r13
 ; AVX2-NEXT:    pushq %r12
 ; AVX2-NEXT:    pushq %rbx
 ; AVX2-NEXT:    subq $16, %rsp
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
 ; AVX2-NEXT:    vpextrq $1, %xmm4, %rbx
-; AVX2-NEXT:    vmovq %xmm4, %rbp
+; AVX2-NEXT:    vmovq %xmm4, %rdx
 ; AVX2-NEXT:    vpextrq $1, %xmm3, %rdi
 ; AVX2-NEXT:    vmovq %xmm3, %rcx
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm2
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX2-NEXT:    vpextrq $1, %xmm3, %rdx
-; AVX2-NEXT:    vmovq %xmm3, %r9
-; AVX2-NEXT:    vpextrq $1, %xmm2, %r11
+; AVX2-NEXT:    vpextrq $1, %xmm3, %r9
+; AVX2-NEXT:    vmovq %xmm3, %r10
+; AVX2-NEXT:    vpextrq $1, %xmm2, %r13
 ; AVX2-NEXT:    vmovq %xmm2, %r12
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX2-NEXT:    vpextrq $1, %xmm3, %r15
 ; AVX2-NEXT:    vmovq %xmm3, %rsi
 ; AVX2-NEXT:    vpextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX2-NEXT:    vmovq %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX2-NEXT:    vmovq %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
-; AVX2-NEXT:    addq %rbx, %rax
-; AVX2-NEXT:    movq %rax, %rbx
-; AVX2-NEXT:    vmovq %xmm4, %r13
-; AVX2-NEXT:    addq %rbp, %r13
-; AVX2-NEXT:    vpextrq $1, %xmm3, %r10
-; AVX2-NEXT:    addq %rdi, %r10
+; AVX2-NEXT:    vpextrq $1, %xmm4, %rbp
+; AVX2-NEXT:    addq %rbx, %rbp
+; AVX2-NEXT:    vmovq %xmm4, %rax
+; AVX2-NEXT:    addq %rdx, %rax
+; AVX2-NEXT:    movq %rax, %r11
+; AVX2-NEXT:    vpextrq $1, %xmm3, %r8
+; AVX2-NEXT:    addq %rdi, %r8
 ; AVX2-NEXT:    vmovq %xmm3, %r14
 ; AVX2-NEXT:    addq %rcx, %r14
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm2
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX2-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX2-NEXT:    addq %rdx, %rax
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    vmovq %xmm3, %r8
-; AVX2-NEXT:    addq %r9, %r8
-; AVX2-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX2-NEXT:    addq %r11, %rax
-; AVX2-NEXT:    movq %rax, %r11
+; AVX2-NEXT:    addq %r9, %rax
+; AVX2-NEXT:    movq %rax, %rbx
+; AVX2-NEXT:    vmovq %xmm3, %rax
+; AVX2-NEXT:    addq %r10, %rax
+; AVX2-NEXT:    movq %rax, %r10
+; AVX2-NEXT:    vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT:    addq %r13, %rcx
 ; AVX2-NEXT:    vmovq %xmm2, %rax
 ; AVX2-NEXT:    addq %r12, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX2-NEXT:    vpextrq $1, %xmm3, %rax
 ; AVX2-NEXT:    addq %r15, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vmovq %xmm3, %rax
 ; AVX2-NEXT:    addq %rsi, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrq $1, %xmm2, %rax
 ; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vmovq %xmm2, %rax
 ; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpextrq $1, %xmm2, %rbp
-; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
+; AVX2-NEXT:    vpextrq $1, %xmm2, %r12
+; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
 ; AVX2-NEXT:    vmovq %xmm2, %r9
 ; AVX2-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rdi
 ; AVX2-NEXT:    addq %rax, %rdi
 ; AVX2-NEXT:    vmovq %xmm1, %rdx
 ; AVX2-NEXT:    vmovq %xmm0, %rsi
 ; AVX2-NEXT:    addq %rdx, %rsi
-; AVX2-NEXT:    addq $-1, %rbx
-; AVX2-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %rbp
+; AVX2-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    movl $0, %eax
 ; AVX2-NEXT:    adcq $-1, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    addq $-1, %r13
-; AVX2-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %r11
+; AVX2-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    movl $0, %eax
 ; AVX2-NEXT:    adcq $-1, %rax
 ; AVX2-NEXT:    movq %rax, (%rsp) # 8-byte Spill
-; AVX2-NEXT:    addq $-1, %r10
-; AVX2-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %r8
+; AVX2-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    movl $0, %eax
 ; AVX2-NEXT:    adcq $-1, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    addq $-1, %r14
 ; AVX2-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    movl $0, %r13d
-; AVX2-NEXT:    adcq $-1, %r13
-; AVX2-NEXT:    addq $-1, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    movl $0, %ebp
+; AVX2-NEXT:    adcq $-1, %rbp
+; AVX2-NEXT:    addq $-1, %rbx
+; AVX2-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    movl $0, %eax
 ; AVX2-NEXT:    adcq $-1, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    addq $-1, %r8
-; AVX2-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %r10
+; AVX2-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    movl $0, %r15d
 ; AVX2-NEXT:    adcq $-1, %r15
-; AVX2-NEXT:    addq $-1, %r11
-; AVX2-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq $-1, %rcx
+; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    movl $0, %ebx
 ; AVX2-NEXT:    adcq $-1, %rbx
 ; AVX2-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX2-NEXT:    movl $0, %r8d
 ; AVX2-NEXT:    adcq $-1, %r8
 ; AVX2-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX2-NEXT:    movl $0, %eax
 ; AVX2-NEXT:    adcq $-1, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX2-NEXT:    movl $0, %eax
 ; AVX2-NEXT:    adcq $-1, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX2-NEXT:    movl $0, %r12d
-; AVX2-NEXT:    adcq $-1, %r12
+; AVX2-NEXT:    movl $0, %r13d
+; AVX2-NEXT:    adcq $-1, %r13
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX2-NEXT:    addq $-1, %rcx
 ; AVX2-NEXT:    movl $0, %r11d
 ; AVX2-NEXT:    adcq $-1, %r11
-; AVX2-NEXT:    addq $-1, %rbp
+; AVX2-NEXT:    addq $-1, %r12
 ; AVX2-NEXT:    movl $0, %r14d
 ; AVX2-NEXT:    adcq $-1, %r14
 ; AVX2-NEXT:    addq $-1, %r9
 ; AVX2-NEXT:    movl $0, %r10d
 ; AVX2-NEXT:    adcq $-1, %r10
 ; AVX2-NEXT:    addq $-1, %rdi
 ; AVX2-NEXT:    movl $0, %edx
 ; AVX2-NEXT:    adcq $-1, %rdx
 ; AVX2-NEXT:    addq $-1, %rsi
 ; AVX2-NEXT:    movl $0, %eax
 ; AVX2-NEXT:    adcq $-1, %rax
 ; AVX2-NEXT:    shldq $63, %rsi, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    shldq $63, %rdi, %rdx
 ; AVX2-NEXT:    shldq $63, %r9, %r10
-; AVX2-NEXT:    shldq $63, %rbp, %r14
+; AVX2-NEXT:    shldq $63, %r12, %r14
 ; AVX2-NEXT:    shldq $63, %rcx, %r11
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX2-NEXT:    shldq $63, %rcx, %r12
+; AVX2-NEXT:    shldq $63, %rcx, %r13
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rcx, %r9
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rcx, %r8
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rax, %rbx
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rax, %r15
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rcx, %rax
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX2-NEXT:    shldq $63, %rcx, %r13
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rcx, %rbp
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT:    shldq $63, %rcx, %r12
 ; AVX2-NEXT:    movq (%rsp), %rdi # 8-byte Reload
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rcx, %rdi
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
 ; AVX2-NEXT:    shldq $63, %rcx, %rsi
 ; AVX2-NEXT:    vmovq %rsi, %xmm8
 ; AVX2-NEXT:    vmovq %rdi, %xmm9
-; AVX2-NEXT:    vmovq %rbp, %xmm10
-; AVX2-NEXT:    vmovq %r13, %xmm11
+; AVX2-NEXT:    vmovq %r12, %xmm10
+; AVX2-NEXT:    vmovq %rbp, %xmm11
 ; AVX2-NEXT:    vmovq %rax, %xmm12
 ; AVX2-NEXT:    vmovq %r15, %xmm13
 ; AVX2-NEXT:    vmovq %rbx, %xmm14
 ; AVX2-NEXT:    vmovq %r8, %xmm15
 ; AVX2-NEXT:    vmovq %r9, %xmm0
 ; AVX2-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
 ; AVX2-NEXT:    # xmm1 = mem[0],zero
-; AVX2-NEXT:    vmovq %r12, %xmm2
+; AVX2-NEXT:    vmovq %r13, %xmm2
 ; AVX2-NEXT:    vmovq %r11, %xmm3
 ; AVX2-NEXT:    vmovq %r14, %xmm4
 ; AVX2-NEXT:    vmovq %r10, %xmm5
 ; AVX2-NEXT:    vmovq %rdx, %xmm6
 ; AVX2-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 8-byte Folded Reload
 ; AVX2-NEXT:    # xmm7 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm9[0],xmm8[0]
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm9 = xmm11[0],xmm10[0]

 ; AVX512-LABEL: not_avg_v16i8_wide_constants:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    pushq %rbp
 ; AVX512-NEXT:    pushq %r15
 ; AVX512-NEXT:    pushq %r14
 ; AVX512-NEXT:    pushq %r13
 ; AVX512-NEXT:    pushq %r12
 ; AVX512-NEXT:    pushq %rbx
-; AVX512-NEXT:    subq $24, %rsp
+; AVX512-NEXT:    subq $16, %rsp
 ; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm3, %xmm4
 ; AVX512-NEXT:    vpextrq $1, %xmm4, %rbx
 ; AVX512-NEXT:    vmovq %xmm4, %rbp
 ; AVX512-NEXT:    vpextrq $1, %xmm3, %rdi
 ; AVX512-NEXT:    vmovq %xmm3, %rsi
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rdx
-; AVX512-NEXT:    vmovq %xmm3, %r8
+; AVX512-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX512-NEXT:    vmovq %xmm3, %r10
 ; AVX512-NEXT:    vpextrq $1, %xmm2, %r13
 ; AVX512-NEXT:    vmovq %xmm2, %r12
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512-NEXT:    vpextrq $1, %xmm3, %r15
+; AVX512-NEXT:    vpextrq $1, %xmm3, %rdx
 ; AVX512-NEXT:    vmovq %xmm3, %r14
 ; AVX512-NEXT:    vpextrq $1, %xmm2, %r9
 ; AVX512-NEXT:    vmovq %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512-NEXT:    vpextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX512-NEXT:    vmovq %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; AVX512-NEXT:    vpextrq $1, %xmm4, %rax
-; AVX512-NEXT:    addq %rbx, %rax
-; AVX512-NEXT:    movq %rax, %rbx
+; AVX512-NEXT:    vpextrq $1, %xmm4, %r11
+; AVX512-NEXT:    addq %rbx, %r11
 ; AVX512-NEXT:    vmovq %xmm4, %rax
 ; AVX512-NEXT:    addq %rbp, %rax
-; AVX512-NEXT:    movq %rax, %rbp
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX512-NEXT:    addq %rdi, %rax
-; AVX512-NEXT:    movq %rax, %rdi
-; AVX512-NEXT:    vmovq %xmm3, %r10
-; AVX512-NEXT:    addq %rsi, %r10
+; AVX512-NEXT:    movq %rax, %rbx
+; AVX512-NEXT:    vpextrq $1, %xmm3, %r8
+; AVX512-NEXT:    addq %rdi, %r8
+; AVX512-NEXT:    vmovq %xmm3, %r15
+; AVX512-NEXT:    addq %rsi, %r15
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX512-NEXT:    addq %rdx, %rcx
+; AVX512-NEXT:    vpextrq $1, %xmm3, %rdi
+; AVX512-NEXT:    addq %rcx, %rdi
 ; AVX512-NEXT:    vmovq %xmm3, %rax
-; AVX512-NEXT:    addq %r8, %rax
-; AVX512-NEXT:    movq %rax, %r8
+; AVX512-NEXT:    addq %r10, %rax
+; AVX512-NEXT:    movq %rax, %r10
 ; AVX512-NEXT:    vpextrq $1, %xmm2, %rsi
 ; AVX512-NEXT:    addq %r13, %rsi
-; AVX512-NEXT:    vmovq %xmm2, %r11
-; AVX512-NEXT:    addq %r12, %r11
+; AVX512-NEXT:    vmovq %xmm2, %rax
+; AVX512-NEXT:    addq %r12, %rax
+; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512-NEXT:    vpextrq $1, %xmm3, %rax
-; AVX512-NEXT:    addq %r15, %rax
+; AVX512-NEXT:    addq %rdx, %rax
 ; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    vmovq %xmm3, %rax
 ; AVX512-NEXT:    addq %r14, %rax
 ; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    vpextrq $1, %xmm2, %rax
 ; AVX512-NEXT:    addq %r9, %rax
 ; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    vmovq %xmm2, %rax
 ; AVX512-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
 ; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512-NEXT:    vpextrq $1, %xmm2, %rax
-; AVX512-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
-; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT:    vpextrq $1, %xmm2, %rbp
+; AVX512-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
 ; AVX512-NEXT:    vmovq %xmm2, %r14
 ; AVX512-NEXT:    addq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
 ; AVX512-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX512-NEXT:    vpextrq $1, %xmm1, %r9
 ; AVX512-NEXT:    addq %rax, %r9
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovq %xmm0, %rcx
 ; AVX512-NEXT:    vmovq %xmm1, %rdx
-; AVX512-NEXT:    addq %rax, %rdx
+; AVX512-NEXT:    addq %rcx, %rdx
+; AVX512-NEXT:    addq $-1, %r11
+; AVX512-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT:    movl $0, %eax
+; AVX512-NEXT:    adcq $-1, %rax
+; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    addq $-1, %rbx
 ; AVX512-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    movl $0, %eax
 ; AVX512-NEXT:    adcq $-1, %rax
+; AVX512-NEXT:    movq %rax, (%rsp) # 8-byte Spill
+; AVX512-NEXT:    addq $-1, %r8
+; AVX512-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT:    movl $0, %eax
+; AVX512-NEXT:    adcq $-1, %rax
 ; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT:    addq $-1, %rbp
-; AVX512-NEXT:    movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT:    addq $-1, %r15
+; AVX512-NEXT:    movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    movl $0, %eax
 ; AVX512-NEXT:    adcq $-1, %rax
 ; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    addq $-1, %rdi
 ; AVX512-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    movl $0, %eax
 ; AVX512-NEXT:    adcq $-1, %rax
-; AVX512-NEXT:    movq %rax, (%rsp) # 8-byte Spill
+; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    addq $-1, %r10
 ; AVX512-NEXT:    movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    movl $0, %eax
 ; AVX512-NEXT:    adcq $-1, %rax
-; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT:    addq $-1, %rcx
-; AVX512-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT:    movl $0, %eax
-; AVX512-NEXT:    adcq $-1, %rax
-; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT:    addq $-1, %r8
-; AVX512-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT:    movl $0, %eax
-; AVX512-NEXT:    adcq $-1, %rax
-; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT:    movq %rax, %rcx
 ; AVX512-NEXT:    addq $-1, %rsi
 ; AVX512-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT:    movl $0, %r13d
-; AVX512-NEXT:    adcq $-1, %r13
-; AVX512-NEXT:    addq $-1, %r11
-; AVX512-NEXT:    movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT:    movl $0, %r15d
-; AVX512-NEXT:    adcq $-1, %r15
-; AVX512-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
-; AVX512-NEXT:    movl $0, %eax
-; AVX512-NEXT:    adcq $-1, %rax
-; AVX512-NEXT:    movq %rax, %rsi
-; AVX512-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX512-NEXT:    movl $0, %r12d
 ; AVX512-NEXT:    adcq $-1, %r12
 ; AVX512-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX512-NEXT:    movl $0, %ebx
 ; AVX512-NEXT:    adcq $-1, %rbx
-; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; AVX512-NEXT:    addq $-1, %rbp
+; AVX512-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT:    movl $0, %r13d
+; AVX512-NEXT:    adcq $-1, %r13
+; AVX512-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT:    movl $0, %r15d
+; AVX512-NEXT:    adcq $-1, %r15
+; AVX512-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; AVX512-NEXT:    movl $0, %r11d
 ; AVX512-NEXT:    adcq $-1, %r11
-; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX512-NEXT:    addq $-1, %rax
+; AVX512-NEXT:    addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT:    movl $0, %r8d
+; AVX512-NEXT:    adcq $-1, %r8
+; AVX512-NEXT:    addq $-1, %rbp
 ; AVX512-NEXT:    movl $0, %r10d
 ; AVX512-NEXT:    adcq $-1, %r10
 ; AVX512-NEXT:    addq $-1, %r14
-; AVX512-NEXT:    movl $0, %r8d
-; AVX512-NEXT:    adcq $-1, %r8
-; AVX512-NEXT:    addq $-1, %r9
 ; AVX512-NEXT:    movl $0, %edi
 ; AVX512-NEXT:    adcq $-1, %rdi
+; AVX512-NEXT:    addq $-1, %r9
+; AVX512-NEXT:    movl $0, %esi
+; AVX512-NEXT:    adcq $-1, %rsi
 ; AVX512-NEXT:    addq $-1, %rdx
-; AVX512-NEXT:    movl $0, %ecx
-; AVX512-NEXT:    adcq $-1, %rcx
-; AVX512-NEXT:    shldq $63, %rdx, %rcx
-; AVX512-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT:    shldq $63, %r9, %rdi
-; AVX512-NEXT:    shldq $63, %r14, %r8
-; AVX512-NEXT:    shldq $63, %rax, %r10
-; AVX512-NEXT:    shldq $63, %rbp, %r11
+; AVX512-NEXT:    movl $0, %eax
+; AVX512-NEXT:    adcq $-1, %rax
+; AVX512-NEXT:    shldq $63, %rdx, %rax
+; AVX512-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT:    shldq $63, %r9, %rsi
+; AVX512-NEXT:    shldq $63, %r14, %rdi
+; AVX512-NEXT:    shldq $63, %rbp, %r10
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; AVX512-NEXT:    shldq $63, %rdx, %rbx
+; AVX512-NEXT:    shldq $63, %rdx, %r8
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; AVX512-NEXT:    shldq $63, %rdx, %r12
+; AVX512-NEXT:    shldq $63, %rdx, %r11
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; AVX512-NEXT:    shldq $63, %rdx, %rsi
-; AVX512-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT:    shldq $63, %rdx, %r15
+; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX512-NEXT:    shldq $63, %rdx, %r13
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX512-NEXT:    shldq $63, %rax, %r15
+; AVX512-NEXT:    shldq $63, %rax, %rbx
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX512-NEXT:    shldq $63, %rax, %r13
-; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX512-NEXT:    shldq $63, %rax, %r12
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX512-NEXT:    shldq $63, %rax, %rsi
+; AVX512-NEXT:    shldq $63, %rax, %rcx
+; AVX512-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX512-NEXT:    shldq $63, %rax, %rcx
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
 ; AVX512-NEXT:    shldq $63, %rdx, %rax
-; AVX512-NEXT:    movq (%rsp), %r14 # 8-byte Reload
+; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
 ; AVX512-NEXT:    shldq $63, %rdx, %r14
-; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; AVX512-NEXT:    movq (%rsp), %r9 # 8-byte Reload
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
 ; AVX512-NEXT:    shldq $63, %rdx, %r9
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
 ; AVX512-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; AVX512-NEXT:    shldq $63, %rdx, %rbp
-; AVX512-NEXT:    vmovq %rbp, %xmm8
+; AVX512-NEXT:    shldq $63, %rbp, %rdx
+; AVX512-NEXT:    vmovq %rdx, %xmm8
 ; AVX512-NEXT:    vmovq %r9, %xmm9
 ; AVX512-NEXT:    vmovq %r14, %xmm10
 ; AVX512-NEXT:    vmovq %rax, %xmm11
 ; AVX512-NEXT:    vmovq %rcx, %xmm12
-; AVX512-NEXT:    vmovq %rsi, %xmm13
-; AVX512-NEXT:    vmovq %r13, %xmm14
-; AVX512-NEXT:    vmovq %r15, %xmm15
-; AVX512-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
-; AVX512-NEXT:    # xmm0 = mem[0],zero
-; AVX512-NEXT:    vmovq %r12, %xmm1
-; AVX512-NEXT:    vmovq %rbx, %xmm2
-; AVX512-NEXT:    vmovq %r11, %xmm3
+; AVX512-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 8-byte Folded Reload
+; AVX512-NEXT:    # xmm13 = mem[0],zero
+; AVX512-NEXT:    vmovq %r12, %xmm14
+; AVX512-NEXT:    vmovq %rbx, %xmm15
+; AVX512-NEXT:    vmovq %r13, %xmm0
+; AVX512-NEXT:    vmovq %r15, %xmm1
+; AVX512-NEXT:    vmovq %r11, %xmm2
+; AVX512-NEXT:    vmovq %r8, %xmm3
 ; AVX512-NEXT:    vmovq %r10, %xmm4
-; AVX512-NEXT:    vmovq %r8, %xmm5
-; AVX512-NEXT:    vmovq %rdi, %xmm6
+; AVX512-NEXT:    vmovq %rdi, %xmm5
+; AVX512-NEXT:    vmovq %rsi, %xmm6
 ; AVX512-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 8-byte Folded Reload
 ; AVX512-NEXT:    # xmm7 = mem[0],zero
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm9[0],xmm8[0]
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm9 = xmm11[0],xmm10[0]
 ; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm9, %ymm8
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm9 = xmm13[0],xmm12[0]
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm15[0],xmm14[0]
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm9
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm9, %zmm8
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm2[0]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm5[0],xmm4[0]
 ; AVX512-NEXT:    vpmovqd %zmm8, %ymm2
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm7[0],xmm6[0]
 ; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm1
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vpmovqd %zmm0, %ymm0
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512-NEXT:    vpmovdb %zmm0, (%rax)
-; AVX512-NEXT:    addq $24, %rsp
+; AVX512-NEXT:    addq $16, %rsp
 ; AVX512-NEXT:    popq %rbx
 ; AVX512-NEXT:    popq %r12
 ; AVX512-NEXT:    popq %r13
 ; AVX512-NEXT:    popq %r14
 ; AVX512-NEXT:    popq %r15
 ; AVX512-NEXT:    popq %rbp
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq

llvm/trunk/test/CodeGen/X86/crash-O0.ll

 ; CHECK-LABEL: addressModeWith32bitIndex:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset %rbp, -16
 ; CHECK-NEXT:    movq %rsp, %rbp
 ; CHECK-NEXT:    .cfi_def_cfa_register %rbp
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    movl %eax, %ecx
-; CHECK-NEXT:    movq %rcx, %rax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-NEXT:    cqto
-; CHECK-NEXT:    movslq %edi, %rsi
-; CHECK-NEXT:    idivq (%rcx,%rsi,8)
+; CHECK-NEXT:    movslq %edi, %rcx
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi ## 8-byte Reload
+; CHECK-NEXT:    idivq (%rsi,%rcx,8)
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:    retq
   %gep = getelementptr i64, i64* null, i32 %V
   %load = load i64, i64* %gep
   %sdiv = sdiv i64 0, %load
   ret i64 %sdiv
 }

llvm/trunk/test/CodeGen/X86/hoist-spill.ll

 ; RUN: llc < %s | FileCheck %s
 
 ; Check no spills to the same stack slot after hoisting.
 ; CHECK: mov{{.}} %{{.*}}, [[SPOFFSET1:-?[0-9]*]](%rsp)
-; CHECK: mov{{.}} %{{.*}}, [[SPOFFSET2:-?[0-9]*]](%rsp)
 ; CHECK-NOT: mov{{.}} %{{.*}}, [[SPOFFSET1]](%rsp)
-; CHECK-NOT: mov{{.}} %{{.*}}, [[SPOFFSET2]](%rsp)
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @a = external global i32*, align 8
 @b = external global i32, align 4
 @d = external global i32*, align 8
 

llvm/trunk/test/CodeGen/X86/machine-cse.ll

return:
   %retval.0 = phi i32 [ 1, %entry ], [ %., %if.end ]
   ret i32 %retval.0
 }
 
 ; rdar://11393714
 define i8* @bsd_memchr(i8* %s, i32 %a, i32 %c, i64 %n) nounwind ssp {
 ; CHECK-LABEL: bsd_memchr:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testq %rcx, %rcx
-; CHECK-NEXT:    je .LBB3_4
+; CHECK-NEXT:    je .LBB3_5
 ; CHECK-NEXT:  # %bb.1: # %preheader
-; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    movzbl %dl, %edx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB3_2: # %do.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpl %edx, %esi
-; CHECK-NEXT:    je .LBB3_5
-; CHECK-NEXT:  # %bb.3: # %do.cond
+; CHECK-NEXT:    je .LBB3_3
+; CHECK-NEXT:  # %bb.4: # %do.cond
 ; CHECK-NEXT:    # in Loop: Header=BB3_2 Depth=1
-; CHECK-NEXT:    incq %rax
+; CHECK-NEXT:    incq %rdi
 ; CHECK-NEXT:    decq %rcx
 ; CHECK-NEXT:    jne .LBB3_2
-; CHECK-NEXT:  .LBB3_4:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:  .LBB3_5: # %return
 ; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB3_3:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    retq
 entry:
   %cmp = icmp eq i64 %n, 0
   br i1 %cmp, label %return, label %preheader
 
 preheader:
   %conv2 = and i32 %c, 255
   br label %do.body
 

llvm/trunk/test/CodeGen/X86/madd.ll

 define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture readonly, i32) local_unnamed_addr #0 {
 ; SSE2-LABEL: _Z10test_shortPsS_i_1024:
 ; SSE2:       # %bb.0: # %entry
 ; SSE2-NEXT:    movl %edx, %eax
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  .LBB3_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx,2), %xmm5
 ; SSE2-NEXT:    movdqu 16(%rdi,%rcx,2), %xmm6
 ; SSE2-NEXT:    movdqu 32(%rdi,%rcx,2), %xmm7
 ; SSE2-NEXT:    movdqu 48(%rdi,%rcx,2), %xmm9
-; SSE2-NEXT:    movdqu (%rsi,%rcx,2), %xmm0
-; SSE2-NEXT:    pmaddwd %xmm5, %xmm0
-; SSE2-NEXT:    paddd %xmm0, %xmm2
-; SSE2-NEXT:    movdqu 16(%rsi,%rcx,2), %xmm0
-; SSE2-NEXT:    pmaddwd %xmm6, %xmm0
-; SSE2-NEXT:    paddd %xmm0, %xmm4
-; SSE2-NEXT:    movdqu 32(%rsi,%rcx,2), %xmm0
-; SSE2-NEXT:    pmaddwd %xmm7, %xmm0
-; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    movdqu 48(%rsi,%rcx,2), %xmm0
-; SSE2-NEXT:    pmaddwd %xmm9, %xmm0
-; SSE2-NEXT:    paddd %xmm0, %xmm3
+; SSE2-NEXT:    movdqu (%rsi,%rcx,2), %xmm1
+; SSE2-NEXT:    pmaddwd %xmm5, %xmm1
+; SSE2-NEXT:    paddd %xmm1, %xmm2
+; SSE2-NEXT:    movdqu 16(%rsi,%rcx,2), %xmm1
+; SSE2-NEXT:    pmaddwd %xmm6, %xmm1
+; SSE2-NEXT:    paddd %xmm1, %xmm4
+; SSE2-NEXT:    movdqu 32(%rsi,%rcx,2), %xmm1
+; SSE2-NEXT:    pmaddwd %xmm7, %xmm1
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    movdqu 48(%rsi,%rcx,2), %xmm1
+; SSE2-NEXT:    pmaddwd %xmm9, %xmm1
+; SSE2-NEXT:    paddd %xmm1, %xmm3
 ; SSE2-NEXT:    addq $16, %rcx
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB3_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    paddd %xmm8, %xmm4
 ; SSE2-NEXT:    paddd %xmm8, %xmm3
 ; SSE2-NEXT:    paddd %xmm4, %xmm3
 ; SSE2-NEXT:    paddd %xmm8, %xmm2
-; SSE2-NEXT:    paddd %xmm8, %xmm1
-; SSE2-NEXT:    paddd %xmm3, %xmm1
-; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm8, %xmm0
+; SSE2-NEXT:    paddd %xmm3, %xmm0
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
 ; AVX1-LABEL: _Z10test_shortPsS_i_1024:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    movl %edx, %eax
 ; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
 ; AVX1-NEXT:    xorl %ecx, %ecx
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1

 define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonly, i32) local_unnamed_addr #0 {
 ; SSE2-LABEL: _Z9test_charPcS_i_1024:
 ; SSE2:       # %bb.0: # %entry
 ; SSE2-NEXT:    movl %edx, %eax
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm9, %xmm9
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  .LBB7_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movq {{.*#+}} xmm5 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm5
 ; SSE2-NEXT:    movq {{.*#+}} xmm6 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm6
 ; SSE2-NEXT:    movq {{.*#+}} xmm7 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm7
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm1
 ; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm2
 ; SSE2-NEXT:    pmaddwd %xmm5, %xmm2
 ; SSE2-NEXT:    paddd %xmm2, %xmm9
 ; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm2
 ; SSE2-NEXT:    pmaddwd %xmm6, %xmm2
 ; SSE2-NEXT:    paddd %xmm2, %xmm4
 ; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm2
 ; SSE2-NEXT:    pmaddwd %xmm7, %xmm2
-; SSE2-NEXT:    paddd %xmm2, %xmm1
+; SSE2-NEXT:    paddd %xmm2, %xmm0
 ; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmaddwd %xmm0, %xmm2
+; SSE2-NEXT:    pmaddwd %xmm1, %xmm2
 ; SSE2-NEXT:    paddd %xmm2, %xmm3
 ; SSE2-NEXT:    addq $32, %rcx
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB7_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    paddd %xmm8, %xmm4
 ; SSE2-NEXT:    paddd %xmm8, %xmm3
 ; SSE2-NEXT:    paddd %xmm4, %xmm3
 ; SSE2-NEXT:    paddd %xmm8, %xmm9
-; SSE2-NEXT:    paddd %xmm8, %xmm1
-; SSE2-NEXT:    paddd %xmm3, %xmm1
-; SSE2-NEXT:    paddd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm8, %xmm0
+; SSE2-NEXT:    paddd %xmm3, %xmm0
+; SSE2-NEXT:    paddd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
 ; AVX1-LABEL: _Z9test_charPcS_i_1024:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    movl %edx, %eax
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %ecx, %ecx
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1

llvm/trunk/test/CodeGen/X86/mmx-arith.ll

 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    popl %edi
 ; X32-NEXT:    popl %ebx
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test3:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    xorl %r8d, %r8d
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    testl %edx, %edx
-; X64-NEXT:    je .LBB3_2
+; X64-NEXT:    je .LBB3_3
+; X64-NEXT:  # %bb.1: # %bb26.preheader
+; X64-NEXT:    xorl %r8d, %r8d
 ; X64-NEXT:    .p2align 4, 0x90
-; X64-NEXT:  .LBB3_1: # %bb26
+; X64-NEXT:  .LBB3_2: # %bb26
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movslq %r8d, %r8
 ; X64-NEXT:    movq (%rdi,%r8,8), %rcx
 ; X64-NEXT:    addq (%rsi,%r8,8), %rcx
 ; X64-NEXT:    addq %rcx, %rax
 ; X64-NEXT:    incl %r8d
 ; X64-NEXT:    cmpl %edx, %r8d
-; X64-NEXT:    jb .LBB3_1
-; X64-NEXT:  .LBB3_2: # %bb31
+; X64-NEXT:    jb .LBB3_2
+; X64-NEXT:  .LBB3_3: # %bb31
 ; X64-NEXT:    retq
 entry:
   %tmp2942 = icmp eq i32 %count, 0
   br i1 %tmp2942, label %bb31, label %bb26
 
 bb26:
   %i.037.0 = phi i32 [ 0, %entry ], [ %tmp25, %bb26 ]
   %sum.035.0 = phi <1 x i64> [ zeroinitializer, %entry ], [ %tmp22, %bb26 ]

llvm/trunk/test/CodeGen/X86/pr32284.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -fast-isel-sink-local-values -O0 -mtriple=x86_64-unknown -mcpu=skx -o - %s | FileCheck %s --check-prefix=X86-O0
 ; RUN: llc -fast-isel-sink-local-values     -mtriple=x86_64-unknown -mcpu=skx -o - %s | FileCheck %s --check-prefix=X64
 ; RUN: llc -fast-isel-sink-local-values -O0 -mtriple=i686-unknown   -mcpu=skx -o - %s | FileCheck %s --check-prefix=686-O0
 ; RUN: llc -fast-isel-sink-local-values     -mtriple=i686-unknown   -mcpu=skx -o - %s | FileCheck %s --check-prefix=686
 
 @c = external constant i8, align 1
 
 define void @foo() {
 ; X86-O0-LABEL: foo:
 ; X86-O0:       # %bb.0: # %entry
 ; X86-O0-NEXT:    xorl %eax, %eax
-; X86-O0-NEXT:    movl %eax, %ecx
-; X86-O0-NEXT:    xorl %eax, %eax
+; X86-O0-NEXT:    xorl %ecx, %ecx
 ; X86-O0-NEXT:    movzbl c, %edx
-; X86-O0-NEXT:    subl %edx, %eax
-; X86-O0-NEXT:    movslq %eax, %rsi
-; X86-O0-NEXT:    subq %rsi, %rcx
-; X86-O0-NEXT:    movb %cl, %dil
+; X86-O0-NEXT:    subl %edx, %ecx
+; X86-O0-NEXT:    movslq %ecx, %rsi
+; X86-O0-NEXT:    subq %rsi, %rax
+; X86-O0-NEXT:    movb %al, %dil
 ; X86-O0-NEXT:    cmpb $0, %dil
 ; X86-O0-NEXT:    setne %dil
 ; X86-O0-NEXT:    andb $1, %dil
 ; X86-O0-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; X86-O0-NEXT:    cmpb $0, c
 ; X86-O0-NEXT:    setne %dil
 ; X86-O0-NEXT:    xorb $-1, %dil
 ; X86-O0-NEXT:    xorb $-1, %dil
 ; X86-O0-NEXT:    andb $1, %dil
-; X86-O0-NEXT:    movzbl %dil, %eax
+; X86-O0-NEXT:    movzbl %dil, %ecx
 ; X86-O0-NEXT:    movzbl c, %edx
-; X86-O0-NEXT:    cmpl %edx, %eax
+; X86-O0-NEXT:    cmpl %edx, %ecx
 ; X86-O0-NEXT:    setle %dil
 ; X86-O0-NEXT:    andb $1, %dil
-; X86-O0-NEXT:    movzbl %dil, %eax
-; X86-O0-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
+; X86-O0-NEXT:    movzbl %dil, %ecx
+; X86-O0-NEXT:    movl %ecx, -{{[0-9]+}}(%rsp)
 ; X86-O0-NEXT:    retq
 ;
 ; X64-LABEL: foo:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movzbl {{.*}}(%rip), %eax
 ; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    testl %eax, %eax
 ; X64-NEXT:    setne %cl

llvm/trunk/test/CodeGen/X86/pr32340.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -fast-isel-sink-local-values -O0 -mtriple=x86_64-unknown-linux-gnu -fast-isel-abort=1 -o - %s | FileCheck %s -check-prefix=X64
 
 @var_825 = external global i16, align 2
 @var_32 = external global i16, align 2
 @var_901 = external global i16, align 2
 @var_826 = external global i64, align 8
 @var_57 = external global i64, align 8
 @var_900 = external global i16, align 2
 @var_28 = external constant i64, align 8
 @var_827 = external global i16, align 2
 
 define void @foo() {
 ; X64-LABEL: foo:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    movl %eax, %ecx
 ; X64-NEXT:    movw $0, var_825
-; X64-NEXT:    movzwl var_32, %eax
+; X64-NEXT:    movzwl var_32, %ecx
 ; X64-NEXT:    movzwl var_901, %edx
-; X64-NEXT:    movl %eax, %esi
+; X64-NEXT:    movl %ecx, %esi
 ; X64-NEXT:    xorl %edx, %esi
-; X64-NEXT:    movl %eax, %edx
+; X64-NEXT:    movl %ecx, %edx
 ; X64-NEXT:    xorl %esi, %edx
-; X64-NEXT:    addl %eax, %edx
+; X64-NEXT:    addl %ecx, %edx
 ; X64-NEXT:    movslq %edx, %rdi
 ; X64-NEXT:    movq %rdi, var_826
-; X64-NEXT:    movzwl var_32, %eax
-; X64-NEXT:    movl %eax, %edi
-; X64-NEXT:    movzwl var_901, %eax
-; X64-NEXT:    xorl $51981, %eax # imm = 0xCB0D
-; X64-NEXT:    movslq %eax, %r8
+; X64-NEXT:    movzwl var_32, %ecx
+; X64-NEXT:    movl %ecx, %edi
+; X64-NEXT:    movzwl var_901, %ecx
+; X64-NEXT:    xorl $51981, %ecx # imm = 0xCB0D
+; X64-NEXT:    movslq %ecx, %r8
 ; X64-NEXT:    movabsq $-1142377792914660288, %r9 # imm = 0xF02575732E06E440
 ; X64-NEXT:    xorq %r9, %r8
 ; X64-NEXT:    movq %rdi, %r9
 ; X64-NEXT:    xorq %r8, %r9
 ; X64-NEXT:    xorq $-1, %r9
 ; X64-NEXT:    xorq %r9, %rdi
 ; X64-NEXT:    movq %rdi, %r8
 ; X64-NEXT:    orq var_57, %r8
 ; X64-NEXT:    orq %r8, %rdi
 ; X64-NEXT:    movw %di, %r10w
 ; X64-NEXT:    movw %r10w, var_900
-; X64-NEXT:    cmpq var_28, %rcx
+; X64-NEXT:    cmpq var_28, %rax
 ; X64-NEXT:    setne %r11b
 ; X64-NEXT:    andb $1, %r11b
-; X64-NEXT:    movzbl %r11b, %eax
-; X64-NEXT:    movw %ax, %r10w
+; X64-NEXT:    movzbl %r11b, %ecx
+; X64-NEXT:    movw %cx, %r10w
 ; X64-NEXT:    movw %r10w, var_827
 ; X64-NEXT:    retq
 entry:
   store i16 0, i16* @var_825, align 2
   %v0 = load i16, i16* @var_32, align 2
   %conv = zext i16 %v0 to i32
   %v2 = load i16, i16* @var_901, align 2
   %conv2 = zext i16 %v2 to i32

llvm/trunk/test/CodeGen/X86/scheduler-backtracking.ll

 ; ILP-NEXT:    pushq %rbp
 ; ILP-NEXT:    pushq %r15
 ; ILP-NEXT:    pushq %r14
 ; ILP-NEXT:    pushq %r13
 ; ILP-NEXT:    pushq %r12
 ; ILP-NEXT:    pushq %rbx
 ; ILP-NEXT:    movq %rcx, %r9
 ; ILP-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; ILP-NEXT:    xorl %eax, %eax
 ; ILP-NEXT:    addq $1, %rsi
 ; ILP-NEXT:    adcq $0, %rdx
 ; ILP-NEXT:    adcq $0, %r9
 ; ILP-NEXT:    adcq $0, %r8
 ; ILP-NEXT:    leal 1(%rsi,%rsi), %edi
 ; ILP-NEXT:    movl $1, %ebp
-; ILP-NEXT:    xorl %r14d, %r14d
+; ILP-NEXT:    xorl %eax, %eax
+; ILP-NEXT:    xorl %r11d, %r11d
 ; ILP-NEXT:    movl %edi, %ecx
-; ILP-NEXT:    shldq %cl, %rbp, %r14
-; ILP-NEXT:    movl $1, %r11d
-; ILP-NEXT:    shlq %cl, %r11
+; ILP-NEXT:    shldq %cl, %rbp, %r11
+; ILP-NEXT:    movl $1, %r14d
+; ILP-NEXT:    shlq %cl, %r14
 ; ILP-NEXT:    movb $-128, %r10b
 ; ILP-NEXT:    subb %dil, %r10b
 ; ILP-NEXT:    movq %r9, %r13
 ; ILP-NEXT:    movl %r10d, %ecx
 ; ILP-NEXT:    shlq %cl, %r13
 ; ILP-NEXT:    movl $1, %r12d
 ; ILP-NEXT:    shrdq %cl, %rax, %r12
 ; ILP-NEXT:    xorl %r15d, %r15d
 ; ILP-NEXT:    movl %edi, %ecx
 ; ILP-NEXT:    shldq %cl, %r15, %r15
-; ILP-NEXT:    movq %rsi, %rbx
-; ILP-NEXT:    shrdq %cl, %rdx, %rbx
+; ILP-NEXT:    movq %rsi, %rbp
+; ILP-NEXT:    shrdq %cl, %rdx, %rbp
 ; ILP-NEXT:    shrq %cl, %rdx
 ; ILP-NEXT:    addb $-128, %cl
 ; ILP-NEXT:    shrdq %cl, %r8, %r9
 ; ILP-NEXT:    testb $64, %dil
-; ILP-NEXT:    cmovneq %r11, %r14
-; ILP-NEXT:    cmoveq %rbx, %rdx
+; ILP-NEXT:    cmovneq %r14, %r11
+; ILP-NEXT:    cmoveq %rbp, %rdx
 ; ILP-NEXT:    cmovneq %rax, %r15
-; ILP-NEXT:    cmovneq %rax, %r11
+; ILP-NEXT:    cmovneq %rax, %r14
 ; ILP-NEXT:    testb $64, %r10b
 ; ILP-NEXT:    cmovneq %rax, %r12
 ; ILP-NEXT:    cmovneq %rax, %r13
-; ILP-NEXT:    movl $1, %ebx
-; ILP-NEXT:    shlq %cl, %rbx
+; ILP-NEXT:    movl $1, %ebp
+; ILP-NEXT:    shlq %cl, %rbp
 ; ILP-NEXT:    orl %edx, %r13d
 ; ILP-NEXT:    xorl %edx, %edx
-; ILP-NEXT:    movl $1, %ebp
-; ILP-NEXT:    shldq %cl, %rbp, %rdx
+; ILP-NEXT:    movl $1, %ebx
+; ILP-NEXT:    shldq %cl, %rbx, %rdx
 ; ILP-NEXT:    shrq %cl, %r8
 ; ILP-NEXT:    testb $64, %cl
 ; ILP-NEXT:    cmoveq %r9, %r8
-; ILP-NEXT:    cmovneq %rbx, %rdx
-; ILP-NEXT:    cmovneq %rax, %rbx
+; ILP-NEXT:    cmovneq %rbp, %rdx
+; ILP-NEXT:    cmovneq %rax, %rbp
 ; ILP-NEXT:    testb %dil, %dil
-; ILP-NEXT:    cmovsq %rax, %r14
 ; ILP-NEXT:    cmovsq %rax, %r11
+; ILP-NEXT:    cmovsq %rax, %r14
 ; ILP-NEXT:    jns .LBB0_2
 ; ILP-NEXT:  # %bb.1:
 ; ILP-NEXT:    movl %r8d, %r13d
 ; ILP-NEXT:  .LBB0_2:
 ; ILP-NEXT:    je .LBB0_4
 ; ILP-NEXT:  # %bb.3:
 ; ILP-NEXT:    movl %r13d, %esi
 ; ILP-NEXT:  .LBB0_4:
-; ILP-NEXT:    cmovnsq %r12, %rbx
-; ILP-NEXT:    cmoveq %rax, %rbx
+; ILP-NEXT:    cmovnsq %r12, %rbp
+; ILP-NEXT:    cmoveq %rax, %rbp
 ; ILP-NEXT:    cmovnsq %r15, %rdx
 ; ILP-NEXT:    cmoveq %rax, %rdx
 ; ILP-NEXT:    testb $1, %sil
 ; ILP-NEXT:    cmovneq %rax, %rdx
 ; ILP-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; ILP-NEXT:    movq %rdx, 24(%rax)
-; ILP-NEXT:    cmovneq %rax, %rbx
-; ILP-NEXT:    movq %rbx, 16(%rax)
-; ILP-NEXT:    cmovneq %rax, %r14
-; ILP-NEXT:    movq %r14, 8(%rax)
+; ILP-NEXT:    cmovneq %rax, %rbp
+; ILP-NEXT:    movq %rbp, 16(%rax)
 ; ILP-NEXT:    cmovneq %rax, %r11
-; ILP-NEXT:    movq %r11, (%rax)
+; ILP-NEXT:    movq %r11, 8(%rax)
+; ILP-NEXT:    cmovneq %rax, %r14
+; ILP-NEXT:    movq %r14, (%rax)
 ; ILP-NEXT:    popq %rbx
 ; ILP-NEXT:    popq %r12
 ; ILP-NEXT:    popq %r13
 ; ILP-NEXT:    popq %r14
 ; ILP-NEXT:    popq %r15
 ; ILP-NEXT:    popq %rbp
 ; ILP-NEXT:    retq
 ;
 ; HYBRID-LABEL: test1:
 ; HYBRID:       # %bb.0:
-; HYBRID-NEXT:    pushq %rbp
 ; HYBRID-NEXT:    pushq %r15
 ; HYBRID-NEXT:    pushq %r14
 ; HYBRID-NEXT:    pushq %r13
 ; HYBRID-NEXT:    pushq %r12
 ; HYBRID-NEXT:    pushq %rbx
 ; HYBRID-NEXT:    movq %rcx, %r9
 ; HYBRID-NEXT:    movq %rdi, %rax
 ; HYBRID-NEXT:    addq $1, %rsi
 ; HYBRID-NEXT:    adcq $0, %rdx
 ; HYBRID-NEXT:    adcq $0, %r9
 ; HYBRID-NEXT:    adcq $0, %r8
-; HYBRID-NEXT:    xorl %r10d, %r10d
 ; HYBRID-NEXT:    leal 1(%rsi,%rsi), %edi
 ; HYBRID-NEXT:    xorl %r14d, %r14d
+; HYBRID-NEXT:    xorl %r15d, %r15d
 ; HYBRID-NEXT:    movl %edi, %ecx
-; HYBRID-NEXT:    shldq %cl, %r14, %r14
+; HYBRID-NEXT:    shldq %cl, %r15, %r15
 ; HYBRID-NEXT:    testb $64, %dil
-; HYBRID-NEXT:    cmovneq %r10, %r14
-; HYBRID-NEXT:    movl $1, %ebp
+; HYBRID-NEXT:    cmovneq %r14, %r15
+; HYBRID-NEXT:    movl $1, %r11d
 ; HYBRID-NEXT:    movl $1, %r12d
 ; HYBRID-NEXT:    shlq %cl, %r12
 ; HYBRID-NEXT:    testb $64, %dil
-; HYBRID-NEXT:    movq %r12, %r11
-; HYBRID-NEXT:    cmovneq %r10, %r11
+; HYBRID-NEXT:    movq %r12, %r10
+; HYBRID-NEXT:    cmovneq %r14, %r10
 ; HYBRID-NEXT:    movq %rsi, %rbx
 ; HYBRID-NEXT:    shrdq %cl, %rdx, %rbx
 ; HYBRID-NEXT:    shrq %cl, %rdx
 ; HYBRID-NEXT:    testb $64, %dil
 ; HYBRID-NEXT:    cmoveq %rbx, %rdx
-; HYBRID-NEXT:    xorl %r15d, %r15d
-; HYBRID-NEXT:    shldq %cl, %rbp, %r15
+; HYBRID-NEXT:    xorl %r13d, %r13d
+; HYBRID-NEXT:    shldq %cl, %r11, %r13
 ; HYBRID-NEXT:    testb $64, %dil
-; HYBRID-NEXT:    cmovneq %r12, %r15
+; HYBRID-NEXT:    cmovneq %r12, %r13
 ; HYBRID-NEXT:    movb $-128, %cl
 ; HYBRID-NEXT:    subb %dil, %cl
-; HYBRID-NEXT:    movq %r9, %r13
-; HYBRID-NEXT:    shlq %cl, %r13
+; HYBRID-NEXT:    movq %r9, %rbx
+; HYBRID-NEXT:    shlq %cl, %rbx
 ; HYBRID-NEXT:    movl $1, %r12d
-; HYBRID-NEXT:    shrdq %cl, %r10, %r12
+; HYBRID-NEXT:    shrdq %cl, %r14, %r12
 ; HYBRID-NEXT:    testb $64, %cl
-; HYBRID-NEXT:    cmovneq %r10, %r12
-; HYBRID-NEXT:    cmovneq %r10, %r13
-; HYBRID-NEXT:    orl %edx, %r13d
+; HYBRID-NEXT:    cmovneq %r14, %r12
+; HYBRID-NEXT:    cmovneq %r14, %rbx
+; HYBRID-NEXT:    orl %edx, %ebx
 ; HYBRID-NEXT:    movl %edi, %ecx
 ; HYBRID-NEXT:    addb $-128, %cl
 ; HYBRID-NEXT:    shrdq %cl, %r8, %r9
 ; HYBRID-NEXT:    shrq %cl, %r8
 ; HYBRID-NEXT:    xorl %edx, %edx
-; HYBRID-NEXT:    shldq %cl, %rbp, %rdx
-; HYBRID-NEXT:    shlq %cl, %rbp
+; HYBRID-NEXT:    shldq %cl, %r11, %rdx
+; HYBRID-NEXT:    shlq %cl, %r11
 ; HYBRID-NEXT:    testb $64, %cl
-; HYBRID-NEXT:    cmovneq %rbp, %rdx
+; HYBRID-NEXT:    cmovneq %r11, %rdx
 ; HYBRID-NEXT:    cmoveq %r9, %r8
-; HYBRID-NEXT:    cmovneq %r10, %rbp
+; HYBRID-NEXT:    cmovneq %r14, %r11
 ; HYBRID-NEXT:    testb %dil, %dil
 ; HYBRID-NEXT:    jns .LBB0_2
 ; HYBRID-NEXT:  # %bb.1:
-; HYBRID-NEXT:    movl %r8d, %r13d
+; HYBRID-NEXT:    movl %r8d, %ebx
 ; HYBRID-NEXT:  .LBB0_2:
 ; HYBRID-NEXT:    je .LBB0_4
 ; HYBRID-NEXT:  # %bb.3:
-; HYBRID-NEXT:    movl %r13d, %esi
+; HYBRID-NEXT:    movl %ebx, %esi
 ; HYBRID-NEXT:  .LBB0_4:
-; HYBRID-NEXT:    cmovsq %r10, %r15
-; HYBRID-NEXT:    cmovnsq %r12, %rbp
-; HYBRID-NEXT:    cmoveq %r10, %rbp
-; HYBRID-NEXT:    cmovnsq %r14, %rdx
-; HYBRID-NEXT:    cmoveq %r10, %rdx
-; HYBRID-NEXT:    cmovsq %r10, %r11
+; HYBRID-NEXT:    cmovsq %r14, %r13
+; HYBRID-NEXT:    cmovnsq %r12, %r11
+; HYBRID-NEXT:    cmoveq %r14, %r11
+; HYBRID-NEXT:    cmovnsq %r15, %rdx
+; HYBRID-NEXT:    cmoveq %r14, %rdx
+; HYBRID-NEXT:    cmovsq %r14, %r10
 ; HYBRID-NEXT:    testb $1, %sil
 ; HYBRID-NEXT:    cmovneq %rax, %rdx
 ; HYBRID-NEXT:    movq %rdx, 24(%rax)
-; HYBRID-NEXT:    cmovneq %rax, %rbp
-; HYBRID-NEXT:    movq %rbp, 16(%rax)
-; HYBRID-NEXT:    cmovneq %rax, %r15
-; HYBRID-NEXT:    movq %r15, 8(%rax)
 ; HYBRID-NEXT:    cmovneq %rax, %r11
-; HYBRID-NEXT:    movq %r11, (%rax)
+; HYBRID-NEXT:    movq %r11, 16(%rax)
+; HYBRID-NEXT:    cmovneq %rax, %r13
+; HYBRID-NEXT:    movq %r13, 8(%rax)
+; HYBRID-NEXT:    cmovneq %rax, %r10
+; HYBRID-NEXT:    movq %r10, (%rax)
 ; HYBRID-NEXT:    popq %rbx
 ; HYBRID-NEXT:    popq %r12
 ; HYBRID-NEXT:    popq %r13
 ; HYBRID-NEXT:    popq %r14
 ; HYBRID-NEXT:    popq %r15
-; HYBRID-NEXT:    popq %rbp
 ; HYBRID-NEXT:    retq
 ;
 ; BURR-LABEL: test1:
 ; BURR:       # %bb.0:
-; BURR-NEXT:    pushq %rbp
 ; BURR-NEXT:    pushq %r15
 ; BURR-NEXT:    pushq %r14
 ; BURR-NEXT:    pushq %r13
 ; BURR-NEXT:    pushq %r12
 ; BURR-NEXT:    pushq %rbx
 ; BURR-NEXT:    movq %rcx, %r9
 ; BURR-NEXT:    movq %rdi, %rax
 ; BURR-NEXT:    addq $1, %rsi
 ; BURR-NEXT:    adcq $0, %rdx
 ; BURR-NEXT:    adcq $0, %r9
 ; BURR-NEXT:    adcq $0, %r8
-; BURR-NEXT:    xorl %r10d, %r10d
 ; BURR-NEXT:    leal 1(%rsi,%rsi), %edi
 ; BURR-NEXT:    xorl %r14d, %r14d
+; BURR-NEXT:    xorl %r15d, %r15d
 ; BURR-NEXT:    movl %edi, %ecx
-; BURR-NEXT:    shldq %cl, %r14, %r14
+; BURR-NEXT:    shldq %cl, %r15, %r15
 ; BURR-NEXT:    testb $64, %dil
-; BURR-NEXT:    cmovneq %r10, %r14
-; BURR-NEXT:    movl $1, %ebp
+; BURR-NEXT:    cmovneq %r14, %r15
+; BURR-NEXT:    movl $1, %r11d
 ; BURR-NEXT:    movl $1, %r12d
 ; BURR-NEXT:    shlq %cl, %r12
 ; BURR-NEXT:    testb $64, %dil
-; BURR-NEXT:    movq %r12, %r11
-; BURR-NEXT:    cmovneq %r10, %r11
+; BURR-NEXT:    movq %r12, %r10
+; BURR-NEXT:    cmovneq %r14, %r10
 ; BURR-NEXT:    movq %rsi, %rbx
 ; BURR-NEXT:    shrdq %cl, %rdx, %rbx
 ; BURR-NEXT:    shrq %cl, %rdx
 ; BURR-NEXT:    testb $64, %dil
 ; BURR-NEXT:    cmoveq %rbx, %rdx
-; BURR-NEXT:    xorl %r15d, %r15d
-; BURR-NEXT:    shldq %cl, %rbp, %r15
+; BURR-NEXT:    xorl %r13d, %r13d
+; BURR-NEXT:    shldq %cl, %r11, %r13
 ; BURR-NEXT:    testb $64, %dil
-; BURR-NEXT:    cmovneq %r12, %r15
+; BURR-NEXT:    cmovneq %r12, %r13
 ; BURR-NEXT:    movb $-128, %cl
 ; BURR-NEXT:    subb %dil, %cl
-; BURR-NEXT:    movq %r9, %r13
-; BURR-NEXT:    shlq %cl, %r13
+; BURR-NEXT:    movq %r9, %rbx
+; BURR-NEXT:    shlq %cl, %rbx
 ; BURR-NEXT:    movl $1, %r12d
-; BURR-NEXT:    shrdq %cl, %r10, %r12
+; BURR-NEXT:    shrdq %cl, %r14, %r12
 ; BURR-NEXT:    testb $64, %cl
-; BURR-NEXT:    cmovneq %r10, %r12
-; BURR-NEXT:    cmovneq %r10, %r13
-; BURR-NEXT:    orl %edx, %r13d
+; BURR-NEXT:    cmovneq %r14, %r12
+; BURR-NEXT:    cmovneq %r14, %rbx
+; BURR-NEXT:    orl %edx, %ebx
 ; BURR-NEXT:    movl %edi, %ecx
 ; BURR-NEXT:    addb $-128, %cl
 ; BURR-NEXT:    shrdq %cl, %r8, %r9
 ; BURR-NEXT:    xorl %edx, %edx
-; BURR-NEXT:    shldq %cl, %rbp, %rdx
+; BURR-NEXT:    shldq %cl, %r11, %rdx
 ; BURR-NEXT:    shrq %cl, %r8
-; BURR-NEXT:    shlq %cl, %rbp
+; BURR-NEXT:    shlq %cl, %r11
 ; BURR-NEXT:    testb $64, %cl
-; BURR-NEXT:    cmovneq %rbp, %rdx
+; BURR-NEXT:    cmovneq %r11, %rdx
 ; BURR-NEXT:    cmoveq %r9, %r8
-; BURR-NEXT:    cmovneq %r10, %rbp
+; BURR-NEXT:    cmovneq %r14, %r11
 ; BURR-NEXT:    testb %dil, %dil
 ; BURR-NEXT:    jns .LBB0_2
 ; BURR-NEXT:  # %bb.1:
-; BURR-NEXT:    movl %r8d, %r13d
+; BURR-NEXT:    movl %r8d, %ebx
 ; BURR-NEXT:  .LBB0_2:
 ; BURR-NEXT:    je .LBB0_4
 ; BURR-NEXT:  # %bb.3:
-; BURR-NEXT:    movl %r13d, %esi
+; BURR-NEXT:    movl %ebx, %esi
 ; BURR-NEXT:  .LBB0_4:
-; BURR-NEXT:    cmovsq %r10, %r15
-; BURR-NEXT:    cmovnsq %r12, %rbp
-; BURR-NEXT:    cmoveq %r10, %rbp
-; BURR-NEXT:    cmovnsq %r14, %rdx
-; BURR-NEXT:    cmoveq %r10, %rdx
-; BURR-NEXT:    cmovsq %r10, %r11
+; BURR-NEXT:    cmovsq %r14, %r13
+; BURR-NEXT:    cmovnsq %r12, %r11
+; BURR-NEXT:    cmoveq %r14, %r11
+; BURR-NEXT:    cmovnsq %r15, %rdx
+; BURR-NEXT:    cmoveq %r14, %rdx
+; BURR-NEXT:    cmovsq %r14, %r10
 ; BURR-NEXT:    testb $1, %sil
 ; BURR-NEXT:    cmovneq %rax, %rdx
 ; BURR-NEXT:    movq %rdx, 24(%rax)
-; BURR-NEXT:    cmovneq %rax, %rbp
-; BURR-NEXT:    movq %rbp, 16(%rax)
-; BURR-NEXT:    cmovneq %rax, %r15
-; BURR-NEXT:    movq %r15, 8(%rax)
 ; BURR-NEXT:    cmovneq %rax, %r11
-; BURR-NEXT:    movq %r11, (%rax)
+; BURR-NEXT:    movq %r11, 16(%rax)
+; BURR-NEXT:    cmovneq %rax, %r13
+; BURR-NEXT:    movq %r13, 8(%rax)
+; BURR-NEXT:    cmovneq %rax, %r10
+; BURR-NEXT:    movq %r10, (%rax)
 ; BURR-NEXT:    popq %rbx
 ; BURR-NEXT:    popq %r12
 ; BURR-NEXT:    popq %r13
 ; BURR-NEXT:    popq %r14
 ; BURR-NEXT:    popq %r15
-; BURR-NEXT:    popq %rbp
 ; BURR-NEXT:    retq
 ;
 ; SRC-LABEL: test1:
 ; SRC:       # %bb.0:
 ; SRC-NEXT:    pushq %rbp
 ; SRC-NEXT:    pushq %r15
 ; SRC-NEXT:    pushq %r14
 ; SRC-NEXT:    pushq %r13

 ; SRC-NEXT:    subb %r11b, %r10b
 ; SRC-NEXT:    movq %r9, %r12
 ; SRC-NEXT:    movl %r10d, %ecx
 ; SRC-NEXT:    shlq %cl, %r12
 ; SRC-NEXT:    movq %rsi, %rbp
 ; SRC-NEXT:    movl %r11d, %ecx
 ; SRC-NEXT:    shrdq %cl, %rdx, %rbp
 ; SRC-NEXT:    shrq %cl, %rdx
-; SRC-NEXT:    xorl %r15d, %r15d
 ; SRC-NEXT:    movl $1, %edi
+; SRC-NEXT:    xorl %r15d, %r15d
 ; SRC-NEXT:    xorl %r14d, %r14d
 ; SRC-NEXT:    shldq %cl, %rdi, %r14
 ; SRC-NEXT:    xorl %r13d, %r13d
 ; SRC-NEXT:    shldq %cl, %r13, %r13
 ; SRC-NEXT:    movl $1, %ebx
 ; SRC-NEXT:    shlq %cl, %rbx
 ; SRC-NEXT:    testb $64, %r11b
 ; SRC-NEXT:    cmoveq %rbp, %rdx

 }
 
 declare i256 @llvm.ctlz.i256(i256, i1) nounwind readnone
 
 define i64 @test4(i64 %a, i64 %b) nounwind {
 ; ILP-LABEL: test4:
 ; ILP:       # %bb.0:
 ; ILP-NEXT:    xorl %ecx, %ecx
-; ILP-NEXT:    xorl %edx, %edx
 ; ILP-NEXT:    addq $1, %rsi
-; ILP-NEXT:    setb %dl
+; ILP-NEXT:    setb %cl
 ; ILP-NEXT:    movl $2, %eax
+; ILP-NEXT:    xorl %edx, %edx
 ; ILP-NEXT:    cmpq %rdi, %rsi
-; ILP-NEXT:    sbbq $0, %rdx
-; ILP-NEXT:    movl $0, %edx
-; ILP-NEXT:    sbbq $0, %rdx
 ; ILP-NEXT:    sbbq $0, %rcx
+; ILP-NEXT:    movl $0, %ecx
+; ILP-NEXT:    sbbq $0, %rcx
+; ILP-NEXT:    sbbq $0, %rdx
 ; ILP-NEXT:    setae %cl
 ; ILP-NEXT:    movzbl %cl, %ecx
 ; ILP-NEXT:    subq %rcx, %rax
 ; ILP-NEXT:    retq
 ;
 ; HYBRID-LABEL: test4:
 ; HYBRID:       # %bb.0:
 ; HYBRID-NEXT:    xorl %eax, %eax
-; HYBRID-NEXT:    xorl %ecx, %ecx
 ; HYBRID-NEXT:    addq $1, %rsi
-; HYBRID-NEXT:    setb %cl
+; HYBRID-NEXT:    setb %al
+; HYBRID-NEXT:    xorl %ecx, %ecx
 ; HYBRID-NEXT:    cmpq %rdi, %rsi
-; HYBRID-NEXT:    sbbq $0, %rcx
-; HYBRID-NEXT:    movl $0, %ecx
-; HYBRID-NEXT:    sbbq $0, %rcx
 ; HYBRID-NEXT:    sbbq $0, %rax
+; HYBRID-NEXT:    movl $0, %eax
+; HYBRID-NEXT:    sbbq $0, %rax
+; HYBRID-NEXT:    sbbq $0, %rcx
 ; HYBRID-NEXT:    setae %al
 ; HYBRID-NEXT:    movzbl %al, %ecx
 ; HYBRID-NEXT:    movl $2, %eax
 ; HYBRID-NEXT:    subq %rcx, %rax
 ; HYBRID-NEXT:    retq
 ;
 ; BURR-LABEL: test4:
 ; BURR:       # %bb.0:
 ; BURR-NEXT:    xorl %eax, %eax
-; BURR-NEXT:    xorl %ecx, %ecx
 ; BURR-NEXT:    addq $1, %rsi
-; BURR-NEXT:    setb %cl
+; BURR-NEXT:    setb %al
+; BURR-NEXT:    xorl %ecx, %ecx
 ; BURR-NEXT:    cmpq %rdi, %rsi
-; BURR-NEXT:    sbbq $0, %rcx
-; BURR-NEXT:    movl $0, %ecx
-; BURR-NEXT:    sbbq $0, %rcx
 ; BURR-NEXT:    sbbq $0, %rax
+; BURR-NEXT:    movl $0, %eax
+; BURR-NEXT:    sbbq $0, %rax
+; BURR-NEXT:    sbbq $0, %rcx
 ; BURR-NEXT:    setae %al
 ; BURR-NEXT:    movzbl %al, %ecx
 ; BURR-NEXT:    movl $2, %eax
 ; BURR-NEXT:    subq %rcx, %rax
 ; BURR-NEXT:    retq
 ;
 ; SRC-LABEL: test4:
 ; SRC:       # %bb.0:

llvm/trunk/test/CodeGen/X86/spill-zero-x86_64.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+; This test checks that we use "movq $0, (%rsp)" to spill a 0 to the stack. It
+; was reduced from a larger function.
+
+; CHECK:    movq $0, (%rsp) # 8-byte Folded Spill
+
+%struct.foo = type { i8*, i32 }
+
+declare void @pluto()
+
+define void @spam() {
+bb:
+  br label %bb13
+
+bb1:                                              ; preds = %bb18
+  call void @pluto()
+  %tmp = getelementptr inbounds %struct.foo, %struct.foo* %tmp20, i64 0, i32 1
+  %tmp2 = bitcast i32* %tmp to %struct.foo**
+  store %struct.foo* null, %struct.foo** %tmp2
+  unreachable
+
+bb3:                                              ; preds = %bb18
+  call void @pluto()
+  store i8* %tmp22, i8** undef
+  unreachable
+
+bb4:                                              ; preds = %bb18
+  call void @pluto()
+  br label %bb13
+
+bb5:                                              ; preds = %bb18
+  %tmp7 = add nsw i32 %tmp23, 1
+  store i8* %tmp22, i8** undef
+  unreachable
+
+bb8:                                              ; preds = %bb18
+  store %struct.foo* %tmp14, %struct.foo** undef
+  unreachable
+
+bb9:                                              ; preds = %bb18
+  %tmp10 = load %struct.foo*, %struct.foo** undef
+  br label %bb13
+
+bb13:                                             ; preds = %bb18, %bb9, %bb4, %bb
+  %tmp14 = phi %struct.foo* [ %tmp14, %bb18 ], [ %tmp14, %bb4 ], [ null, %bb ], [ %tmp10, %bb9 ]
+  %tmp15 = phi %struct.foo* [ %tmp26, %bb18 ], [ %tmp26, %bb4 ], [ null, %bb ], [ %tmp26, %bb9 ]
+  %tmp16 = phi i32 [ %tmp23, %bb18 ], [ %tmp23, %bb4 ], [ 0, %bb ], [ %tmp23, %bb9 ]
+  br label %bb17
+
+bb17:                                             ; preds = %bb13
+  br i1 false, label %bb27, label %bb18
+
+bb18:                                             ; preds = %bb17
+  %tmp19 = load %struct.foo*, %struct.foo** undef
+  %tmp20 = getelementptr inbounds %struct.foo, %struct.foo* %tmp19, i64 0
+  %tmp21 = getelementptr inbounds %struct.foo, %struct.foo* %tmp20, i64 0, i32 0
+  %tmp22 = load i8*, i8** %tmp21
+  %tmp23 = add nsw i32 %tmp16, -1
+  %tmp24 = getelementptr inbounds %struct.foo, %struct.foo* %tmp15, i64 0, i32 1
+  %tmp25 = bitcast i32* %tmp24 to %struct.foo**
+  %tmp26 = load %struct.foo*, %struct.foo** %tmp25
+  switch i32 undef, label %bb9 [
+    i32 1, label %bb1
+    i32 2, label %bb3
+    i32 3, label %bb4
+    i32 4, label %bb5
+    i32 5, label %bb13
+    i32 6, label %bb8
+  ]
+
+bb27:                                             ; preds = %bb17
+  ret void
+}

llvm/trunk/test/CodeGen/X86/swifterror.ll

 ; CHECK-APPLE: callq {{.*}}foo
 ; CHECK-APPLE: testq %r12, %r12
 ; CHECK-APPLE: jne
 ; Access part of the error object and save it to error_ref
 ; CHECK-APPLE: movb 8(%rdi)
 ; CHECK-APPLE: callq {{.*}}free
 
 ; CHECK-O0-LABEL: caller:
-; CHECK-O0: xorl
-; CHECK-O0: movl %{{.*}}, %r12d
+; CHECK-O0: xorl %r12d, %r12d
 ; CHECK-O0: callq {{.*}}foo
 ; CHECK-O0: jne
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   store %swift_error* null, %swift_error** %error_ptr_ref
   %call = call float @foo(%swift_error** swifterror %error_ptr_ref)
   %error_from_foo = load %swift_error*, %swift_error** %error_ptr_ref
   %had_error_from_foo = icmp ne %swift_error* %error_from_foo, null

 ; CHECK-APPLE: ucomiss
 ; CHECK-APPLE: jbe
 ; Access part of the error object and save it to error_ref
 ; CHECK-APPLE: movb 8(%r12)
 ; CHECK-APPLE: movq %r12, %rdi
 ; CHECK-APPLE: callq {{.*}}free
 
 ; CHECK-O0-LABEL: caller2:
-; CHECK-O0: xorl
-; CHECK-O0: movl %{{.*}}, %r12d
+; CHECK-O0: xorl %r12d, %r12d
 ; CHECK-O0: callq {{.*}}foo
 ; CHECK-O0: movq %r12, [[ID:%[a-z]+]]
 ; CHECK-O0: cmpq $0, %r12
 ; CHECK-O0: jne
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   br label %bb_loop
 bb_loop:

 ; CHECK-APPLE: testq %r12, %r12
 ; CHECK-APPLE: jne
 ; Access part of the error object and save it to error_ref
 ; CHECK-APPLE: movb 8(%rdi),
 ; CHECK-APPLE: movb %{{.*}},
 ; CHECK-APPLE: callq {{.*}}free
 
 ; CHECK-O0-LABEL: caller3:
-; CHECK-O0: xorl
-; CHECK-O0: movl {{.*}}, %r12d
+; CHECK-O0: xorl %r12d, %r12d
 ; CHECK-O0: movl $1, %esi
 ; CHECK-O0: movq {{.*}}, %rdi
 ; CHECK-O0: callq {{.*}}foo_sret
 ; CHECK-O0: movq %r12,
 ; CHECK-O0: cmpq $0
 ; CHECK-O0: jne
 ; Access part of the error object and save it to error_ref
 ; CHECK-O0: movb 8(%{{.*}}),

 ; CHECK-APPLE: jne
 ; Access part of the error object and save it to error_ref
 ; CHECK-APPLE: movb 8(%rdi)
 ; CHECK-APPLE: callq {{.*}}free
 
 ; CHECK-O0-LABEL: caller_with_multiple_swifterror_values:
 
 ; The first swifterror value:
-; CHECK-O0: xorl
-; CHECK-O0: movl %{{.*}}, %r12d
+; CHECK-O0: xorl %r12d, %r12d
 ; CHECK-O0: callq {{.*}}foo
 ; CHECK-O0: jne
 
 ; The second swifterror value:
-; CHECK-O0: xorl
-; CHECK-O0: movl %{{.*}}, %r12d
+; CHECK-O0: xorl %r12d, %r12d
 ; CHECK-O0: callq {{.*}}foo
 ; CHECK-O0: jne
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   store %swift_error* null, %swift_error** %error_ptr_ref
   %call = call float @foo(%swift_error** swifterror %error_ptr_ref)
   %error_from_foo = load %swift_error*, %swift_error** %error_ptr_ref
   %had_error_from_foo = icmp ne %swift_error* %error_from_foo, null

 }
 
 
 declare swiftcc void @foo2(%swift_error** swifterror)
 
 ; Make sure we properly assign registers during fast-isel.
 ; CHECK-O0-LABEL: testAssign
 ; CHECK-O0:        pushq   %r12
-; CHECK-O0:        xorl    [[ZERO:%[a-z0-9]+]], [[ZERO]]
-; CHECK-O0:        movl    [[ZERO]], %r12d
+; CHECK-O0:        xorl    %r12d, %r12d
 ; CHECK-O0:        callq   _foo2
 ; CHECK-O0:        movq    %r12, [[SLOT:[-a-z0-9\(\)\%]*]]
 ;
 ; CHECK-O0:        movq    [[SLOT]], %rax
 ; CHECK-O0:        popq    %r12
 ; CHECK-O0:        retq
 
 ; CHECK-APPLE-LABEL: testAssign

 a:
   %error = load %swift_error*, %swift_error** %err
   ret %swift_error* %error
 }
 
 
 ; CHECK-O0-LABEL: testAssign4
 ; CHECK-O0:        callq   _foo2
-; CHECK-O0:        xorl    %ecx, %ecx
-; CHECK-O0:        movl    %ecx, %eax
+; CHECK-O0:        xorl    %eax, %eax
 ; CHECK-O0:        movq    %rax, [[SLOT:[-a-z0-9\(\)\%]*]]
 ; CHECK-O0:        movq    [[SLOT]], %rax
 ; CHECK-O0:        movq    %rax, [[SLOT2:[-a-z0-9\(\)\%]*]]
 ; CHECK-O0:        movq    [[SLOT2]], %r12
 ; CHECK-O0:        retq
 
 ; CHECK-APPLE-LABEL: testAssign4
 ; CHECK-APPLE:        callq   _foo2