MachineCopyPropagation: Add full copy propagation mode

[MatzeB]

Summary

Disclaimer: This turned out to be pretty much a copy of Jonas Paulsons patch http://reviews.llvm.org/D20531 just with a different motivation / comments... Uploading for reference.

By default MachineCopyPropagation only removes copies between values
that are identical anyway. This patch adds support for the classical
variant of copy propagation which transforms uses of a register:

%x = COPY %y
   use %x

is transformed to:

%x = COPY %y
   use %y

The transformation improves dependencies and gives the post register
allocation scheduler more freedom. It may also be better for out of
order execution engines that do not track copies.

The mode is opt-in because some targets have additional rules making
some register renames invalid.

[MatzeB]

(Realized I had a type in the cl::opt name)

include/llvm/Target/TargetSubtargetInfo.h

public:
   ///
   /// Override to provide custom PBQP constraints.
   virtual std::unique_ptr<PBQPRAConstraint> getCustomPBQPConstraints() const {
     return nullptr;
   }
 
   /// Enable tracking of subregister liveness in register allocator.
   virtual bool enableSubRegLiveness() const { return false; }
+
+  /// Enable copy propagation for arbitrary register uses.
+  virtual bool enableFullCopyPropagation() const { return false; }
 };
 
 } // End llvm namespace
 
 #endif

lib/CodeGen/MachineCopyPropagation.cpp

 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "codegen-cp"
 
 STATISTIC(NumDeletes, "Number of dead copies deleted");
 
+static cl::opt<bool>
+EnableFullCopyPropagation("full-copy-propagation",
+  cl::desc("Replace uses with earlier copy sources where possible"),
+  cl::init(true));
+
 namespace {
   typedef SmallVector<unsigned, 4> RegList;
   typedef DenseMap<unsigned, RegList> SourceMap;
   typedef DenseMap<unsigned, MachineInstr*> Reg2MIMap;
 
   class MachineCopyPropagation : public MachineFunctionPass {
+    const MachineFunction *MF;
     const TargetRegisterInfo *TRI;
     const TargetInstrInfo *TII;
     const MachineRegisterInfo *MRI;
 
   public:
     static char ID; // Pass identification, replacement for typeid
     MachineCopyPropagation() : MachineFunctionPass(ID) {
       initializeMachineCopyPropagationPass(*PassRegistry::getPassRegistry());
public:
     MachineFunctionProperties getRequiredProperties() const override {
       return MachineFunctionProperties().set(
           MachineFunctionProperties::Property::AllVRegsAllocated);
     }
 
   private:
     void ClobberRegister(unsigned Reg);
     void CopyPropagateBlock(MachineBasicBlock &MBB);
+    bool rename(MachineInstr &MI, unsigned FromReg, unsigned ToReg) const;
     bool eraseIfRedundant(MachineInstr &Copy, unsigned Src, unsigned Def);
 
     /// Candidates for deletion.
     SmallSetVector<MachineInstr*, 8> MaybeDeadCopies;
     /// Def -> available copies map.
     Reg2MIMap AvailCopyMap;
     /// Def -> copies map.
     Reg2MIMap CopyMap;
     /// Src -> Def map
     SourceMap SrcMap;
     bool Changed;
+    bool FullCopyPropagation;
   };
 }
 char MachineCopyPropagation::ID = 0;
 char &llvm::MachineCopyPropagationID = MachineCopyPropagation::ID;
 
 INITIALIZE_PASS(MachineCopyPropagation, "machine-cp",
                 "Machine Copy Propagation Pass", false, false)
 
for (MachineInstr &MI :
     MI.clearRegisterKills(CopyDef, TRI);
 
   Copy.eraseFromParent();
   Changed = true;
   ++NumDeletes;
   return true;
 }
 
+/// Change register use operands reading \p FromReg to \p ToReg where possible.
+bool MachineCopyPropagation::rename(MachineInstr &MI, unsigned FromReg,
+                                    unsigned ToReg) const {
+  assert(TargetRegisterInfo::isPhysicalRegister(FromReg));
+  assert(TargetRegisterInfo::isPhysicalRegister(ToReg));
+  // Check that it is save to rename on all operands.
+  const MCInstrDesc &Desc = MI.getDesc();
+  for (const MachineOperand &MO : MI.operands()) {
+    if (!MO.isReg() || !MO.isUse() || MO.getReg() != FromReg)
+      continue;
+    unsigned OpNum = MI.getOperandNo(&MO);
+    // Operands not mention in the MCInstrDesc are probably due to ABI
+    // constraints and should not be touched.
+    if (OpNum > Desc.getNumOperands() ||
+        Desc.getOperandConstraint(OpNum, MCOI::TIED_TO) != -1)
+      return false;
+
+    const TargetRegisterClass *RC = TII->getRegClass(Desc, OpNum, TRI, *MF);
+    if (RC == nullptr || !RC->contains(ToReg))
+      return false;
+  }
+
+  // Rename register on use operands.
+  for (MachineOperand &MO : MI.operands()) {
+    if (!MO.isReg() || !MO.isUse() || MO.getReg() != FromReg)
+      continue;
+    assert(MO.getSubReg() == 0);
+    MO.setReg(ToReg);
+  }
+  return true;
+}
+
 void MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
   DEBUG(dbgs() << "MCP: CopyPropagateBlock " << MBB.getName() << "\n");
 
   for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) {
     MachineInstr *MI = &*I;
     ++I;
 
     if (MI->isCopy()) {
if (MI->isCopy()) {
         DestList.push_back(Def);
 
       continue;
     }
 
     // Not a copy.
     SmallVector<unsigned, 2> Defs;
     const MachineOperand *RegMask = nullptr;
-    for (const MachineOperand &MO : MI->operands()) {
+    for (MachineOperand &MO : MI->operands()) {
       if (MO.isRegMask())
         RegMask = &MO;
       if (!MO.isReg())
         continue;
       unsigned Reg = MO.getReg();
       if (!Reg)
         continue;
 
       assert(!TargetRegisterInfo::isVirtualRegister(Reg) &&
              "MachineCopyPropagation should be run after register allocation!");
 
       if (MO.isDef()) {
         Defs.push_back(Reg);
         continue;
       }
 
+      // Full copy propagation. Translate
+      //    x = COPY y
+      //    use x
+      // into
+      //    x = COPY y
+      //    use y
+      // This reduces dependencies possibly helping post-ra schedulers and
+      // out of order execution engines.
+      if (FullCopyPropagation && MO.readsReg() && !MRI->isReserved(Reg)) {
+        Reg2MIMap::iterator CI = AvailCopyMap.find(Reg);
+        if (CI != AvailCopyMap.end()) {
+          MachineInstr &Copy = *CI->second;
+          assert(Copy.isCopy());
+          unsigned Def = Copy.getOperand(0).getReg();
+          unsigned Src = Copy.getOperand(1).getReg();
+          assert(!MRI->isReserved(Def));
+          if (Reg == Def && !MRI->isReserved(Src)) {
+            bool Renamed = rename(*MI, Def, Src);
+            if (Renamed) {
+              // Clear potential kill flags on the way.
+              for (MachineInstr &MI :
+                   make_range(Copy.getIterator(), MI->getIterator()))
+                MI.clearRegisterKills(Src, TRI);
+              Reg = Src;
+            }
+          }
+        }
+      }
+
       // If 'Reg' is defined by a copy, the copy is no longer a candidate
       // for elimination.
       for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
         Reg2MIMap::iterator CI = CopyMap.find(*AI);
         if (CI != CopyMap.end()) {
           DEBUG(dbgs() << "MCP: Copy is used - not dead: "; CI->second->dump());
           MaybeDeadCopies.remove(CI->second);
         }

 }
 
 bool MachineCopyPropagation::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(*MF.getFunction()))
     return false;
 
   Changed = false;
 
-  TRI = MF.getSubtarget().getRegisterInfo();
-  TII = MF.getSubtarget().getInstrInfo();
+  this->MF = &MF;
+  const TargetSubtargetInfo &STI = MF.getSubtarget();
+  TRI = STI.getRegisterInfo();
+  TII = STI.getInstrInfo();
   MRI = &MF.getRegInfo();
+  FullCopyPropagation = EnableFullCopyPropagation &
+                        STI.enableFullCopyPropagation();
 
   for (MachineBasicBlock &MBB : MF)
     CopyPropagateBlock(MBB);
 
   return Changed;
 }
 

lib/Target/AArch64/AArch64Subtarget.h

public:
   const char *getBZeroEntry() const;
 
   void overrideSchedPolicy(MachineSchedPolicy &Policy, MachineInstr *begin,
                            MachineInstr *end,
                            unsigned NumRegionInstrs) const override;
 
   bool enableEarlyIfConversion() const override;
 
+  bool enableFullCopyPropagation() const override { return true; }
+
   std::unique_ptr<PBQPRAConstraint> getCustomPBQPConstraints() const override;
 };
 } // End llvm namespace
 
 #endif

lib/Target/ARM/ARMSubtarget.h

public:
   bool enableMachineScheduler() const override;
 
   /// True for some subtargets at > -O0.
   bool enablePostRAScheduler() const override;
 
   // enableAtomicExpand- True if we need to expand our atomics.
   bool enableAtomicExpand() const override;
 
+  bool enableFullCopyPropagation() const override { return true; }
+
   /// getInstrItins - Return the instruction itineraries based on subtarget
   /// selection.
   const InstrItineraryData *getInstrItineraryData() const override {
     return &InstrItins;
   }
 
   /// getStackAlignment - Returns the minimum alignment known to hold of the
   /// stack frame on entry to the function and which must be maintained by every

lib/Target/X86/X86Subtarget.h

public:
   /// compiler runtime or math libraries.
   bool hasSinCos() const;
 
   /// Enable the MachineScheduler pass for all X86 subtargets.
   bool enableMachineScheduler() const override { return true; }
 
   bool enableEarlyIfConversion() const override;
 
+  bool enableFullCopyPropagation() const override { return true; }
+
   /// Return the instruction itineraries based on the subtarget selection.
   const InstrItineraryData *getInstrItineraryData() const override {
     return &InstrItins;
   }
 
   AntiDepBreakMode getAntiDepBreakMode() const override {
     return TargetSubtargetInfo::ANTIDEP_CRITICAL;
   }
 };
 
 } // End llvm namespace
 
 #endif

test/CodeGen/AArch64/f16-instructions.ll

then:
   ret void
 else:
   store i32 0, i32* %p2
   ret void
 }
 
 ; CHECK-LABEL: test_phi:
 ; CHECK: mov  x[[PTR:[0-9]+]], x0
-; CHECK: ldr  h[[AB:[0-9]+]], [x[[PTR]]]
+; CHECK: ldr  h[[AB:[0-9]+]], [x0]
 ; CHECK: [[LOOP:LBB[0-9_]+]]:
 ; CHECK: mov.16b  v[[R:[0-9]+]], v[[AB]]
 ; CHECK: ldr  h[[AB]], [x[[PTR]]]
 ; CHECK: mov  x0, x[[PTR]]
 ; CHECK: bl {{_?}}test_dummy
 ; CHECK: mov.16b  v0, v[[R]]
 ; CHECK: ret
 define half @test_phi(half* %p1) #0 {

test/CodeGen/AArch64/flags-multiuse.ll

 ; RUN: llc -mtriple=aarch64-none-linux-gnu -aarch64-atomic-cfg-tidy=0 -verify-machineinstrs -o - %s | FileCheck %s
 
 ; LLVM should be able to cope with multiple uses of the same flag-setting
 ; instruction at different points of a routine. Either by rematerializing the
 ; compare or by saving and restoring the flag register.
 
 declare void @bar()
 
 @var = global i32 0
 
 define i32 @test_multiflag(i32 %n, i32 %m, i32 %o) {
 ; CHECK-LABEL: test_multiflag:
 
   %test = icmp ne i32 %n, %m
-; CHECK: cmp [[LHS:w[0-9]+]], [[RHS:w[0-9]+]]
+; CHECK: mov [[RHSCOPY:w[0-9]+]], [[RHS:w[0-9]+]]
+; CHECK: mov [[LHSCOPY:w[0-9]+]], [[LHS:w[0-9]+]]
+; CHECK: cmp [[LHS]], [[RHS]]
 
   %val = zext i1 %test to i32
 ; CHECK: cset {{[xw][0-9]+}}, ne
 
   store i32 %val, i32* @var
 
   call void @bar()
 ; CHECK: bl bar
 
   ; Currently, the comparison is emitted again. An MSR/MRS pair would also be
   ; acceptable, but assuming the call preserves NZCV is not.
   br i1 %test, label %iftrue, label %iffalse
-; CHECK: cmp [[LHS]], [[RHS]]
+; CHECK: cmp [[LHSCOPY]], [[RHSCOPY]]
 ; CHECK: b.eq
 
 iftrue:
   ret i32 42
 iffalse:
   ret i32 0
 }

test/CodeGen/AArch64/merge-store.ll

 ; PR26827 - Merge stores causes wrong dependency.
 %struct1 = type { %struct1*, %struct1*, i32, i32, i16, i16, void (i32, i32, i8*)*, i8* }
 @gv0 = internal unnamed_addr global i32 0, align 4
 @gv1 = internal unnamed_addr global %struct1** null, align 8
 
 define void @test(%struct1* %fde, i32 %fd, void (i32, i32, i8*)* %func, i8* %arg)  {
 ;CHECK-LABEL: test
 entry:
-;A53: mov [[DATA:w[0-9]+]], w1
 ;A53: str q{{[0-9]+}}, {{.*}}
 ;A53: str q{{[0-9]+}}, {{.*}}
-;A53: str [[DATA]], {{.*}}
+;A53: str w1, {{.*}}
 
   %0 = bitcast %struct1* %fde to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 40, i32 8, i1 false)
   %state = getelementptr inbounds %struct1, %struct1* %fde, i64 0, i32 4
   store i16 256, i16* %state, align 8
   %fd1 = getelementptr inbounds %struct1, %struct1* %fde, i64 0, i32 2
   store i32 %fd, i32* %fd1, align 8
   %force_eof = getelementptr inbounds %struct1, %struct1* %fde, i64 0, i32 3

test/CodeGen/AArch64/neg-imm.ll

 ; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs -o - %s | FileCheck %s
 ; LSR used to pick a sub-optimal solution due to the target responding
 ; conservatively to isLegalAddImmediate for negative values.
 
 declare void @foo(i32)
 
 define void @test(i32 %px) {
 ; CHECK_LABEL: test:
 ; CHECK_LABEL: %entry
-; CHECK: subs
-; CHECK-NEXT: csel
+; CHECK: subs [[REG0:w[0-9]+]],
+; CHECK: csel {{w[0-9]+}}, wzr, [[REG0]]
 entry:
   %sub = add nsw i32 %px, -1
   %cmp = icmp slt i32 %px, 1
   %.sub = select i1 %cmp, i32 0, i32 %sub
   br label %for.body
 
 for.body:
 ; CHECK_LABEL: %for.body

test/CodeGen/ARM/atomic-op.ll

entry:
   ret i32 %0
 }
 
 define i32 @test_cmpxchg_fail_order(i32 *%addr, i32 %desired, i32 %new) {
 ; CHECK-LABEL: test_cmpxchg_fail_order:
 
   %pair = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst monotonic
   %oldval = extractvalue { i32, i1 } %pair, 0
-; CHECK-ARMV7:     ldrex   [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]]
+; CHECK-ARMV7:     mov     r[[ADDR:[0-9]+]], r0
+; CHECK-ARMV7:     ldrex   [[OLDVAL:r[0-9]+]], [r0]
 ; CHECK-ARMV7:     cmp     [[OLDVAL]], r1
 ; CHECK-ARMV7:     bne     [[FAIL_BB:\.?LBB[0-9]+_[0-9]+]]
 ; CHECK-ARMV7:     dmb ish
 ; CHECK-ARMV7: [[LOOP_BB:\.?LBB.*]]:
 ; CHECK-ARMV7:     strex   [[SUCCESS:r[0-9]+]], r2, [r[[ADDR]]]
 ; CHECK-ARMV7:     cmp     [[SUCCESS]], #0
 ; CHECK-ARMV7:     beq     [[SUCCESS_BB:\.?LBB.*]]
 ; CHECK-ARMV7:     ldrex   [[OLDVAL]], [r[[ADDR]]]
 ; CHECK-ARMV7:     cmp     [[OLDVAL]], r1
 ; CHECK-ARMV7:     beq     [[LOOP_BB]]
 ; CHECK-ARMV7: [[FAIL_BB]]:
 ; CHECK-ARMV7:     clrex
 ; CHECK-ARMV7:     bx      lr
 ; CHECK-ARMV7: [[SUCCESS_BB]]:
 ; CHECK-ARMV7:     dmb     ish
 ; CHECK-ARMV7:     bx      lr
 
-; CHECK-T2:     ldrex   [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]]
+; CHECK-T2:     mov     r[[ADDR:[0-9]+]], r0
+; CHECK-T2:     ldrex   [[OLDVAL:r[0-9]+]], [r0]
 ; CHECK-T2:     cmp     [[OLDVAL]], r1
 ; CHECK-T2:     bne     [[FAIL_BB:\.?LBB.*]]
 ; CHECK-T2:     dmb ish
 ; CHECK-T2: [[LOOP_BB:\.?LBB.*]]:
 ; CHECK-T2:     strex   [[SUCCESS:r[0-9]+]], r2, [r[[ADDR]]]
 ; CHECK-T2:     cmp     [[SUCCESS]], #0
 ; CHECK-T2:     dmbeq   ish
 ; CHECK-T2:     bxeq    lr

test/CodeGen/ARM/swifterror.ll

 ; CHECK-APPLE-LABEL: foo_loop:
 ; CHECK-APPLE: mov [[CODE:r[0-9]+]], r0
 ; swifterror is kept in a register
 ; CHECK-APPLE: mov [[ID:r[0-9]+]], r6
 ; CHECK-APPLE: cmp [[CODE]], #0
 ; CHECK-APPLE: beq
 ; CHECK-APPLE: mov r0, #16
 ; CHECK-APPLE: malloc
-; CHECK-APPLE: strb r{{.*}}, [{{.*}}[[ID]], #8]
+; CHECK-APPLE: strb r{{.*}}, [r0, #8]
 ; CHECK-APPLE: ble
 ; CHECK-APPLE: mov r6, [[ID]]
 
 ; CHECK-O0-LABEL: foo_loop:
 ; CHECK-O0: mov r{{.*}}, r6
 ; CHECK-O0: cmp r{{.*}}, #0
 ; CHECK-O0: beq
 ; CHECK-O0-DAG: movw r{{.*}}, #1

test/CodeGen/Thumb/thumb-shrink-wrapping.ll

if.end:
   ret i32 42
 }
 
 declare void @abort() #0
 
 define i32 @b_to_bx(i32 %value) {
 ; CHECK-LABEL: b_to_bx:
 ; DISABLE: push {r7, lr}
-; CHECK: cmp r1, #49
+; CHECK: cmp r0, #49
 ; CHECK-NEXT: bgt [[ELSE_LABEL:LBB[0-9_]+]]
 ; ENABLE: push {r7, lr}
 
 ; CHECK: bl
 ; DISABLE-V5-NEXT: pop {r7, pc}
 ; DISABLE-V4T-NEXT: b [[END_LABEL:LBB[0-9_]+]]
 
 ; ENABLE-V5-NEXT: pop {r7, pc}

test/CodeGen/X86/avx512-bugfix-25270.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
 
 declare void @Print__512(<16 x i32>) #0
 
 define void @bar__512(<16 x i32>* %var) #0 {
 ; CHECK-LABEL: bar__512:
 ; CHECK:       ## BB#0: ## %allocas
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    subq $112, %rsp
 ; CHECK-NEXT:    movq %rdi, %rbx
-; CHECK-NEXT:    vmovdqu32 (%rbx), %zmm0
+; CHECK-NEXT:    vmovdqu32 (%rdi), %zmm0
 ; CHECK-NEXT:    vmovups %zmm0, (%rsp) ## 64-byte Spill
 ; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1
-; CHECK-NEXT:    vmovdqa32 %zmm1, (%rbx)
+; CHECK-NEXT:    vmovdqa32 %zmm1, (%rdi)
 ; CHECK-NEXT:    callq _Print__512
 ; CHECK-NEXT:    vmovups (%rsp), %zmm0 ## 64-byte Reload
 ; CHECK-NEXT:    callq _Print__512
 ; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0
 ; CHECK-NEXT:    vmovdqa32 %zmm0, (%rbx)
 ; CHECK-NEXT:    addq $112, %rsp
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq

test/CodeGen/X86/avx512-calling-conv.ll

 ; KNL_X32-NEXT:    .cfi_offset %ebx, -8
 ; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; KNL_X32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; KNL_X32-NEXT:    movl %edi, (%esp)
 ; KNL_X32-NEXT:    calll _test11
 ; KNL_X32-NEXT:    movl %eax, %ebx
-; KNL_X32-NEXT:    movzbl %bl, %eax
+; KNL_X32-NEXT:    movzbl %al, %eax
 ; KNL_X32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; KNL_X32-NEXT:    movl %esi, {{[0-9]+}}(%esp)
 ; KNL_X32-NEXT:    movl %edi, (%esp)
 ; KNL_X32-NEXT:    calll _test10
 ; KNL_X32-NEXT:    xorl %ecx, %ecx
 ; KNL_X32-NEXT:    testb $1, %bl
 ; KNL_X32-NEXT:    cmovel %ecx, %eax
 ; KNL_X32-NEXT:    addl $16, %esp

test/CodeGen/X86/fp128-i128.ll

 entry:
   %0 = bitcast fp128 %x to i128
   %bf.clear = and i128 %0, -18446744073709551616
   %1 = bitcast i128 %bf.clear to fp128
   %add = fadd fp128 %1, %x
   ret fp128 %add
 ; CHECK-LABEL: TestI128_4:
 ; CHECK:       movaps %xmm0, %xmm1
-; CHECK-NEXT:  movaps %xmm1, 16(%rsp)
+; CHECK-NEXT:  movaps %xmm0, 16(%rsp)
 ; CHECK-NEXT:  movq 24(%rsp), %rax
 ; CHECK-NEXT:  movq %rax, 8(%rsp)
 ; CHECK-NEXT:  movq $0, (%rsp)
 ; CHECK-NEXT:  movaps (%rsp), %xmm0
 ; CHECK-NEXT:  callq __addtf3
 ; CHECK:       retq
 }
 

 entry:
   %0 = bitcast fp128 %x to i128
   %bf.clear = and i128 %0, -18446744073709551616
   %1 = bitcast i128 %bf.clear to fp128
   %add = fadd fp128 %1, %x
   ret fp128 %add
 ; CHECK-LABEL: acosl:
 ; CHECK:       movaps %xmm0, %xmm1
-; CHECK-NEXT:  movaps %xmm1, 16(%rsp)
+; CHECK-NEXT:  movaps %xmm0, 16(%rsp)
 ; CHECK-NEXT:  movq 24(%rsp), %rax
 ; CHECK-NEXT:  movq %rax, 8(%rsp)
 ; CHECK-NEXT:  movq $0, (%rsp)
 ; CHECK-NEXT:  movaps (%rsp), %xmm0
 ; CHECK-NEXT:  callq __addtf3
 ; CHECK:       retq
 }
 

test/CodeGen/X86/localescape.ll

define void @print_framealloc_from_fp(i8* %fp) {
   %b2 = getelementptr i32, i32* %b, i32 1
   %b2.val = load i32, i32* %b2
   call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %b2.val)
   ret void
 }
 
 ; X64-LABEL: print_framealloc_from_fp:
 ; X64: movq %rcx, %[[parent_fp:[a-z]+]]
-; X64: movl .Lalloc_func$frame_escape_0(%[[parent_fp]]), %edx
+; X64: movl .Lalloc_func$frame_escape_0(%rcx), %edx
 ; X64: leaq {{.*}}(%rip), %[[str:[a-z]+]]
 ; X64: movq %[[str]], %rcx
 ; X64: callq printf
 ; X64: movl .Lalloc_func$frame_escape_1(%[[parent_fp]]), %edx
 ; X64: movq %[[str]], %rcx
 ; X64: callq printf
 ; X64: movl    $42, .Lalloc_func$frame_escape_1(%[[parent_fp]])
 ; X64: retq

test/CodeGen/X86/mul128.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
 
 define i128 @foo(i128 %t, i128 %u) {
 ; X64-LABEL: foo:
 ; X64:       # BB#0:
 ; X64-NEXT:    movq %rdx, %r8
 ; X64-NEXT:    imulq %rdi, %rcx
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    mulq %r8
+; X64-NEXT:    mulq %rdx
 ; X64-NEXT:    addq %rcx, %rdx
 ; X64-NEXT:    imulq %r8, %rsi
 ; X64-NEXT:    addq %rsi, %rdx
 ; X64-NEXT:    retq
   %k = mul i128 %t, %u
   ret i128 %k
 }

test/CodeGen/X86/pmul.ll

 ; SSE2-NEXT:    pand %xmm5, %xmm3
 ; SSE2-NEXT:    packuswb %xmm6, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: mul_v64i8c:
 ; SSE41:       # BB#0: # %entry
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
 ; SSE41-NEXT:    pmovsxbw {{.*}}(%rip), %xmm6
 ; SSE41-NEXT:    pmullw %xmm6, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
 ; SSE41-NEXT:    pand %xmm7, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
 ; SSE41-NEXT:    pmullw %xmm6, %xmm1
 ; SSE41-NEXT:    pand %xmm7, %xmm1

 ; SSE2-NEXT:    packuswb %xmm5, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: mul_v64i8:
 ; SSE41:       # BB#0: # %entry
 ; SSE41-NEXT:    movdqa %xmm1, %xmm8
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pmovsxbw %xmm4, %xmm9
-; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
+; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
 ; SSE41-NEXT:    pmullw %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
 ; SSE41-NEXT:    pand %xmm9, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
 ; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
 ; SSE41-NEXT:    pmullw %xmm4, %xmm1

test/CodeGen/X86/pr11415.ll

 ; RUN: llc -mtriple=x86_64-pc-linux %s -o - -regalloc=fast -optimize-regalloc=0 | FileCheck %s
 
 ; We used to consider the early clobber in the second asm statement as
 ; defining %0 before it was read. This caused us to omit the
 ; movq  -8(%rsp), %rdx
 
 ; CHECK:  #APP
 ; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: movq  %rcx, %rax
-; CHECK-NEXT: movq  %rax, -8(%rsp)
+; CHECK-NEXT: movq  %rcx, -8(%rsp)
 ; CHECK-NEXT: movq  -8(%rsp), %rdx
 ; CHECK-NEXT: #APP
 ; CHECK-NEXT: #NO_APP
 ; CHECK-NEXT: movq  %rdx, %rax
 ; CHECK-NEXT: movq  %rdx, -8(%rsp)
 ; CHECK-NEXT: ret
 
 define i64 @foo() {
 entry:
   %0 = tail call i64 asm "", "={cx}"() nounwind
   %1 = tail call i64 asm "", "=&r,0,r,~{rax}"(i64 %0, i64 %0) nounwind
   ret i64 %1
 }

test/CodeGen/X86/shrink-wrap-chkstk.ll

 
 false:
   %tmp.0 = phi i32 [ %tmp4, %true ], [ %a, %0 ]
   ret i32 %tmp.0
 }
 
 ; CHECK-LABEL: @use_eax_before_prologue@8: # @use_eax_before_prologue
 ; CHECK: movl %ecx, %eax
-; CHECK: cmpl %edx, %eax
+; CHECK: cmpl %edx, %ecx
 ; CHECK: jge LBB1_2
 ; CHECK: pushl %eax
 ; CHECK: movl $4092, %eax
 ; CHECK: calll __chkstk
 ; CHECK: movl 4092(%esp), %eax
 ; CHECK: calll _doSomething
 ; CHECK: LBB1_2:
 ; CHECK: retl

test/CodeGen/X86/vec_int_to_fp.ll

 ;
 ; Unsigned Integer to Float
 ;
 
 define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 ; SSE-LABEL: uitofp_2i64_to_4f32:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
-; SSE-NEXT:    movd %xmm1, %rax
+; SSE-NEXT:    movd %xmm0, %rax
 ; SSE-NEXT:    movl %eax, %ecx
 ; SSE-NEXT:    andl $1, %ecx
 ; SSE-NEXT:    testq %rax, %rax
 ; SSE-NEXT:    js .LBB38_1
 ; SSE-NEXT:  # BB#2:
 ; SSE-NEXT:    xorps %xmm0, %xmm0
 ; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
 ; SSE-NEXT:    jmp .LBB38_3

test/CodeGen/X86/vec_minmax_sint.ll

 ; SSE41-NEXT:    blendvpd %xmm2, %xmm1
 ; SSE41-NEXT:    movapd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; SSE42-LABEL: max_ge_v2i64:
 ; SSE42:       # BB#0:
 ; SSE42-NEXT:    movdqa %xmm0, %xmm2
 ; SSE42-NEXT:    movdqa %xmm1, %xmm3
-; SSE42-NEXT:    pcmpgtq %xmm2, %xmm3
+; SSE42-NEXT:    pcmpgtq %xmm0, %xmm3
 ; SSE42-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE42-NEXT:    pxor %xmm3, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm2, %xmm1
 ; SSE42-NEXT:    movapd %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX-LABEL: max_ge_v2i64:
 ; AVX:       # BB#0:

test/CodeGen/X86/vector-blend.ll

 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: blend_neg_logic_v4i32_2:
 ; SSE41:       # BB#0: # %entry
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    psrld $31, %xmm1
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubd %xmm2, %xmm3
+; SSE41-NEXT:    psubd %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    blendvps %xmm2, %xmm3
 ; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: blend_neg_logic_v4i32_2:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vpsrld $31, %xmm1, %xmm1

test/CodeGen/X86/vector-sext.ll

 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_16i8_to_8i32:
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSSE3-NEXT:    psrad $24, %xmm0
 ; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[u,u,u,4,u,u,u,5,u,u,u,6,u,u,u,7]
 ; SSSE3-NEXT:    psrad $24, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: sext_16i8_to_8i32:
 ; SSE41:       # BB#0: # %entry
entry:
   %C = sext <4 x i8> %B to <4 x i64>
   ret <4 x i64> %C
 }
 
 define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: sext_16i8_to_8i64:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrad $31, %xmm2
 ; SSE2-NEXT:    psrad $24, %xmm0
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]

 ; SSE2-NEXT:    psrad $31, %xmm4
 ; SSE2-NEXT:    psrad $24, %xmm3
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_16i8_to_8i64:
 ; SSSE3:       # BB#0: # %entry
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm2
 ; SSSE3-NEXT:    psrad $31, %xmm2
 ; SSSE3-NEXT:    psrad $24, %xmm0
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
 ; SSSE3-NEXT:    psrld $16, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]

test/CodeGen/X86/vector-zext.ll

 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: zext_16i8_to_16i16:
 ; SSE41:       # BB#0: # %entry
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: zext_16i8_to_16i16:
 ; AVX1:       # BB#0: # %entry
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero

 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: zext_8i16_to_8i32:
 ; SSE41:       # BB#0: # %entry
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: zext_8i16_to_8i32:
 ; AVX1:       # BB#0: # %entry
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero

 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: zext_4i32_to_4i64:
 ; SSE41:       # BB#0: # %entry
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; SSE41-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: zext_4i32_to_4i64:
 ; AVX1:       # BB#0: # %entry
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero

 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuf_zext_8i16_to_8i32:
 ; SSE41:       # BB#0: # %entry
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: shuf_zext_8i16_to_8i32:
 ; AVX1:       # BB#0: # %entry
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero

 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuf_zext_4i32_to_4i64:
 ; SSE41:       # BB#0: # %entry
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; SSE41-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: shuf_zext_4i32_to_4i64:
 ; AVX1:       # BB#0: # %entry
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero
 ; AVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vblendpd {{.*#+}} xmm0 = xmm2[0],xmm0[1]

test/CodeGen/X86/vselect-minmax.ll

 
 define <64 x i8> @test100(<64 x i8> %a, <64 x i8> %b) {
 ; SSE2-LABEL: test100:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm3, %xmm8
 ; SSE2-NEXT:    movdqa %xmm2, %xmm9
 ; SSE2-NEXT:    movdqa %xmm0, %xmm10
 ; SSE2-NEXT:    movdqa %xmm7, %xmm12
-; SSE2-NEXT:    pcmpgtb %xmm8, %xmm12
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm12
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE2-NEXT:    movdqa %xmm12, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa %xmm6, %xmm13
-; SSE2-NEXT:    pcmpgtb %xmm9, %xmm13
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm13
 ; SSE2-NEXT:    movdqa %xmm13, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
 ; SSE2-NEXT:    movdqa %xmm5, %xmm14
 ; SSE2-NEXT:    pcmpgtb %xmm1, %xmm14
 ; SSE2-NEXT:    movdqa %xmm14, %xmm11
 ; SSE2-NEXT:    pxor %xmm0, %xmm11
 ; SSE2-NEXT:    movdqa %xmm4, %xmm15
 ; SSE2-NEXT:    pcmpgtb %xmm10, %xmm15

 
 define <16 x i32> @test116(<16 x i32> %a, <16 x i32> %b) {
 ; SSE2-LABEL: test116:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm3, %xmm8
 ; SSE2-NEXT:    movdqa %xmm2, %xmm9
 ; SSE2-NEXT:    movdqa %xmm0, %xmm10
 ; SSE2-NEXT:    movdqa %xmm7, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm12
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE2-NEXT:    movdqa %xmm12, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa %xmm6, %xmm13
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm13
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm13
 ; SSE2-NEXT:    movdqa %xmm13, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
 ; SSE2-NEXT:    movdqa %xmm5, %xmm14
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm14
 ; SSE2-NEXT:    movdqa %xmm14, %xmm11
 ; SSE2-NEXT:    pxor %xmm0, %xmm11
 ; SSE2-NEXT:    movdqa %xmm4, %xmm15
 ; SSE2-NEXT:    pcmpgtd %xmm10, %xmm15
entry:
   %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b
   ret <8 x i64> %sel
 }
 
 define <8 x i64> @test122(<8 x i64> %a, <8 x i64> %b) {
 ; SSE2-LABEL: test122:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
-; SSE2-NEXT:    movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm7, -{{[0-9]+}}(%rsp) # 16-byte Spill
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,0,2147483648,0]
 ; SSE2-NEXT:    pxor %xmm10, %xmm8
 ; SSE2-NEXT:    movdqa %xmm7, %xmm0
 ; SSE2-NEXT:    pxor %xmm10, %xmm0
entry:
   %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b
   ret <8 x i64> %sel
 }
 
 define <8 x i64> @test124(<8 x i64> %a, <8 x i64> %b) {
 ; SSE2-LABEL: test124:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm7, %xmm11
-; SSE2-NEXT:    movdqa %xmm11, -{{[0-9]+}}(%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm7, -{{[0-9]+}}(%rsp) # 16-byte Spill
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,0,2147483648,0]
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
 ; SSE2-NEXT:    pxor %xmm10, %xmm8
 ; SSE2-NEXT:    movdqa %xmm11, %xmm0
entry:
   %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b
   ret <8 x i64> %sel
 }
 
 define <8 x i64> @test126(<8 x i64> %a, <8 x i64> %b) {
 ; SSE2-LABEL: test126:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
-; SSE2-NEXT:    movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm7, -{{[0-9]+}}(%rsp) # 16-byte Spill
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    pxor %xmm10, %xmm8
 ; SSE2-NEXT:    movdqa %xmm7, %xmm0
 ; SSE2-NEXT:    pxor %xmm10, %xmm0
entry:
   %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b
   ret <8 x i64> %sel
 }
 
 define <8 x i64> @test128(<8 x i64> %a, <8 x i64> %b) {
 ; SSE2-LABEL: test128:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm7, %xmm11
-; SSE2-NEXT:    movdqa %xmm11, -{{[0-9]+}}(%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm7, -{{[0-9]+}}(%rsp) # 16-byte Spill
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
 ; SSE2-NEXT:    pxor %xmm10, %xmm8
 ; SSE2-NEXT:    movdqa %xmm11, %xmm0

 ; SSE2-NEXT:    movdqa %xmm2, %xmm8
 ; SSE2-NEXT:    movdqa %xmm0, %xmm10
 ; SSE2-NEXT:    movdqa %xmm7, %xmm12
 ; SSE2-NEXT:    pcmpgtb %xmm3, %xmm12
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE2-NEXT:    movdqa %xmm12, %xmm9
 ; SSE2-NEXT:    pxor %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa %xmm6, %xmm13
-; SSE2-NEXT:    pcmpgtb %xmm8, %xmm13
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm13
 ; SSE2-NEXT:    movdqa %xmm13, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
 ; SSE2-NEXT:    movdqa %xmm5, %xmm14
 ; SSE2-NEXT:    pcmpgtb %xmm1, %xmm14
 ; SSE2-NEXT:    movdqa %xmm14, %xmm11
 ; SSE2-NEXT:    pxor %xmm0, %xmm11
 ; SSE2-NEXT:    movdqa %xmm4, %xmm15
 ; SSE2-NEXT:    pcmpgtb %xmm10, %xmm15

 ; SSE2-NEXT:    movdqa %xmm2, %xmm8
 ; SSE2-NEXT:    movdqa %xmm0, %xmm10
 ; SSE2-NEXT:    movdqa %xmm7, %xmm12
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm12
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE2-NEXT:    movdqa %xmm12, %xmm9
 ; SSE2-NEXT:    pxor %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa %xmm6, %xmm13
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm13
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm13
 ; SSE2-NEXT:    movdqa %xmm13, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
 ; SSE2-NEXT:    movdqa %xmm5, %xmm14
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm14
 ; SSE2-NEXT:    movdqa %xmm14, %xmm11
 ; SSE2-NEXT:    pxor %xmm0, %xmm11
 ; SSE2-NEXT:    movdqa %xmm4, %xmm15
 ; SSE2-NEXT:    pcmpgtd %xmm10, %xmm15
entry:
   %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a
   ret <8 x i64> %sel
 }
 
 define <8 x i64> @test154(<8 x i64> %a, <8 x i64> %b) {
 ; SSE2-LABEL: test154:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
-; SSE2-NEXT:    movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm7, -{{[0-9]+}}(%rsp) # 16-byte Spill
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,0,2147483648,0]
 ; SSE2-NEXT:    pxor %xmm10, %xmm8
 ; SSE2-NEXT:    movdqa %xmm7, %xmm0
 ; SSE2-NEXT:    pxor %xmm10, %xmm0
entry:
   %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a
   ret <8 x i64> %sel
 }
 
 define <8 x i64> @test156(<8 x i64> %a, <8 x i64> %b) {
 ; SSE2-LABEL: test156:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm7, %xmm11
-; SSE2-NEXT:    movdqa %xmm11, -{{[0-9]+}}(%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm7, -{{[0-9]+}}(%rsp) # 16-byte Spill
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,0,2147483648,0]
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
 ; SSE2-NEXT:    pxor %xmm10, %xmm8
 ; SSE2-NEXT:    movdqa %xmm11, %xmm0
entry:
   %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a
   ret <8 x i64> %sel
 }
 
 define <8 x i64> @test158(<8 x i64> %a, <8 x i64> %b) {
 ; SSE2-LABEL: test158:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
-; SSE2-NEXT:    movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm7, -{{[0-9]+}}(%rsp) # 16-byte Spill
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    pxor %xmm10, %xmm8
 ; SSE2-NEXT:    movdqa %xmm7, %xmm0
 ; SSE2-NEXT:    pxor %xmm10, %xmm0
entry:
   %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a
   ret <8 x i64> %sel
 }
 
 define <8 x i64> @test160(<8 x i64> %a, <8 x i64> %b) {
 ; SSE2-LABEL: test160:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    movdqa %xmm7, %xmm11
-; SSE2-NEXT:    movdqa %xmm11, -{{[0-9]+}}(%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm7, -{{[0-9]+}}(%rsp) # 16-byte Spill
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
 ; SSE2-NEXT:    pxor %xmm10, %xmm8
 ; SSE2-NEXT:    movdqa %xmm11, %xmm0

 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: test180:
 ; SSE4:       # BB#0: # %entry
 ; SSE4-NEXT:    movdqa %xmm0, %xmm2
 ; SSE4-NEXT:    movdqa %xmm1, %xmm3
-; SSE4-NEXT:    pcmpgtq %xmm2, %xmm3
+; SSE4-NEXT:    pcmpgtq %xmm0, %xmm3
 ; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE4-NEXT:    pxor %xmm3, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm2, %xmm1
 ; SSE4-NEXT:    movapd %xmm1, %xmm0
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test180:
 ; AVX1:       # BB#0: # %entry

 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: test188:
 ; SSE4:       # BB#0: # %entry
 ; SSE4-NEXT:    movdqa %xmm0, %xmm2
 ; SSE4-NEXT:    movdqa %xmm1, %xmm3
-; SSE4-NEXT:    pcmpgtq %xmm2, %xmm3
+; SSE4-NEXT:    pcmpgtq %xmm0, %xmm3
 ; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE4-NEXT:    pxor %xmm3, %xmm0
 ; SSE4-NEXT:    blendvpd %xmm1, %xmm2
 ; SSE4-NEXT:    movapd %xmm2, %xmm0
 ; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test188:
 ; AVX1:       # BB#0: # %entry

test/CodeGen/X86/x86-shrink-wrap-unwind.ll

 ;
 ; Prologue code.
 ; (What we push does not matter. It should be some random sratch register.)
 ; CHECK: pushq
 ;
 ; Compare the arguments and jump to exit.
 ; After the prologue is set.
 ; CHECK: movl %edi, [[ARG0CPY:%e[a-z]+]]
-; CHECK-NEXT: cmpl %esi, [[ARG0CPY]]
+; CHECK-NEXT: cmpl %esi, %edi
 ; CHECK-NEXT: jge [[EXIT_LABEL:LBB[0-9_]+]]
 ;
 ; Store %a in the alloca.
 ; CHECK: movl [[ARG0CPY]], 4(%rsp)
 ; Set the alloca address in the second argument.
 ; CHECK-NEXT: leaq 4(%rsp), %rsi
 ; Set the first argument to zero.
 ; CHECK-NEXT: xorl %edi, %edi

 ; CHECK-LABEL: frameUnwind:
 ;
 ; Compare the arguments and jump to exit.
 ; No prologue needed.
 ;
 ; Compare the arguments and jump to exit.
 ; After the prologue is set.
 ; CHECK: movl %edi, [[ARG0CPY:%e[a-z]+]]
-; CHECK-NEXT: cmpl %esi, [[ARG0CPY]]
+; CHECK-NEXT: cmpl %esi, %edi
 ; CHECK-NEXT: jge [[EXIT_LABEL:LBB[0-9_]+]]
 ;
 ; Prologue code.
 ; CHECK: pushq %rbp
 ; CHECK: movq %rsp, %rbp
 ;
 ; Store %a in the alloca.
 ; CHECK: movl [[ARG0CPY]], -4(%rbp)

 ; CHECK-LABEL: framelessnoUnwind:
 ;
 ; Compare the arguments and jump to exit.
 ; No prologue needed.
 ;
 ; Compare the arguments and jump to exit.
 ; After the prologue is set.
 ; CHECK: movl %edi, [[ARG0CPY:%e[a-z]+]]
-; CHECK-NEXT: cmpl %esi, [[ARG0CPY]]
+; CHECK-NEXT: cmpl %esi, %edi
 ; CHECK-NEXT: jge [[EXIT_LABEL:LBB[0-9_]+]]
 ;
 ; Prologue code.
 ; (What we push does not matter. It should be some random sratch register.)
 ; CHECK: pushq
 ;
 ; Store %a in the alloca.
 ; CHECK: movl [[ARG0CPY]], 4(%rsp)

test/CodeGen/X86/x86-shrink-wrapping.ll

 
 
 ; Initial motivating example: Simple diamond with a call just on one side.
 ; CHECK-LABEL: foo:
 ;
 ; Compare the arguments and jump to exit.
 ; No prologue needed.
 ; ENABLE: movl %edi, [[ARG0CPY:%e[a-z]+]]
-; ENABLE-NEXT: cmpl %esi, [[ARG0CPY]]
+; ENABLE-NEXT: cmpl %esi, %edi
 ; ENABLE-NEXT: jge [[EXIT_LABEL:LBB[0-9_]+]]
 ;
 ; Prologue code.
 ; (What we push does not matter. It should be some random sratch register.)
 ; CHECK: pushq
 ;
 ; Compare the arguments and jump to exit.
 ; After the prologue is set.
 ; DISABLE: movl %edi, [[ARG0CPY:%e[a-z]+]]
-; DISABLE-NEXT: cmpl %esi, [[ARG0CPY]]
+; DISABLE-NEXT: cmpl %esi, %edi
 ; DISABLE-NEXT: jge [[EXIT_LABEL:LBB[0-9_]+]]
 ;
 ; Store %a in the alloca.
 ; CHECK: movl [[ARG0CPY]], 4(%rsp)
 ; Set the alloca address in the second argument.
 ; CHECK-NEXT: leaq 4(%rsp), %rsi
 ; Set the first argument to zero.
 ; CHECK-NEXT: xorl %edi, %edi