[X86][DAG] Switch X86 Target to post-legalized store merge

[niravd]

Summary

Move store merge to happen after intrinsic lowering to allow lowered
stores to be merged.

Some regressions due in MergeConsecutiveStores to missing
insert_subvector that are addressed in follow up patch.

[niravd]

Rebase on improved bitcast store

[RKSimon]

test/CodeGen/X86/oddshuffles.ll

86 ↗

(On Diff #103762)

This is scaring me - what is the reasoning behind permitting the store of a <4 x i32> and not just the <3 x i32> ?

[niravd]

Rebase given corrected D34569 patch which was failing to check candidate store sizes in some merging cases.

[niravd]

test/CodeGen/X86/oddshuffles.ll

86 ↗

(On Diff #103762)

This is due to a mistake in a parent patch which dropped a type size check. It should be fixed now.

[niravd]

Rebase and general update. No real change.

[RKSimon]

lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp

61 ↗

(On Diff #108511)

One use and fits on the line, so we can just do:

SDValue Base = DAG.getTargetLoweringInfo().unwrapAddress(Ptr);

test/CodeGen/X86/bitcast-i256.ll

25 ↗

(On Diff #108511)

Get rid of this - did the script add it?

[niravd]

Address comments.

[RKSimon]

test/CodeGen/X86/bigstructret.ll

1 ↗

(On Diff #108915)

I've regenerated this test file, so it needs rebasing.

test/CodeGen/X86/extract-store.ll

528 ↗

(On Diff #108915)

Any idea why this still doesn't merge? Same for @extract_f128_1 as well.

[niravd]

Update test.

[niravd]

test/CodeGen/X86/extract-store.ll

528 ↗

(On Diff #108915)

At merge time the component stores have an alignment of 1 and TLI.allowsMisalignedMemoryAccess believe this is legal but slow because Subtarget.isUnalignedMem16Slow() is set so we don't bother merging.

[RKSimon]

LGTM

[RKSimon]

@niravd Anything stopping this going in?

llvm/trunk/include/llvm/Target/TargetLowering.h

public:
   bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool Signed) const;
 
   /// Try to simplify a setcc built with the specified operands and cc. If it is
   /// unable to simplify it, return a null SDValue.
   SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
                         bool foldBooleans, DAGCombinerInfo &DCI,
                         const SDLoc &dl) const;
 
+  // For targets which wrap address, unwrap for analysis.
+  virtual SDValue unwrapAddress(SDValue N) const { return N; }
+
   /// Returns true (and the GlobalValue and the offset) if the node is a
   /// GlobalAddress + offset.
   virtual bool
   isGAPlusOffset(SDNode *N, const GlobalValue* &GA, int64_t &Offset) const;
 
   /// This method will be invoked for all target nodes and for any
   /// target-independent nodes that the target has registered with invoke it
   /// for.

llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp

 //===-- llvm/CodeGen/SelectionDAGAddressAnalysis.cpp ------- DAG Address
 //Analysis ---*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/Target/TargetLowering.h"
 
 namespace llvm {
 
 bool BaseIndexOffset::equalBaseIndex(BaseIndexOffset &Other,
                                      const SelectionDAG &DAG, int64_t &Off) {
   // Initial Offset difference.
   Off = Other.Offset - Offset;
 
if (auto *A = dyn_cast<FrameIndexSDNode>(Base))
         }
   }
   return false;
 }
 
 /// Parses tree in Ptr for base, index, offset addresses.
 BaseIndexOffset BaseIndexOffset::match(SDValue Ptr, const SelectionDAG &DAG) {
   // (((B + I*M) + c)) + c ...
-  SDValue Base = Ptr;
+  SDValue Base = DAG.getTargetLoweringInfo().unwrapAddress(Ptr);
   SDValue Index = SDValue();
   int64_t Offset = 0;
   bool IsIndexSignExt = false;
 
   // Consume constant adds & ors with appropriate masking.
   while (Base->getOpcode() == ISD::ADD || Base->getOpcode() == ISD::OR) {
     if (auto *C = dyn_cast<ConstantSDNode>(Base->getOperand(1))) {
       // Only consider ORs which act as adds.

llvm/trunk/lib/Target/X86/X86ISelLowering.h

public:
 
     MachineBasicBlock *
     EmitInstrWithCustomInserter(MachineInstr &MI,
                                 MachineBasicBlock *MBB) const override;
 
     /// This method returns the name of a target specific DAG node.
     const char *getTargetNodeName(unsigned Opcode) const override;
 
+    bool mergeStoresAfterLegalization() const override { return true; }
+
     bool isCheapToSpeculateCttz() const override;
 
     bool isCheapToSpeculateCtlz() const override;
 
     bool isCtlzFast() const override;
 
     bool hasBitPreservingFPLogic(EVT VT) const override {
       return VT == MVT::f32 || VT == MVT::f64 || VT.isVector();
void computeKnownBitsForTargetNode(const SDValue Op,
                                        unsigned Depth = 0) const override;
 
     /// Determine the number of bits in the operation that are sign bits.
     unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
                                              const APInt &DemandedElts,
                                              const SelectionDAG &DAG,
                                              unsigned Depth) const override;
 
+    SDValue unwrapAddress(SDValue N) const override;
+
     bool isGAPlusOffset(SDNode *N, const GlobalValue* &GA,
                         int64_t &Offset) const override;
 
     SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
 
     bool ExpandInlineAsm(CallInst *CI) const override;
 
     ConstraintType getConstraintType(StringRef Constraint) const override;

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

case X86ISD::VPCOMU:
     // Vector compares return zero/all-bits result values.
     return VTBits;
   }
 
   // Fallback case.
   return 1;
 }
 
+SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
+  if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP)
+    return N->getOperand(0);
+  return N;
+}
+
 /// Returns true (and the GlobalValue and the offset) if the node is a
 /// GlobalAddress + offset.
 bool X86TargetLowering::isGAPlusOffset(SDNode *N,
                                        const GlobalValue* &GA,
                                        int64_t &Offset) const {
   if (N->getOpcode() == X86ISD::Wrapper) {
     if (isa<GlobalAddressSDNode>(N->getOperand(0))) {
       GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();

llvm/trunk/test/CodeGen/X86/MergeConsecutiveStores.ll

define void @merge_vec_element_store(<8 x float> %v, float* %ptr) {
   store float %vecext2, float* %arrayidx2, align 4
   store float %vecext3, float* %arrayidx3, align 4
   store float %vecext4, float* %arrayidx4, align 4
   store float %vecext5, float* %arrayidx5, align 4
   store float %vecext6, float* %arrayidx6, align 4
   store float %vecext7, float* %arrayidx7, align 4
   ret void
 
-; CHECK-LABEL: merge_vec_element_store
-; CHECK: vmovups
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; CHECK: vextractf128 $1, %ymm0, %xmm1
+; CHECK: vinsertf128  $1, %xmm1, %ymm0, %ymm0
+; CHECK: retq
+
+; This is what should be generated:
+; FIXME-LABEL: merge_vec_element_store
+; FIXME: vmovups
+; FIXME-NEXT: vzeroupper
+; FIXME-NEXT: retq
 }
 
 ; PR21711 - Merge vector stores into wider vector stores.
 ; These should be merged into 32-byte stores.
 define void @merge_vec_extract_stores(<8 x float> %v1, <8 x float> %v2, <4 x float>* %ptr) {
   %idx0 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 3
   %idx1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 4
   %idx2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 5
   %idx3 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 6
   %shuffle0 = shufflevector <8 x float> %v1, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle1 = shufflevector <8 x float> %v1, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %shuffle2 = shufflevector <8 x float> %v2, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %shuffle3 = shufflevector <8 x float> %v2, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   store <4 x float> %shuffle0, <4 x float>* %idx0, align 16
   store <4 x float> %shuffle1, <4 x float>* %idx1, align 16
   store <4 x float> %shuffle2, <4 x float>* %idx2, align 16
   store <4 x float> %shuffle3, <4 x float>* %idx3, align 16
   ret void
 
-; CHECK-LABEL: merge_vec_extract_stores
-; CHECK:      vmovups %ymm0, 48(%rdi)
-; CHECK-NEXT: vmovups %ymm1, 80(%rdi)
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; These vblendpd are obviously redundant.
+; CHECK: vblendpd $12, %ymm0, %ymm0, %ymm0 # ymm0 = ymm0[0,1,2,3]
+; CHECK: vmovupd  %ymm0, 48(%rdi)
+; CHECK: vblendpd $12, %ymm1, %ymm1, %ymm0 # ymm0 = ymm1[0,1,2,3]
+; CHECK: vmovupd  %ymm0, 80(%rdi)
+
+; This is what should be generated:
+; FIXME-LABEL: merge_vec_extract_stores
+; FIXME:      vmovups %ymm0, 48(%rdi)
+; FIXME-NEXT: vmovups %ymm1, 80(%rdi)
+; FIXME-NEXT: vzeroupper
+; FIXME-NEXT: retq
 }
 
 ; Merging vector stores when sourced from vector loads.
 define void @merge_vec_stores_from_loads(<4 x float>* %v, <4 x float>* %ptr) {
   %load_idx0 = getelementptr inbounds <4 x float>, <4 x float>* %v, i64 0
   %load_idx1 = getelementptr inbounds <4 x float>, <4 x float>* %v, i64 1
   %v0 = load <4 x float>, <4 x float>* %load_idx0
   %v1 = load <4 x float>, <4 x float>* %load_idx1

 ; CHECK-LABEL: merge_vec_stores_of_constants
 ; CHECK:      vxorps
 ; CHECK-NEXT: vmovaps
 ; CHECK-NEXT: vmovaps
 ; CHECK-NEXT: retq
 }
 
 ; This is a minimized test based on real code that was failing.
-; We could merge stores (and loads) like this...
-
+; This should now be merged.
 define void @merge_vec_element_and_scalar_load([6 x i64]* %array) {
   %idx0 = getelementptr inbounds [6 x i64], [6 x i64]* %array, i64 0, i64 0
   %idx1 = getelementptr inbounds [6 x i64], [6 x i64]* %array, i64 0, i64 1
   %idx4 = getelementptr inbounds [6 x i64], [6 x i64]* %array, i64 0, i64 4
   %idx5 = getelementptr inbounds [6 x i64], [6 x i64]* %array, i64 0, i64 5
 
   %a0 = load i64, i64* %idx0, align 8
   store i64 %a0, i64* %idx4, align 8
 
   %b = bitcast i64* %idx1 to <2 x i64>*
   %v = load <2 x i64>, <2 x i64>* %b, align 8
   %a1 = extractelement <2 x i64> %v, i32 0
   store i64 %a1, i64* %idx5, align 8
   ret void
 
 ; CHECK-LABEL: merge_vec_element_and_scalar_load
-; CHECK:      movq  (%rdi), %rax
-; CHECK-NEXT: movq  8(%rdi), %rcx
-; CHECK-NEXT: movq  %rax, 32(%rdi)
-; CHECK-NEXT: movq  %rcx, 40(%rdi)
+; CHECK:      vmovups (%rdi), %xmm0
+; CHECK-NEXT: vmovups %xmm0, 32(%rdi)
 ; CHECK-NEXT: retq
 }
 
 ; Don't let a non-consecutive store thwart merging of the last two.
 define void @almost_consecutive_stores(i8* %p) {
   store i8 0, i8* %p
   %p1 = getelementptr i8, i8* %p, i64 42
   store i8 1, i8* %p1

llvm/trunk/test/CodeGen/X86/bigstructret.ll

 entry:
   %0 = insertvalue %0 zeroinitializer, i32 12, 0
   %1 = insertvalue %0 %0, i32 24, 1
   %2 = insertvalue %0 %1, i32 48, 2
   %3 = insertvalue %0 %2, i32 24601, 3
   ret %0 %3
 }
 
+
 define fastcc %1 @ReturnBigStruct2() nounwind readnone {
 ; X86-LABEL: ReturnBigStruct2:
 ; X86:       # BB#0: # %entry
 ; X86-NEXT:    movl $48, 4(%ecx)
 ; X86-NEXT:    movb $1, 2(%ecx)
-; X86-NEXT:    movb $1, 1(%ecx)
-; X86-NEXT:    movb $0, (%ecx)
+; X86-NEXT:    movw $256, (%ecx) # imm = 0x100
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: ReturnBigStruct2:
 ; X64:       # BB#0: # %entry
 ; X64-NEXT:    movl $48, 4(%rdi)
 ; X64-NEXT:    movb $1, 2(%rdi)
-; X64-NEXT:    movb $1, 1(%rdi)
-; X64-NEXT:    movb $0, (%rdi)
+; X64-NEXT:    movw $256, (%rdi) # imm = 0x100
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    retq
 entry:
   %0 = insertvalue %1 zeroinitializer, i1 false, 0
   %1 = insertvalue %1 %0, i1 true, 1
   %2 = insertvalue %1 %1, i1 true, 2
   %3 = insertvalue %1 %2, i32 48, 3
   ret %1 %3
 }

llvm/trunk/test/CodeGen/X86/bitcast-i256.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,-slow-unaligned-mem-32 | FileCheck %s --check-prefix=FAST
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+slow-unaligned-mem-32 | FileCheck %s --check-prefix=SLOW
 
 define i256 @foo(<8 x i32> %a) {
 ; FAST-LABEL: foo:
 ; FAST:       # BB#0:
 ; FAST-NEXT:    vmovups %ymm0, (%rdi)
 ; FAST-NEXT:    movq %rdi, %rax
 ; FAST-NEXT:    vzeroupper
 ; FAST-NEXT:    retq
 ;
 ; SLOW-LABEL: foo:
 ; SLOW:       # BB#0:
-; SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; SLOW-NEXT:    vpextrq $1, %xmm1, 24(%rdi)
-; SLOW-NEXT:    vmovq %xmm1, 16(%rdi)
-; SLOW-NEXT:    vpextrq $1, %xmm0, 8(%rdi)
-; SLOW-NEXT:    vmovq %xmm0, (%rdi)
+; SLOW-NEXT:    vextractf128 $1, %ymm0, 16(%rdi)
+; SLOW-NEXT:    vmovups %xmm0, (%rdi)
 ; SLOW-NEXT:    movq %rdi, %rax
 ; SLOW-NEXT:    vzeroupper
 ; SLOW-NEXT:    retq
   %r = bitcast <8 x i32> %a to i256
   ret i256 %r
 }

llvm/trunk/test/CodeGen/X86/constant-combines.ll

 ; The DAG combiner at one point contained bugs that given enough permutations
 ; would incorrectly form an illegal operation for the last of these stores when
 ; it folded it to a zero too late to legalize the zero store operation. If this
 ; ever starts forming a zero store instead of movss, the test case has stopped
 ; being useful.
 ;
 ; CHECK-LABEL: PR22524:
 ; CHECK:       # BB#0: # %entry
-; CHECK-NEXT:    movl $0, 4(%rdi)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movd %eax, %xmm0
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    mulss %xmm0, %xmm1
-; CHECK-NEXT:    movl $0, (%rdi)
+; CHECK-NEXT:    movq $0, (%rdi)
 ; CHECK-NEXT:    movss %xmm1, 4(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   %0 = getelementptr inbounds { float, float }, { float, float }* %arg,  i32 0, i32 1
   store float 0.000000e+00, float* %0, align 4
   %1 = getelementptr inbounds { float, float }, { float, float }* %arg, i64 0,  i32 0
   %2 = bitcast float* %1 to i64*
   %3 = load i64, i64* %2, align 8

llvm/trunk/test/CodeGen/X86/extract-store.ll

 ; AVX-X64-NEXT:    vmovhpd %xmm0, (%rdi)
 ; AVX-X64-NEXT:    retq
   %vecext = extractelement <2 x double> %foo, i32 1
   store double %vecext, double* %dst, align 1
   ret void
 }
 
 define void @extract_f128_0(fp128* nocapture %dst, <2 x fp128> %foo) nounwind {
-; X32-LABEL: extract_f128_0:
-; X32:       # BB#0:
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl %esi, 12(%edi)
-; X32-NEXT:    movl %edx, 8(%edi)
-; X32-NEXT:    movl %ecx, 4(%edi)
-; X32-NEXT:    movl %eax, (%edi)
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    retl
+; SSE-X32-LABEL: extract_f128_0:
+; SSE-X32:       # BB#0:
+; SSE-X32-NEXT:    pushl %edi
+; SSE-X32-NEXT:    pushl %esi
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; SSE-X32-NEXT:    movl %esi, 12(%edi)
+; SSE-X32-NEXT:    movl %edx, 8(%edi)
+; SSE-X32-NEXT:    movl %ecx, 4(%edi)
+; SSE-X32-NEXT:    movl %eax, (%edi)
+; SSE-X32-NEXT:    popl %esi
+; SSE-X32-NEXT:    popl %edi
+; SSE-X32-NEXT:    retl
 ;
 ; SSE2-X64-LABEL: extract_f128_0:
 ; SSE2-X64:       # BB#0:
 ; SSE2-X64-NEXT:    movq %rdx, 8(%rdi)
 ; SSE2-X64-NEXT:    movq %rsi, (%rdi)
 ; SSE2-X64-NEXT:    retq
 ;
 ; SSE41-X64-LABEL: extract_f128_0:
 ; SSE41-X64:       # BB#0:
 ; SSE41-X64-NEXT:    movq %rdx, 8(%rdi)
 ; SSE41-X64-NEXT:    movq %rsi, (%rdi)
 ; SSE41-X64-NEXT:    retq
 ;
+; AVX-X32-LABEL: extract_f128_0:
+; AVX-X32:       # BB#0:
+; AVX-X32-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; AVX-X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX-X32-NEXT:    vmovups %xmm0, (%eax)
+; AVX-X32-NEXT:    retl
+;
 ; AVX-X64-LABEL: extract_f128_0:
 ; AVX-X64:       # BB#0:
 ; AVX-X64-NEXT:    movq %rdx, 8(%rdi)
 ; AVX-X64-NEXT:    movq %rsi, (%rdi)
 ; AVX-X64-NEXT:    retq
 ;
 ; SSE-F128-LABEL: extract_f128_0:
 ; SSE-F128:       # BB#0:
 ; SSE-F128-NEXT:    movaps %xmm0, (%rdi)
 ; SSE-F128-NEXT:    retq
   %vecext = extractelement <2 x fp128> %foo, i32 0
   store fp128 %vecext, fp128* %dst, align 1
   ret void
 }
 
 define void @extract_f128_1(fp128* nocapture %dst, <2 x fp128> %foo) nounwind {
-; X32-LABEL: extract_f128_1:
-; X32:       # BB#0:
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl %esi, 12(%edi)
-; X32-NEXT:    movl %edx, 8(%edi)
-; X32-NEXT:    movl %ecx, 4(%edi)
-; X32-NEXT:    movl %eax, (%edi)
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    retl
+; SSE-X32-LABEL: extract_f128_1:
+; SSE-X32:       # BB#0:
+; SSE-X32-NEXT:    pushl %edi
+; SSE-X32-NEXT:    pushl %esi
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; SSE-X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; SSE-X32-NEXT:    movl %esi, 12(%edi)
+; SSE-X32-NEXT:    movl %edx, 8(%edi)
+; SSE-X32-NEXT:    movl %ecx, 4(%edi)
+; SSE-X32-NEXT:    movl %eax, (%edi)
+; SSE-X32-NEXT:    popl %esi
+; SSE-X32-NEXT:    popl %edi
+; SSE-X32-NEXT:    retl
 ;
 ; SSE2-X64-LABEL: extract_f128_1:
 ; SSE2-X64:       # BB#0:
 ; SSE2-X64-NEXT:    movq %r8, 8(%rdi)
 ; SSE2-X64-NEXT:    movq %rcx, (%rdi)
 ; SSE2-X64-NEXT:    retq
 ;
 ; SSE41-X64-LABEL: extract_f128_1:
 ; SSE41-X64:       # BB#0:
 ; SSE41-X64-NEXT:    movq %r8, 8(%rdi)
 ; SSE41-X64-NEXT:    movq %rcx, (%rdi)
 ; SSE41-X64-NEXT:    retq
 ;
+; AVX-X32-LABEL: extract_f128_1:
+; AVX-X32:       # BB#0:
+; AVX-X32-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; AVX-X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX-X32-NEXT:    vmovups %xmm0, (%eax)
+; AVX-X32-NEXT:    retl
+;
 ; AVX-X64-LABEL: extract_f128_1:
 ; AVX-X64:       # BB#0:
 ; AVX-X64-NEXT:    movq %r8, 8(%rdi)
 ; AVX-X64-NEXT:    movq %rcx, (%rdi)
 ; AVX-X64-NEXT:    retq
 ;
 ; SSE-F128-LABEL: extract_f128_1:
 ; SSE-F128:       # BB#0:

llvm/trunk/test/CodeGen/X86/fold-vector-sext-crash2.ll

 }
 
 define <2 x i256> @test_zext1() {
   %Se = zext <2 x i8> <i8 -1, i8 -2> to <2 x i256>
   %Shuff = shufflevector <2 x i256> zeroinitializer, <2 x i256> %Se, <2 x i32> <i32 1, i32 3>
   ret <2 x i256> %Shuff
 
   ; X64-LABEL: test_zext1
-  ; X64:       movq $0
-  ; X64-NEXT:  movq $0
+  ; X64: xorps   %xmm0, %xmm0
+  ; X64: movaps  %xmm0
+  ; X64: movaps  %xmm0
+  ; X64: movaps  %xmm0
   ; X64-NEXT:  movq $0
   ; X64-NEXT:  movq $254
 
   ; X32-LABEL: test_zext1
   ; X32:       movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $254
 }
 
 define <2 x i256> @test_zext2() {
   %Se = zext <2 x i128> <i128 -1, i128 -2> to <2 x i256>
   %Shuff = shufflevector <2 x i256> zeroinitializer, <2 x i256> %Se, <2 x i32> <i32 1, i32 3>
   ret <2 x i256> %Shuff
 
   ; X64-LABEL: test_zext2
-  ; X64:       movq $0
-  ; X64-NEXT:  movq $0
+  ; X64: xorps   %xmm0, %xmm0
+  ; X64-NEXT: movaps  %xmm0
+  ; X64-NEXT: movaps  %xmm0
+  ; X64-NEXT: movaps  %xmm0  
   ; X64-NEXT:  movq $-1
   ; X64-NEXT:  movq $-2
 
   ; X32-LABEL: test_zext2
   ; X32:       movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $-1
   ; X32-NEXT:  movl $-1
   ; X32-NEXT:  movl $-1
   ; X32-NEXT:  movl $-2
 }

llvm/trunk/test/CodeGen/X86/legalize-shl-vec.ll

 ; X32-NEXT:    movl $0, 12(%eax)
 ; X32-NEXT:    movl $0, 8(%eax)
 ; X32-NEXT:    movl $0, 4(%eax)
 ; X32-NEXT:    movl $0, (%eax)
 ; X32-NEXT:    retl $4
 ;
 ; X64-LABEL: test_shl:
 ; X64:       # BB#0:
-; X64-NEXT:    movq $0, 56(%rdi)
-; X64-NEXT:    movq $0, 48(%rdi)
-; X64-NEXT:    movq $0, 40(%rdi)
-; X64-NEXT:    movq $0, 32(%rdi)
-; X64-NEXT:    movq $0, 24(%rdi)
-; X64-NEXT:    movq $0, 16(%rdi)
-; X64-NEXT:    movq $0, 8(%rdi)
-; X64-NEXT:    movq $0, (%rdi)
+; X64-NEXT:    xorps %xmm0, %xmm0
+; X64-NEXT:    movaps %xmm0, 48(%rdi)
+; X64-NEXT:    movaps %xmm0, 32(%rdi)
+; X64-NEXT:    movaps %xmm0, 16(%rdi)
+; X64-NEXT:    movaps %xmm0, (%rdi)
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    retq
   %Amt = insertelement <2 x i256> undef, i256 -1, i32 0
   %Out = shl <2 x i256> %In, %Amt
   ret <2 x i256> %Out
 }
 
 define <2 x i256> @test_srl(<2 x i256> %In) {

 ; X32-NEXT:    movl $0, 12(%eax)
 ; X32-NEXT:    movl $0, 8(%eax)
 ; X32-NEXT:    movl $0, 4(%eax)
 ; X32-NEXT:    movl $0, (%eax)
 ; X32-NEXT:    retl $4
 ;
 ; X64-LABEL: test_srl:
 ; X64:       # BB#0:
-; X64-NEXT:    movq $0, 56(%rdi)
-; X64-NEXT:    movq $0, 48(%rdi)
-; X64-NEXT:    movq $0, 40(%rdi)
-; X64-NEXT:    movq $0, 32(%rdi)
-; X64-NEXT:    movq $0, 24(%rdi)
-; X64-NEXT:    movq $0, 16(%rdi)
-; X64-NEXT:    movq $0, 8(%rdi)
-; X64-NEXT:    movq $0, (%rdi)
+; X64-NEXT:    xorps %xmm0, %xmm0
+; X64-NEXT:    movaps %xmm0, 48(%rdi)
+; X64-NEXT:    movaps %xmm0, 32(%rdi)
+; X64-NEXT:    movaps %xmm0, 16(%rdi)
+; X64-NEXT:    movaps %xmm0, (%rdi)
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    retq
   %Amt = insertelement <2 x i256> undef, i256 -1, i32 0
   %Out = lshr <2 x i256> %In, %Amt
   ret <2 x i256> %Out
 }
 
 define <2 x i256> @test_sra(<2 x i256> %In) {

llvm/trunk/test/CodeGen/X86/merge-consecutive-loads-128.ll

 ;
 ; AVX-LABEL: merge_8i16_i16_23u567u9:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovups 4(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_8i16_i16_23u567u9:
 ; X32-SSE1:       # BB#0:
-; X32-SSE1-NEXT:    pushl %ebp
+; X32-SSE1-NEXT:    pushl %edi
 ; X32-SSE1-NEXT:  .Lcfi6:
 ; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %ebx
+; X32-SSE1-NEXT:    pushl %esi
 ; X32-SSE1-NEXT:  .Lcfi7:
 ; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    pushl %edi
 ; X32-SSE1-NEXT:  .Lcfi8:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 16
-; X32-SSE1-NEXT:    pushl %esi
+; X32-SSE1-NEXT:    .cfi_offset %esi, -12
 ; X32-SSE1-NEXT:  .Lcfi9:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 20
-; X32-SSE1-NEXT:  .Lcfi10:
-; X32-SSE1-NEXT:    .cfi_offset %esi, -20
-; X32-SSE1-NEXT:  .Lcfi11:
-; X32-SSE1-NEXT:    .cfi_offset %edi, -16
-; X32-SSE1-NEXT:  .Lcfi12:
-; X32-SSE1-NEXT:    .cfi_offset %ebx, -12
-; X32-SSE1-NEXT:  .Lcfi13:
-; X32-SSE1-NEXT:    .cfi_offset %ebp, -8
+; X32-SSE1-NEXT:    .cfi_offset %edi, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movzwl 4(%ecx), %edx
-; X32-SSE1-NEXT:    movzwl 6(%ecx), %esi
-; X32-SSE1-NEXT:    movzwl 10(%ecx), %edi
-; X32-SSE1-NEXT:    movzwl 12(%ecx), %ebx
-; X32-SSE1-NEXT:    movzwl 14(%ecx), %ebp
+; X32-SSE1-NEXT:    movl 4(%ecx), %edx
+; X32-SSE1-NEXT:    movl 10(%ecx), %esi
+; X32-SSE1-NEXT:    movzwl 14(%ecx), %edi
 ; X32-SSE1-NEXT:    movzwl 18(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %bp, 10(%eax)
-; X32-SSE1-NEXT:    movw %bx, 8(%eax)
+; X32-SSE1-NEXT:    movw %di, 10(%eax)
 ; X32-SSE1-NEXT:    movw %cx, 14(%eax)
-; X32-SSE1-NEXT:    movw %si, 2(%eax)
-; X32-SSE1-NEXT:    movw %dx, (%eax)
-; X32-SSE1-NEXT:    movw %di, 6(%eax)
+; X32-SSE1-NEXT:    movl %edx, (%eax)
+; X32-SSE1-NEXT:    movl %esi, 6(%eax)
 ; X32-SSE1-NEXT:    popl %esi
 ; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    popl %ebx
-; X32-SSE1-NEXT:    popl %ebp
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_8i16_i16_23u567u9:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movups 4(%eax), %xmm0
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 2

 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_8i16_i16_34uuuuuu:
 ; X32-SSE1:       # BB#0:
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movzwl 6(%ecx), %edx
-; X32-SSE1-NEXT:    movzwl 8(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %cx, 2(%eax)
-; X32-SSE1-NEXT:    movw %dx, (%eax)
+; X32-SSE1-NEXT:    movl 6(%ecx), %ecx
+; X32-SSE1-NEXT:    movl %ecx, (%eax)
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_8i16_i16_34uuuuuu:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 3

 ;
 ; AVX-LABEL: merge_8i16_i16_45u7zzzz:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_8i16_i16_45u7zzzz:
 ; X32-SSE1:       # BB#0:
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:  .Lcfi14:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:  .Lcfi15:
-; X32-SSE1-NEXT:    .cfi_offset %esi, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movzwl 8(%ecx), %edx
-; X32-SSE1-NEXT:    movzwl 10(%ecx), %esi
+; X32-SSE1-NEXT:    movl 8(%ecx), %edx
 ; X32-SSE1-NEXT:    movzwl 14(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %si, 2(%eax)
-; X32-SSE1-NEXT:    movw %dx, (%eax)
+; X32-SSE1-NEXT:    movl %edx, (%eax)
 ; X32-SSE1-NEXT:    movw %cx, 6(%eax)
-; X32-SSE1-NEXT:    movw $0, 14(%eax)
-; X32-SSE1-NEXT:    movw $0, 12(%eax)
-; X32-SSE1-NEXT:    movw $0, 10(%eax)
-; X32-SSE1-NEXT:    movw $0, 8(%eax)
-; X32-SSE1-NEXT:    popl %esi
+; X32-SSE1-NEXT:    movl $0, 12(%eax)
+; X32-SSE1-NEXT:    movl $0, 8(%eax)
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_8i16_i16_45u7zzzz:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 4

 ;
 ; AVX-LABEL: merge_16i8_i8_01u3456789ABCDuF:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovups (%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_16i8_i8_01u3456789ABCDuF:
 ; X32-SSE1:       # BB#0:
+; X32-SSE1-NEXT:    pushl %ebp
+; X32-SSE1-NEXT:  .Lcfi10:
+; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
 ; X32-SSE1-NEXT:    pushl %ebx
+; X32-SSE1-NEXT:  .Lcfi11:
+; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X32-SSE1-NEXT:    pushl %edi
+; X32-SSE1-NEXT:  .Lcfi12:
+; X32-SSE1-NEXT:    .cfi_def_cfa_offset 16
+; X32-SSE1-NEXT:    pushl %esi
+; X32-SSE1-NEXT:  .Lcfi13:
+; X32-SSE1-NEXT:    .cfi_def_cfa_offset 20
+; X32-SSE1-NEXT:  .Lcfi14:
+; X32-SSE1-NEXT:    .cfi_offset %esi, -20
+; X32-SSE1-NEXT:  .Lcfi15:
+; X32-SSE1-NEXT:    .cfi_offset %edi, -16
 ; X32-SSE1-NEXT:  .Lcfi16:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    subl $12, %esp
+; X32-SSE1-NEXT:    .cfi_offset %ebx, -12
 ; X32-SSE1-NEXT:  .Lcfi17:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 20
-; X32-SSE1-NEXT:  .Lcfi18:
-; X32-SSE1-NEXT:    .cfi_offset %ebx, -8
+; X32-SSE1-NEXT:    .cfi_offset %ebp, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movb (%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 1(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 3(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 4(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 5(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 6(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 7(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 8(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 9(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 10(%ecx), %bh
-; X32-SSE1-NEXT:    movb 11(%ecx), %bl
-; X32-SSE1-NEXT:    movb 12(%ecx), %dh
+; X32-SSE1-NEXT:    movzwl (%ecx), %ebp
+; X32-SSE1-NEXT:    movl 3(%ecx), %esi
+; X32-SSE1-NEXT:    movl 7(%ecx), %edi
+; X32-SSE1-NEXT:    movzwl 11(%ecx), %ebx
 ; X32-SSE1-NEXT:    movb 13(%ecx), %dl
 ; X32-SSE1-NEXT:    movb 15(%ecx), %cl
 ; X32-SSE1-NEXT:    movb %dl, 13(%eax)
-; X32-SSE1-NEXT:    movb %dh, 12(%eax)
 ; X32-SSE1-NEXT:    movb %cl, 15(%eax)
-; X32-SSE1-NEXT:    movb %bl, 11(%eax)
-; X32-SSE1-NEXT:    movb %bh, 10(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 9(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 8(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 7(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 6(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 5(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 4(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 1(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, (%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 3(%eax)
-; X32-SSE1-NEXT:    addl $12, %esp
+; X32-SSE1-NEXT:    movw %bx, 11(%eax)
+; X32-SSE1-NEXT:    movl %edi, 7(%eax)
+; X32-SSE1-NEXT:    movw %bp, (%eax)
+; X32-SSE1-NEXT:    movl %esi, 3(%eax)
+; X32-SSE1-NEXT:    popl %esi
+; X32-SSE1-NEXT:    popl %edi
 ; X32-SSE1-NEXT:    popl %ebx
+; X32-SSE1-NEXT:    popl %ebp
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_16i8_i8_01u3456789ABCDuF:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movups (%eax), %xmm0
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0

 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
 ; X32-SSE1:       # BB#0:
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movb (%ecx), %dl
-; X32-SSE1-NEXT:    movb 1(%ecx), %dh
+; X32-SSE1-NEXT:    movzwl (%ecx), %edx
 ; X32-SSE1-NEXT:    movb 3(%ecx), %cl
-; X32-SSE1-NEXT:    movb %dh, 1(%eax)
-; X32-SSE1-NEXT:    movb %dl, (%eax)
+; X32-SSE1-NEXT:    movw %dx, (%eax)
 ; X32-SSE1-NEXT:    movb %cl, 3(%eax)
 ; X32-SSE1-NEXT:    movb $0, 15(%eax)
-; X32-SSE1-NEXT:    movb $0, 14(%eax)
-; X32-SSE1-NEXT:    movb $0, 13(%eax)
-; X32-SSE1-NEXT:    movb $0, 7(%eax)
-; X32-SSE1-NEXT:    movb $0, 6(%eax)
+; X32-SSE1-NEXT:    movw $0, 13(%eax)
+; X32-SSE1-NEXT:    movw $0, 6(%eax)
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0

 ;
 ; AVX-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
 ; X32-SSE1:       # BB#0:
-; X32-SSE1-NEXT:    pushl %ebx
-; X32-SSE1-NEXT:  .Lcfi19:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %eax
-; X32-SSE1-NEXT:  .Lcfi20:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:  .Lcfi21:
-; X32-SSE1-NEXT:    .cfi_offset %ebx, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movb (%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 1(%ecx), %dh
-; X32-SSE1-NEXT:    movb 2(%ecx), %bl
-; X32-SSE1-NEXT:    movb 3(%ecx), %bh
-; X32-SSE1-NEXT:    movb 6(%ecx), %dl
-; X32-SSE1-NEXT:    movb 7(%ecx), %cl
-; X32-SSE1-NEXT:    movb %cl, 7(%eax)
-; X32-SSE1-NEXT:    movb %dl, 6(%eax)
-; X32-SSE1-NEXT:    movb %bh, 3(%eax)
-; X32-SSE1-NEXT:    movb %bl, 2(%eax)
-; X32-SSE1-NEXT:    movb %dh, 1(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, (%eax)
+; X32-SSE1-NEXT:    movl (%ecx), %edx
+; X32-SSE1-NEXT:    movzwl 6(%ecx), %ecx
+; X32-SSE1-NEXT:    movw %cx, 6(%eax)
+; X32-SSE1-NEXT:    movl %edx, (%eax)
 ; X32-SSE1-NEXT:    movb $0, 15(%eax)
-; X32-SSE1-NEXT:    movb $0, 14(%eax)
-; X32-SSE1-NEXT:    movb $0, 13(%eax)
-; X32-SSE1-NEXT:    addl $4, %esp
-; X32-SSE1-NEXT:    popl %ebx
+; X32-SSE1-NEXT:    movw $0, 13(%eax)
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0

 ; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_2i64_i64_12_volatile:
 ; X32-SSE1:       # BB#0:
 ; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:  .Lcfi22:
+; X32-SSE1-NEXT:  .Lcfi18:
 ; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
 ; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:  .Lcfi23:
+; X32-SSE1-NEXT:  .Lcfi19:
 ; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:  .Lcfi24:
+; X32-SSE1-NEXT:  .Lcfi20:
 ; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:  .Lcfi25:
+; X32-SSE1-NEXT:  .Lcfi21:
 ; X32-SSE1-NEXT:    .cfi_offset %edi, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-SSE1-NEXT:    movl 8(%ecx), %edx
 ; X32-SSE1-NEXT:    movl 12(%ecx), %esi
 ; X32-SSE1-NEXT:    movl 16(%ecx), %edi
 ; X32-SSE1-NEXT:    movl 20(%ecx), %ecx
 ; X32-SSE1-NEXT:    movl %ecx, 12(%eax)

llvm/trunk/test/CodeGen/X86/merge-store-partially-alias-loads.ll

 ; REQUIRES: asserts
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck -check-prefix=X86 %s
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -debug-only=isel < %s 2>&1 | FileCheck -check-prefix=DBGDAG %s
 
 ; It's OK to merge the load / store of the first 2 components, but
 ; they must not be placed on the same chain after merging.
 
 ; X86-LABEL: {{^}}merge_store_partial_overlap_load:
 ; X86-DAG: movzwl ([[BASEREG:%[a-z]+]]), %e[[LO2:[a-z]+]]
 ; X86-DAG: movb 2([[BASEREG]]), [[HI1:%[a-z]+]]
 
 ; X86-NEXT: movw %[[LO2]], 1([[BASEREG]])
 ; X86-NEXT: movb [[HI1]], 3([[BASEREG]])
 ; X86-NEXT: retq
 
-; DBGDAG-LABEL: Optimized lowered selection DAG: BB#0 'merge_store_partial_overlap_load:'
+; DBGDAG-LABEL: Optimized legalized selection DAG: BB#0 'merge_store_partial_overlap_load:'
 ; DBGDAG: [[ENTRYTOKEN:t[0-9]+]]: ch = EntryToken
 ; DBGDAG-DAG: [[BASEPTR:t[0-9]+]]: i64,ch = CopyFromReg [[ENTRYTOKEN]],
 ; DBGDAG-DAG: [[ADDPTR:t[0-9]+]]: i64 = add [[BASEPTR]], Constant:i64<2>
 
 ; DBGDAG-DAG: [[LD2:t[0-9]+]]: i16,ch = load<LD2[%tmp81](align=1)> [[ENTRYTOKEN]], [[BASEPTR]], undef:i64
 ; DBGDAG-DAG: [[LD1:t[0-9]+]]: i8,ch = load<LD1[%tmp12]> [[ENTRYTOKEN]], [[ADDPTR]], undef:i64
 
 ; DBGDAG-DAG: [[ST1:t[0-9]+]]: ch = store<ST1[%tmp14]> [[ENTRYTOKEN]], [[LD1]], t{{[0-9]+}}, undef:i64
 ; DBGDAG-DAG: [[LOADTOKEN:t[0-9]+]]: ch = TokenFactor [[LD2]]:1, [[LD1]]:1
 ; DBGDAG-DAG: [[ST2:t[0-9]+]]: ch = store<ST2[%tmp10](align=1)> [[LOADTOKEN]], [[LD2]], t{{[0-9]+}}, undef:i64
 
 ; DBGDAG: X86ISD::RET_FLAG t{{[0-9]+}},
 
-; DBGDAG: Type-legalized selection DAG: BB#0 'merge_store_partial_overlap_load:'
+; DBGDAG-LABEL: Instruction selection begins
 define void @merge_store_partial_overlap_load([4 x i8]* %tmp) {
   %tmp8 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 0
   %tmp10 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 1
   %tmp12 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 2
   %tmp14 = getelementptr [4 x i8], [4 x i8]* %tmp, i32 0, i8 3
 
   %tmp9 = load i8, i8* %tmp8, align 1   ; base + 0
   %tmp11 = load i8, i8* %tmp10, align 1 ; base + 1

llvm/trunk/test/CodeGen/X86/no-sse2-avg.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; REQUIRES: asserts
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2 | FileCheck %s
 
 define <16 x i8> @PR27973() {
 ; CHECK-LABEL: PR27973:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    movb $0, 15(%rdi)
-; CHECK-NEXT:    movb $0, 14(%rdi)
-; CHECK-NEXT:    movb $0, 13(%rdi)
-; CHECK-NEXT:    movb $0, 12(%rdi)
-; CHECK-NEXT:    movb $0, 11(%rdi)
-; CHECK-NEXT:    movb $0, 10(%rdi)
-; CHECK-NEXT:    movb $0, 9(%rdi)
-; CHECK-NEXT:    movb $0, 8(%rdi)
-; CHECK-NEXT:    movb $0, 7(%rdi)
-; CHECK-NEXT:    movb $0, 6(%rdi)
-; CHECK-NEXT:    movb $0, 5(%rdi)
-; CHECK-NEXT:    movb $0, 4(%rdi)
-; CHECK-NEXT:    movb $0, 3(%rdi)
-; CHECK-NEXT:    movb $0, 2(%rdi)
-; CHECK-NEXT:    movb $0, 1(%rdi)
-; CHECK-NEXT:    movb $0, (%rdi)
+; CHECK-NEXT:    movq $0, 8(%rdi)
+; CHECK-NEXT:    movq $0, (%rdi)
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = zext <16 x i8> zeroinitializer to <16 x i32>
   %t1 = add nuw nsw <16 x i32> %t0, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %t2 = lshr <16 x i32> %t1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %t3 = trunc <16 x i32> %t2 to <16 x i8>
   ret <16 x i8> %t3
 }

llvm/trunk/test/CodeGen/X86/stdarg.ll

 ; RUN: llc -stack-symbol-ordering=0 < %s -mtriple=x86_64-linux | FileCheck %s
 
 %struct.__va_list_tag = type { i32, i32, i8*, i8* }
 
 define void @foo(i32 %x, ...) nounwind {
 entry:
   %ap = alloca [1 x %struct.__va_list_tag], align 8; <[1 x %struct.__va_list_tag]*> [#uses=2]
   %ap12 = bitcast [1 x %struct.__va_list_tag]* %ap to i8*; <i8*> [#uses=2]
   call void @llvm.va_start(i8* %ap12)
 ; CHECK: testb %al, %al
 
 ; These test for specific offsets, which is very fragile. Still, the test needs
 ; to ensure that va_list has the correct element types.
 ;
 ; CHECK-DAG: movq {{.*}}, 192(%rsp)
 ; CHECK-DAG: movq {{.*}}, 184(%rsp)
-; CHECK-DAG: movl {{.*}}, 180(%rsp)
-; CHECK-DAG: movl {{.*}}, 176(%rsp)
+; CHECK-DAG: movq {{.*}}, 176(%rsp)
   %ap3 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %ap, i64 0, i64 0; <%struct.__va_list_tag*> [#uses=1]
   call void @bar(%struct.__va_list_tag* %ap3) nounwind
   call void @llvm.va_end(i8* %ap12)
   ret void
 }
 
 declare void @llvm.va_start(i8*) nounwind
 
 declare void @bar(%struct.__va_list_tag*)
 
 declare void @llvm.va_end(i8*) nounwind

llvm/trunk/test/CodeGen/X86/stores-merging.ll

 ; RUN: llc < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %structTy = type { i8, i32, i32 }
 
 @e = common global %structTy zeroinitializer, align 4
 
 ;; Ensure that MergeConsecutiveStores doesn't incorrectly reorder
 ;; store operations.  The first test stores in increasing address
 ;; order, the second in decreasing -- but in both cases should have
 ;; the same result in memory in the end.
 
 ; CHECK-LABEL: redundant_stores_merging:
-; CHECK:   movabsq $528280977409, %rax
-; CHECK:   movq    %rax, e+4(%rip)
-; CHECK:   movl    $456, e+8(%rip)
+; CHECK: movabsq $1958505086977, %rax
+; CHECK: movq    %rax, e+4(%rip)
 define void @redundant_stores_merging() {
 entry:
   store i32 1, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 1), align 4
   store i32 123, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   store i32 456, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   ret void
 }
 
 ;; This variant tests PR25154.
 ; CHECK-LABEL: redundant_stores_merging_reverse:
-; CHECK:   movabsq $528280977409, %rax
-; CHECK:   movq    %rax, e+4(%rip)
-; CHECK:   movl    $456, e+8(%rip)
+; CHECK: movabsq $1958505086977, %rax
+; CHECK: movq  %rax, e+4(%rip)
 define void @redundant_stores_merging_reverse() {
 entry:
   store i32 123, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   store i32 456, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   store i32 1, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 1), align 4
   ret void
 }