[X86][SSE]] Lower BUILD_VECTOR with repeated ops as BUILD_VECTOR + VECTOR_SHUFFLE

[RKSimon]

Summary

It can be costly to transfer from the gprs to the xmm registers and can prevent loads merging.

This patch splits vXi16/vXi32/vXi64 BUILD_VECTORS that use the same operand in multiple elements into a BUILD_VECTOR with only a single insertion of each of those elements and then performs an unary shuffle to duplicate the values.

There are a couple of minor regressions this patch unearths due to some missing MOVDDUP/BROADCAST folds that I will address in a future patch.

Note: Now that vector shuffle lowering and combining is pretty good we should be reusing that instead of duplicating so much in LowerBUILD_VECTOR - this is the first of several patches to address this.

[RKSimon]

ping?

[RKSimon]

ping?

[spatel]

lib/Target/X86/X86ISelLowering.cpp

6112–6113 ↗

(On Diff #93069)

"build vector of repeated ops" translated to "splat" in my mind when I read this. I think we guarantee that case won't make it this far, so assert that condition?
How about "build vector with repeated ops (but not a full splat)"?

6115 ↗

(On Diff #93069)

I'd rather not use "Permute" in the name here since that implies one of those specific AVX instructions. "lowerBuildVectorWithRepeatedEltsUsingShuffle"?

6129 ↗

(On Diff #93069)

I prefer to put a verb on these kinds of bools - "HasRepeatedElts"?

6138 ↗

(On Diff #93069)

Run-on: "repeated, so don't"

[RKSimon]

Updated based on Sanjay's feedback.

[RKSimon]

lib/Target/X86/X86ISelLowering.cpp

6112–6113 ↗

(On Diff #93069)

Splats can occur here as buildvector broadcast lowering only handles a few cases for where we have a legal BROADCAST instruction (AVX1 onwards - it doesn't even deal with MOVDDUP AFAICT.

[spatel]

LGTM.

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

return (Index == 0) ? V.getOperand(0)
                         : DAG.getUNDEF(VT.getVectorElementType());
 
   if (V.getOpcode() == ISD::BUILD_VECTOR)
     return V.getOperand(Index);
 
   return SDValue();
 }
 
+// Attempt to lower a build vector of repeated elts as a build vector of unique
+// ops followed by a shuffle.
+static SDValue
+lowerBuildVectorWithRepeatedEltsUsingShuffle(SDValue V, SelectionDAG &DAG,
+                                             const X86Subtarget &Subtarget) {
+  MVT VT = V.getSimpleValueType();
+  unsigned NumElts = VT.getVectorNumElements();
+
+  // TODO - vXi8 insertions+shuffles often cause PSHUFBs which can lead to
+  // excessive/bulky shuffle mask creation.
+  if (VT.getScalarSizeInBits() < 16)
+    return SDValue();
+
+  // Create list of unique operands to be passed to a build vector and a shuffle
+  // mask describing the repetitions.
+  // TODO - we currently insert the first occurances in place - sometimes it
+  // might be better to insert them in other locations for shuffle efficiency.
+  bool HasRepeatedElts = false;
+  SmallVector<int, 16> Mask(NumElts, SM_SentinelUndef);
+  SmallVector<SDValue, 16> Uniques(V->op_begin(), V->op_end());
+  for (unsigned i = 0; i != NumElts; ++i) {
+    SDValue Op = Uniques[i];
+    if (Op.isUndef())
+      continue;
+    Mask[i] = i;
+
+    // Zeros can be efficiently repeated, so don't shuffle these.
+    if (X86::isZeroNode(Op))
+      continue;
+
+    // If any repeated operands are found then mark the build vector entry as
+    // undef and setup a copy in the shuffle mask.
+    for (unsigned j = i + 1; j != NumElts; ++j)
+      if (Op == Uniques[j]) {
+        HasRepeatedElts = true;
+        Mask[j] = i;
+        Uniques[j] = DAG.getUNDEF(VT.getScalarType());
+      }
+  }
+
+  if (!HasRepeatedElts)
+    return SDValue();
+
+  SDLoc DL(V);
+  return DAG.getVectorShuffle(VT, DL, DAG.getBuildVector(VT, DL, Uniques),
+                              DAG.getUNDEF(VT), Mask);
+}
+
 /// Custom lower build_vector of v16i8.
 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
                                      unsigned NumNonZero, unsigned NumZero,
                                      SelectionDAG &DAG,
                                      const X86Subtarget &Subtarget) {
   if (NumNonZero > 8)
     return SDValue();
 
if (Values.size() == 1) {
     return SDValue();
   }
 
   // A vector full of immediates; various special cases are already
   // handled, so this is best done with a single constant-pool load.
   if (IsAllConstants)
     return SDValue();
 
-  // See if we can use a vector load to get all of the elements.
   if (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) {
+    // See if we can use a vector load to get all of the elements.
     SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
     if (SDValue LD = EltsFromConsecutiveLoads(VT, Ops, dl, DAG, false))
       return LD;
+
+    // Attempt to lower a build vector of repeated elts as single insertions
+    // followed by a shuffle.
+    if (SDValue V =
+            lowerBuildVectorWithRepeatedEltsUsingShuffle(Op, DAG, Subtarget))
+      return V;
   }
 
   // For AVX-length vectors, build the individual 128-bit pieces and use
   // shuffles to put them in place.
   if (VT.is256BitVector() || VT.is512BitVector()) {
     SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
 
     EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2);

llvm/trunk/test/CodeGen/X86/avx-intrinsics-fast-isel.ll

; X64-NEXT:    retq
   %res7 = insertelement <8 x i32> %res6, i32 %a0, i32 7
   %res = bitcast <8 x i32> %res7 to <4 x i64>
   ret <4 x i64> %res
 }
 
 define <4 x i64> @test_mm256_set1_epi64x(i64 %a0) nounwind {
 ; X32-LABEL: test_mm256_set1_epi64x:
 ; X32:       # BB#0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_set1_epi64x:
 ; X64:       # BB#0:
 ; X64-NEXT:    vmovq %rdi, %xmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0

llvm/trunk/test/CodeGen/X86/avx-vbroadcast.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+avx | FileCheck %s --check-prefix=X32
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s --check-prefix=X64
 
 define <4 x i64> @A(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: A:
 ; X32:       ## BB#0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: A:
 ; X64:       ## BB#0: ## %entry
 ; X64-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; X64-NEXT:    retq
 entry:
   %q = load i64, i64* %ptr, align 8
   %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0
   %vecinit2.i = insertelement <4 x i64> %vecinit.i, i64 %q, i32 1
   %vecinit4.i = insertelement <4 x i64> %vecinit2.i, i64 %q, i32 2
   %vecinit6.i = insertelement <4 x i64> %vecinit4.i, i64 %q, i32 3
   ret <4 x i64> %vecinit6.i
 }
 
 define <4 x i64> @A2(i64* %ptr, i64* %ptr2) nounwind uwtable readnone ssp {
 ; X32-LABEL: A2:
 ; X32:       ## BB#0: ## %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:  Lcfi0:
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:  Lcfi1:
+; X32-NEXT:    .cfi_offset %esi, -8
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl (%ecx), %edx
-; X32-NEXT:    movl 4(%ecx), %ecx
-; X32-NEXT:    movl %ecx, 4(%eax)
+; X32-NEXT:    movl 4(%ecx), %esi
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    movl %esi, 4(%eax)
 ; X32-NEXT:    movl %edx, (%eax)
-; X32-NEXT:    vmovd %edx, %xmm0
-; X32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; X32-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: A2:
 ; X64:       ## BB#0: ## %entry
 ; X64-NEXT:    movq (%rdi), %rax
 ; X64-NEXT:    vmovq %rax, %xmm0
 ; X64-NEXT:    movq %rax, (%rsi)
 ; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]

 }
 
 ; Unsupported vbroadcasts
 
 define <2 x i64> @G(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: G:
 ; X32:       ## BB#0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X32-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: G:
 ; X64:       ## BB#0: ## %entry
 ; X64-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %q = load i64, i64* %ptr, align 8
   %vecinit.i = insertelement <2 x i64> undef, i64 %q, i32 0
   %vecinit2.i = insertelement <2 x i64> %vecinit.i, i64 %q, i32 1
   ret <2 x i64> %vecinit2.i
 }
 
 define <2 x i64> @G2(i64* %ptr, i64* %ptr2) nounwind uwtable readnone ssp {
 ; X32-LABEL: G2:
 ; X32:       ## BB#0: ## %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:  Lcfi2:
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:  Lcfi3:
+; X32-NEXT:    .cfi_offset %esi, -8
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl (%ecx), %edx
-; X32-NEXT:    movl 4(%ecx), %ecx
-; X32-NEXT:    movl %ecx, 4(%eax)
+; X32-NEXT:    movl 4(%ecx), %esi
+; X32-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    movl %esi, 4(%eax)
 ; X32-NEXT:    movl %edx, (%eax)
-; X32-NEXT:    vmovd %edx, %xmm0
-; X32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: G2:
 ; X64:       ## BB#0: ## %entry
 ; X64-NEXT:    movq (%rdi), %rax
 ; X64-NEXT:    movq %rax, (%rsi)
 ; X64-NEXT:    vmovq %rax, %xmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]

llvm/trunk/test/CodeGen/X86/avx2-vbroadcast.ll

entry:
   %q7 = insertelement <8 x i32> %q6, i32 %q, i32 7
   ret <8 x i32> %q7
 }
 
 define <2 x i64> @Q64(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: Q64:
 ; X32:       ## BB#0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X32-NEXT:    vpbroadcastq (%eax), %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: Q64:
 ; X64:       ## BB#0: ## %entry
 ; X64-NEXT:    vpbroadcastq (%rdi), %xmm0
 ; X64-NEXT:    retq
 entry:
   %q = load i64, i64* %ptr, align 4
   %q0 = insertelement <2 x i64> undef, i64 %q, i32 0
   %q1 = insertelement <2 x i64> %q0, i64 %q, i32 1
   ret <2 x i64> %q1
 }
 
 define <4 x i64> @QQ64(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: QQ64:
 ; X32:       ## BB#0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    vbroadcastsd %xmm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: QQ64:
 ; X64:       ## BB#0: ## %entry
 ; X64-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; X64-NEXT:    retq
 entry:
   %q = load i64, i64* %ptr, align 4

 ; X32-LABEL: isel_crash_2q:
 ; X32:       ## BB#0: ## %entry
 ; X32-NEXT:    subl $60, %esp
 ; X32-NEXT:  Lcfi12:
 ; X32-NEXT:    .cfi_def_cfa_offset 64
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; X32-NEXT:    vmovaps %xmm0, (%esp)
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm1
-; X32-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
+; X32-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    vpbroadcastq %xmm1, %xmm1
 ; X32-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
 ; X32-NEXT:    vmovdqa %xmm1, {{[0-9]+}}(%esp)
 ; X32-NEXT:    addl $60, %esp
 ; X32-NEXT:    retl
 ;
 ; X64-AVX2-LABEL: isel_crash_2q:
 ; X64-AVX2:       ## BB#0: ## %entry
 ; X64-AVX2-NEXT:    vxorps %xmm0, %xmm0, %xmm0

 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:  Lcfi15:
 ; X32-NEXT:    .cfi_def_cfa_register %ebp
 ; X32-NEXT:    andl $-32, %esp
 ; X32-NEXT:    subl $128, %esp
 ; X32-NEXT:    movl 8(%ebp), %eax
 ; X32-NEXT:    vxorps %ymm0, %ymm0, %ymm0
 ; X32-NEXT:    vmovaps %ymm0, (%esp)
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm1
-; X32-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; X32-NEXT:    vinserti128 $1, %xmm1, %ymm1, %ymm1
+; X32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    vbroadcastsd %xmm1, %ymm1
 ; X32-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    vmovdqa %ymm1, {{[0-9]+}}(%esp)
+; X32-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%esp)
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
 ; X64-AVX2-LABEL: isel_crash_4q:
 ; X64-AVX2:       ## BB#0: ## %eintry
 ; X64-AVX2-NEXT:    pushq %rbp

llvm/trunk/test/CodeGen/X86/merge-consecutive-loads-128.ll

; X32-SSE41-NEXT:    retl
   ret <4 x float> %res3
 }
 
 ;
 ; Non-consecutive test.
 ;
 
 define <4 x float> @merge_4f32_f32_X0YY(float* %ptr0, float* %ptr1) nounwind uwtable noinline ssp {
-; SSE-LABEL: merge_4f32_f32_X0YY:
-; SSE:       # BB#0:
-; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
-; SSE-NEXT:    retq
+; SSE2-LABEL: merge_4f32_f32_X0YY:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: merge_4f32_f32_X0YY:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],zero,mem[0],zero
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: merge_4f32_f32_X0YY:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0,0]
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],zero,mem[0],zero
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,2,2]
 ; AVX-NEXT:    retq
 ;
-; X32-SSE-LABEL: merge_4f32_f32_X0YY:
-; X32-SSE:       # BB#0:
-; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
-; X32-SSE-NEXT:    retl
+; X32-SSE1-LABEL: merge_4f32_f32_X0YY:
+; X32-SSE1:       # BB#0:
+; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X32-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-SSE1-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
+; X32-SSE1-NEXT:    retl
+;
+; X32-SSE41-LABEL: merge_4f32_f32_X0YY:
+; X32-SSE41:       # BB#0:
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],zero,mem[0],zero
+; X32-SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; X32-SSE41-NEXT:    retl
   %val0 = load float, float* %ptr0, align 4
   %val1 = load float, float* %ptr1, align 4
   %res0 = insertelement <4 x float> undef, float %val0, i32 0
   %res1 = insertelement <4 x float> %res0, float 0.000000e+00, i32 1
   %res2 = insertelement <4 x float> %res1, float %val1, i32 2
   %res3 = insertelement <4 x float> %res2, float %val1, i32 3
   ret <4 x float> %res3
 }

llvm/trunk/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll

 }
 
 ; TODO test_mm_set1_epi64
 
 define <2 x i64> @test_mm_set1_epi64x(i64 %a0) nounwind {
 ; X32-LABEL: test_mm_set1_epi64x:
 ; X32:       # BB#0:
 ; X32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
 ; X32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; X32-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_set1_epi64x:
 ; X64:       # BB#0:
 ; X64-NEXT:    movd %rdi, %xmm0
 ; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X64-NEXT:    retq
   %res0  = insertelement <2 x i64> undef, i64 %a0, i32 0

llvm/trunk/test/CodeGen/X86/vec_fp_to_int.ll

 ; VEX:       # BB#0:
 ; VEX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; VEX-NEXT:    vcvttsd2si %xmm1, %rax
 ; VEX-NEXT:    vcvttsd2si %xmm0, %rcx
 ; VEX-NEXT:    vmovd %ecx, %xmm0
 ; VEX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
 ; VEX-NEXT:    vcvttsd2si %xmm0, %rax
 ; VEX-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
-; VEX-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; VEX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
 ; VEX-NEXT:    retq
 ;
 ; AVX512F-LABEL: fptoui_4f64_to_2i32:
 ; AVX512F:       # BB#0:
 ; AVX512F-NEXT:    # kill: %XMM0<def> %XMM0<kill> %ZMM0<def>
 ; AVX512F-NEXT:    vcvttpd2udq %zmm0, %ymm0
 ; AVX512F-NEXT:    # kill: %XMM0<def> %XMM0<kill> %YMM0<kill>
 ; AVX512F-NEXT:    vzeroupper

llvm/trunk/test/CodeGen/X86/vec_int_to_fp.ll

 ; SSE-NEXT:    cvtsi2ssq %rax, %xmm2
 ; SSE-NEXT:    movd %xmm0, %rax
 ; SSE-NEXT:    cvtsi2ssq %rax, %xmm1
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE-NEXT:    movd %xmm0, %rax
 ; SSE-NEXT:    xorps %xmm0, %xmm0
 ; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
-; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1,2,2]
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; VEX-LABEL: sitofp_4i64_to_4f32_undef:
 ; VEX:       # BB#0:
 ; VEX-NEXT:    vpextrq $1, %xmm0, %rax
 ; VEX-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm1
 ; VEX-NEXT:    vmovq %xmm0, %rax

 ; SSE-NEXT:    movq %rax, %rcx
 ; SSE-NEXT:    shrq %rcx
 ; SSE-NEXT:    andl $1, %eax
 ; SSE-NEXT:    orq %rcx, %rax
 ; SSE-NEXT:    xorps %xmm1, %xmm1
 ; SSE-NEXT:    cvtsi2ssq %rax, %xmm1
 ; SSE-NEXT:    addss %xmm1, %xmm1
 ; SSE-NEXT:  .LBB41_8:
-; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,2,2]
 ; SSE-NEXT:    retq
 ;
 ; VEX-LABEL: uitofp_4i64_to_4f32_undef:
 ; VEX:       # BB#0:
 ; VEX-NEXT:    vpextrq $1, %xmm0, %rax
 ; VEX-NEXT:    testq %rax, %rax
 ; VEX-NEXT:    js .LBB41_1
 ; VEX-NEXT:  # BB#2:

llvm/trunk/test/CodeGen/X86/vector-sext.ll

 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; X32-SSE41-LABEL: load_sext_2i1_to_2i64:
 ; X32-SSE41:       # BB#0: # %entry
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movzbl (%eax), %eax
 ; X32-SSE41-NEXT:    movl %eax, %ecx
-; X32-SSE41-NEXT:    shll $31, %ecx
+; X32-SSE41-NEXT:    shll $30, %ecx
 ; X32-SSE41-NEXT:    sarl $31, %ecx
-; X32-SSE41-NEXT:    movd %ecx, %xmm0
-; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm0
-; X32-SSE41-NEXT:    shll $30, %eax
+; X32-SSE41-NEXT:    shll $31, %eax
 ; X32-SSE41-NEXT:    sarl $31, %eax
-; X32-SSE41-NEXT:    pinsrd $2, %eax, %xmm0
-; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm0
+; X32-SSE41-NEXT:    movd %eax, %xmm0
+; X32-SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
 ; X32-SSE41-NEXT:    retl
 entry:
  %X = load <2 x i1>, <2 x i1>* %ptr
  %Y = sext <2 x i1> %X to <2 x i64>
  ret <2 x i64> %Y
 }
 
 define <2 x i64> @load_sext_2i8_to_2i64(<2 x i8> *%ptr) {

llvm/trunk/test/CodeGen/X86/vector-shuffle-combining-xop.ll

 ; X64-NEXT:    retq
   %res0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   %res1 = bitcast <4 x i32> %res0 to <16 x i8>
   %res2 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %res1, <16 x i8> undef, <16 x i8> <i8 2, i8 3, i8 0, i8 1, i8 128, i8 128, i8 128, i8 128, i8 10, i8 11, i8 8, i8 9, i8 14, i8 15, i8 12, i8 13>)
   %res3 = bitcast <16 x i8> %res2 to <4 x i32>
   ret <4 x i32> %res3
 }
 
-; FIXME: Duplicated load in i686
 define void @buildvector_v4f32_0404(float %a, float %b, <4 x float>* %ptr) {
 ; X32-LABEL: buildvector_v4f32_0404:
 ; X32:       # BB#0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; X32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
-; X32-NEXT:    vmovaps %xmm0, (%eax)
+; X32-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X32-NEXT:    vmovapd %xmm0, (%eax)
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: buildvector_v4f32_0404:
 ; X64:       # BB#0:
-; X64-NEXT:    vpermil2ps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[0],xmm1[0]
-; X64-NEXT:    vmovaps %xmm0, (%rdi)
+; X64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X64-NEXT:    vmovapd %xmm0, (%rdi)
 ; X64-NEXT:    retq
   %v0 = insertelement <4 x float> undef, float %a, i32 0
   %v1 = insertelement <4 x float> %v0,   float %b, i32 1
   %v2 = insertelement <4 x float> %v1,   float %a, i32 2
   %v3 = insertelement <4 x float> %v2,   float %b, i32 3
   store <4 x float> %v3, <4 x float>* %ptr
   ret void
 }

llvm/trunk/test/CodeGen/X86/vshift-1.ll

entry:
   store <2 x i64> %shl, <2 x i64>* %dst
   ret void
 }
 
 define void @shift1b(<2 x i64> %val, <2 x i64>* %dst, i64 %amt) nounwind {
 ; X32-LABEL: shift1b:
 ; X32:       # BB#0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
-; X32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; X32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
-; X32-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-NEXT:    psllq %xmm2, %xmm0
+; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; X32-NEXT:    psllq %xmm1, %xmm0
 ; X32-NEXT:    movdqa %xmm0, (%eax)
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: shift1b:
 ; X64:       # BB#0: # %entry
 ; X64-NEXT:    movd %rsi, %xmm1
 ; X64-NEXT:    psllq %xmm1, %xmm0
 ; X64-NEXT:    movdqa %xmm0, (%rdi)

llvm/trunk/test/CodeGen/X86/vshift-2.ll

entry:
   store <2 x i64> %lshr, <2 x i64>* %dst
   ret void
 }
 
 define void @shift1b(<2 x i64> %val, <2 x i64>* %dst, i64 %amt) nounwind {
 ; X32-LABEL: shift1b:
 ; X32:       # BB#0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
-; X32-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; X32-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
-; X32-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-NEXT:    psrlq %xmm2, %xmm0
+; X32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; X32-NEXT:    psrlq %xmm1, %xmm0
 ; X32-NEXT:    movdqa %xmm0, (%eax)
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: shift1b:
 ; X64:       # BB#0: # %entry
 ; X64-NEXT:    movd %rsi, %xmm1
 ; X64-NEXT:    psrlq %xmm1, %xmm0
 ; X64-NEXT:    movdqa %xmm0, (%rdi)