[X86] Teach LowerBUILD_VECTOR to recognize pair-wise splats of 32-bit elements and use a 64-bit broadcast

[craig.topper]

Summary

If we are splatting pairs of 32-bit elements, we can use a 64-bit broadcast to get the job done.

We could probably could probably do this with other sizes too, for example four 16-bit elements. Or we could broadcast pairs of 16-bit elements using a 32-bit element broadcast. But I've left that as a future improvement.

I've also restricted this to AVX2 only because we can only broadcast loads under AVX.

Looks like we may still need a DAG combine for VBROADCAST + VZEXT_LOAD to fold the loads in insertelement-shuffle.ll and vector-shuffle-combining-xop.ll

[spatel]

lib/Target/X86/X86ISelLowering.cpp

8124–8133

Initialize the first 2 elements to simplify the code?

SmallVector<SDValue, 4> Ops({Op->getOperand(0), Op->getOperand(1)});
bool IsSplatPair = true;
for (unsigned i = 2; i != NumElems; ++i) {
  if (Ops[i % 2] != Op->getOperand(i)) {
    IsSplatPair = false;
    break;
  }
}

I think this would also be easier to read if it was split off into a helper function / lambda because you could just early return when you detect that it's not a splat pair.

8138

The VTs are confusingly named. ExtVT is the current vector element VT (way back at line 7891). Can we rename things to make this clearer as a preliminary clean-up (ExtVT -> EltVT)?

[RKSimon]

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

490

We'd gain from INSERT_VECTOR_ELT support being added to EltsFromConsecutiveLoads - merging multiple consecutive scalar loads into a single scalar load+insert into a zero/undef vector.

[craig.topper]

Address Sanjay's comments.

[spatel]

LGTM.

lib/Target/X86/X86ISelLowering.cpp

X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   // See if we can use a vector load to get all of the elements.
   {
     SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
     if (SDValue LD =
             EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false))
       return LD;
   }
 
+  // If this is a splat of pairs of 32-bit elements, we can use a narrower
+  // build_vector and broadcast it.
+  // TODO: We could probably generalize this more.
+  if (Subtarget.hasAVX2() && EVTBits == 32 && Values.size() == 2) {
+    SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
+                       DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
+    auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
+      // Make sure all the even/odd operands match.
+      for (unsigned i = 2; i != NumElems; ++i)
+        if (Ops[i % 2] != Op.getOperand(i))
+          return false;
+      return true;
+    };
+    if (CanSplat(Op, NumElems, Ops)) {
+      MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64;
+      MVT NarrowVT = MVT::getVectorVT(EltVT, 4);
+      // Create a new build vector and cast to v2i64/v2f64.
+      SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
+                                     DAG.getBuildVector(NarrowVT, dl, Ops));
+      // Broadcast from v2i64/v2f64 and cast to final VT.
+      MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems/2);
+      return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
+                                            NewBV));
+    }
+  }
+
   // For AVX-length vectors, build the individual 128-bit pieces and use
   // shuffles to put them in place.
   if (VT.getSizeInBits() > 128) {
     MVT HVT = MVT::getVectorVT(EltVT, NumElems/2);
 
     // Build both the lower and upper subvector.
     SDValue Lower =
         DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2));

test/CodeGen/X86/avx2-vbroadcast.ll

entry:
   %q7 = insertelement <8 x i32> %q6, i32 %q, i32 7
   ret <8 x i32> %q7
 }
 
 define <2 x i64> @Q64(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: Q64:
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X32-NEXT:    vpbroadcastq (%eax), %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: Q64:
 ; X64:       ## %bb.0: ## %entry
 ; X64-NEXT:    vpbroadcastq (%rdi), %xmm0
 ; X64-NEXT:    retq
 entry:
   %q = load i64, i64* %ptr, align 4
   %q0 = insertelement <2 x i64> undef, i64 %q, i32 0
   %q1 = insertelement <2 x i64> %q0, i64 %q, i32 1
   ret <2 x i64> %q1
 }
 
 define <4 x i64> @QQ64(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: QQ64:
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    vbroadcastsd %xmm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: QQ64:
 ; X64:       ## %bb.0: ## %entry
 ; X64-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; X64-NEXT:    retq
 entry:
   %q = load i64, i64* %ptr, align 4

 define void @isel_crash_2q(i64* %cV_R.addr) {
 ; X32-LABEL: isel_crash_2q:
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    subl $60, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 64
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; X32-NEXT:    vmovaps %xmm0, (%esp)
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm1
-; X32-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
+; X32-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    vpbroadcastq %xmm1, %xmm1
 ; X32-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
 ; X32-NEXT:    vmovdqa %xmm1, {{[0-9]+}}(%esp)
 ; X32-NEXT:    addl $60, %esp
 ; X32-NEXT:    retl
 ;
 ; X64-AVX2-LABEL: isel_crash_2q:
 ; X64-AVX2:       ## %bb.0: ## %entry
 ; X64-AVX2-NEXT:    vxorps %xmm0, %xmm0, %xmm0

 ; X32-NEXT:    .cfi_offset %ebp, -8
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    .cfi_def_cfa_register %ebp
 ; X32-NEXT:    andl $-32, %esp
 ; X32-NEXT:    subl $128, %esp
 ; X32-NEXT:    movl 8(%ebp), %eax
 ; X32-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; X32-NEXT:    vmovaps %ymm0, (%esp)
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm1
-; X32-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; X32-NEXT:    vinserti128 $1, %xmm1, %ymm1, %ymm1
+; X32-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X32-NEXT:    vbroadcastsd %xmm1, %ymm1
 ; X32-NEXT:    vmovaps %ymm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    vmovdqa %ymm1, {{[0-9]+}}(%esp)
+; X32-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%esp)
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
 ; X64-AVX2-LABEL: isel_crash_4q:
 ; X64-AVX2:       ## %bb.0: ## %eintry
 ; X64-AVX2-NEXT:    pushq %rbp

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

entry:
   %1 = select <16 x i1> %0, <16 x i32> %vecinit15.i.i, <16 x i32> zeroinitializer
   %2 = bitcast <16 x i32> %1 to <8 x i64>
   ret <8 x i64> %2
 }
 
 define <8 x i64> @test_mm512_mask_set1_epi64(<8 x i64> %__O, i8 zeroext %__M, i64 %__A) {
 ; X32-LABEL: test_mm512_mask_set1_epi64:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    vmovd %edx, %xmm1
-; X32-NEXT:    vpinsrd $1, %ecx, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $2, %edx, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $3, %ecx, %xmm1, %xmm1
-; X32-NEXT:    vinserti128 $1, %xmm1, %ymm1, %ymm1
+; X32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
 ; X32-NEXT:    kmovw %eax, %k1
-; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm0 {%k1}
+; X32-NEXT:    vpbroadcastq %xmm1, %zmm0 {%k1}
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_set1_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpbroadcastq %rsi, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %vecinit.i.i = insertelement <8 x i64> undef, i64 %__A, i32 0
   %vecinit7.i.i = shufflevector <8 x i64> %vecinit.i.i, <8 x i64> undef, <8 x i32> zeroinitializer
   %0 = bitcast i8 %__M to <8 x i1>
   %1 = select <8 x i1> %0, <8 x i64> %vecinit7.i.i, <8 x i64> %__O
   ret <8 x i64> %1
 }
 
 define <8 x i64> @test_mm512_maskz_set1_epi64(i8 zeroext %__M, i64 %__A)  {
 ; X32-LABEL: test_mm512_maskz_set1_epi64:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    vmovd %edx, %xmm0
-; X32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
 ; X32-NEXT:    kmovw %eax, %k1
-; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0 {%k1} {z}
+; X32-NEXT:    vpbroadcastq %xmm0, %zmm0 {%k1} {z}
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_set1_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpbroadcastq %rsi, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:

test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll

entry:
   %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> zeroinitializer
   ret <2 x i64> %2
 }
 
 
 define <4 x i64> @test_mm256_mask_set1_epi64(<4 x i64> %__O, i8 zeroext %__M, i64 %__A) {
 ; X32-LABEL: test_mm256_mask_set1_epi64:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X32-NEXT:    vmovd %ecx, %xmm1
-; X32-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm1, %xmm1
-; X32-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm1
-; X32-NEXT:    vinserti128 $1, %xmm1, %ymm1, %ymm1
-; X32-NEXT:    kmovw %edx, %k1
-; X32-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vpbroadcastq %xmm1, %ymm0 {%k1}
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_mask_set1_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpbroadcastq %rsi, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %vecinit.i.i = insertelement <4 x i64> undef, i64 %__A, i32 0
   %vecinit3.i.i = shufflevector <4 x i64> %vecinit.i.i, <4 x i64> undef, <4 x i32> zeroinitializer
   %0 = bitcast i8 %__M to <8 x i1>
   %extract.i = shufflevector <8 x i1> %0, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %1 = select <4 x i1> %extract.i, <4 x i64> %vecinit3.i.i, <4 x i64> %__O
   ret <4 x i64> %1
 }
 
 define <4 x i64> @test_mm256_maskz_set1_epi64(i8 zeroext %__M, i64 %__A)  {
 ; X32-LABEL: test_mm256_maskz_set1_epi64:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
-; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X32-NEXT:    kmovw %edx, %k1
-; X32-NEXT:    vmovdqa64 %ymm0, %ymm0 {%k1} {z}
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vpbroadcastq %xmm0, %ymm0 {%k1} {z}
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_maskz_set1_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpbroadcastq %rsi, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:

test/CodeGen/X86/broadcastm-lowering.ll

 ; AVX512VLCDBW-NEXT:    retq
 ;
 ; X86-AVX512VLCDBW-LABEL: test_mm512_epi64:
 ; X86-AVX512VLCDBW:       # %bb.0: # %entry
 ; X86-AVX512VLCDBW-NEXT:    vpcmpeqd %ymm1, %ymm0, %k0
 ; X86-AVX512VLCDBW-NEXT:    kmovd %k0, %eax
 ; X86-AVX512VLCDBW-NEXT:    movzbl %al, %eax
 ; X86-AVX512VLCDBW-NEXT:    vmovd %eax, %xmm0
-; X86-AVX512VLCDBW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3],zero,zero,zero,zero,xmm0[0,1,2,3],zero,zero,zero,zero
-; X86-AVX512VLCDBW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X86-AVX512VLCDBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; X86-AVX512VLCDBW-NEXT:    vpbroadcastq %xmm0, %zmm0
 ; X86-AVX512VLCDBW-NEXT:    retl
 entry:
   %0 = icmp eq <8 x i32> %a, %b
   %1 = bitcast <8 x i1> %0 to i8
   %conv.i = zext i8 %1 to i64
   %vecinit.i.i = insertelement <8 x i64> undef, i64 %conv.i, i32 0
   %vecinit7.i.i = shufflevector <8 x i64> %vecinit.i.i, <8 x i64> undef, <8 x i32> zeroinitializer
   ret <8 x i64> %vecinit7.i.i

 ; AVX512VLCDBW-NEXT:    retq
 ;
 ; X86-AVX512VLCDBW-LABEL: test_mm256_epi64:
 ; X86-AVX512VLCDBW:       # %bb.0: # %entry
 ; X86-AVX512VLCDBW-NEXT:    vpcmpeqd %ymm1, %ymm0, %k0
 ; X86-AVX512VLCDBW-NEXT:    kmovd %k0, %eax
 ; X86-AVX512VLCDBW-NEXT:    movzbl %al, %eax
 ; X86-AVX512VLCDBW-NEXT:    vmovd %eax, %xmm0
-; X86-AVX512VLCDBW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3],zero,zero,zero,zero,xmm0[0,1,2,3],zero,zero,zero,zero
-; X86-AVX512VLCDBW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X86-AVX512VLCDBW-NEXT:    vpbroadcastq %xmm0, %ymm0
 ; X86-AVX512VLCDBW-NEXT:    retl
 entry:
   %0 = icmp eq <8 x i32> %a, %b
   %1 = bitcast <8 x i1> %0 to i8
   %conv.i = zext i8 %1 to i64
   %vecinit.i.i = insertelement <4 x i64> undef, i64 %conv.i, i32 0
   %vecinit3.i.i = shufflevector <4 x i64> %vecinit.i.i, <4 x i64> undef, <4 x i32> zeroinitializer
   ret <4 x i64> %vecinit3.i.i

test/CodeGen/X86/insertelement-shuffle.ll

 ; X32_AVX256-LABEL: insert_subvector_into_undef:
 ; X32_AVX256:       # %bb.0:
 ; X32_AVX256-NEXT:    pushl %ebp
 ; X32_AVX256-NEXT:    movl %esp, %ebp
 ; X32_AVX256-NEXT:    andl $-8, %esp
 ; X32_AVX256-NEXT:    subl $8, %esp
 ; X32_AVX256-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X32_AVX256-NEXT:    vmovlps %xmm0, (%esp)
-; X32_AVX256-NEXT:    movl (%esp), %eax
-; X32_AVX256-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32_AVX256-NEXT:    vmovd %eax, %xmm0
-; X32_AVX256-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; X32_AVX256-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
-; X32_AVX256-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
-; X32_AVX256-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X32_AVX256-NEXT:    vmovdqa %ymm0, %ymm1
+; X32_AVX256-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X32_AVX256-NEXT:    vbroadcastsd %xmm0, %ymm0
+; X32_AVX256-NEXT:    vmovaps %ymm0, %ymm1
 ; X32_AVX256-NEXT:    movl %ebp, %esp
 ; X32_AVX256-NEXT:    popl %ebp
 ; X32_AVX256-NEXT:    retl
 ;
 ; X64_AVX256-LABEL: insert_subvector_into_undef:
 ; X64_AVX256:       # %bb.0:
 ; X64_AVX256-NEXT:    vmovd %edi, %xmm0
 ; X64_AVX256-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0

test/CodeGen/X86/vector-shuffle-combining-xop.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=X32
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefix=X32
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=X64
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=X32 --check-prefix=X86AVX
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefix=X32 --check-prefix=X86AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=X64 --check-prefix=X64AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefix=X64 --check-prefix=X64AVX2
 
 declare <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double>, <2 x double>, <2 x i64>, i8) nounwind readnone
 declare <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double>, <4 x double>, <4 x i64>, i8) nounwind readnone
 
 declare <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float>, <4 x float>, <4 x i32>, i8) nounwind readnone
 declare <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float>, <8 x float>, <8 x i32>, i8) nounwind readnone
 
 declare <16 x i8> @llvm.x86.xop.vpperm(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
; X64-NEXT:    retq
   %res1 = bitcast <4 x i32> %res0 to <16 x i8>
   %res2 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %res1, <16 x i8> undef, <16 x i8> <i8 2, i8 3, i8 0, i8 1, i8 128, i8 128, i8 128, i8 128, i8 10, i8 11, i8 8, i8 9, i8 14, i8 15, i8 12, i8 13>)
   %res3 = bitcast <16 x i8> %res2 to <4 x i32>
   ret <4 x i32> %res3
 }
 
 ; FIXME: Duplicated load in i686
 define void @buildvector_v4f32_0404(float %a, float %b, <4 x float>* %ptr) {
-; X32-LABEL: buildvector_v4f32_0404:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; X32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
-; X32-NEXT:    vmovaps %xmm0, (%eax)
-; X32-NEXT:    retl
-;
-; X64-LABEL: buildvector_v4f32_0404:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermil2ps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[0],xmm1[0]
-; X64-NEXT:    vmovaps %xmm0, (%rdi)
-; X64-NEXT:    retq
+; X86AVX-LABEL: buildvector_v4f32_0404:
+; X86AVX:       # %bb.0:
+; X86AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; X86AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; X86AVX-NEXT:    vmovaps %xmm0, (%eax)
+; X86AVX-NEXT:    retl
+;
+; X86AVX2-LABEL: buildvector_v4f32_0404:
+; X86AVX2:       # %bb.0:
+; X86AVX2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86AVX2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X86AVX2-NEXT:    vmovapd %xmm0, (%eax)
+; X86AVX2-NEXT:    retl
+;
+; X64AVX-LABEL: buildvector_v4f32_0404:
+; X64AVX:       # %bb.0:
+; X64AVX-NEXT:    vpermil2ps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[0],xmm1[0]
+; X64AVX-NEXT:    vmovaps %xmm0, (%rdi)
+; X64AVX-NEXT:    retq
+;
+; X64AVX2-LABEL: buildvector_v4f32_0404:
+; X64AVX2:       # %bb.0:
+; X64AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; X64AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X64AVX2-NEXT:    vmovapd %xmm0, (%rdi)
+; X64AVX2-NEXT:    retq
   %v0 = insertelement <4 x float> undef, float %a, i32 0
   %v1 = insertelement <4 x float> %v0,   float %b, i32 1
   %v2 = insertelement <4 x float> %v1,   float %a, i32 2
   %v3 = insertelement <4 x float> %v2,   float %b, i32 3
   store <4 x float> %v3, <4 x float>* %ptr
   ret void
 }