[X86][AVX] Add shuffle masking support for EltsFromConsecutiveLoads

[RKSimon]

Summary

Add support for the case where we have a consecutive load (which must include the first + last elements) with a mixture of undef/zero elements. We load the vector and then apply a shuffle to clear the zero'd elements.

[RKSimon]

rebased to latest

[spatel]

LGTM. See inline for a couple of code comment additions.

lib/Target/X86/X86ISelLowering.cpp

5589 ↗

(On Diff #48194)

Update comment to say something like: "If we have zero elts, we'll shuffle those with the vector load." ?

5647 ↗

(On Diff #48194)

Could use a comment here to explain what's going on. Something like: "Generate a mask for a shufflevector of the vector load and a zero vector."

[RKSimon]

Thanks Sanjay - we're getting closer to the masked load cases now!

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   int FirstLoadedElt = LoadMask.find_first();
   SDValue EltBase = PeekThroughBitcast(Elts[FirstLoadedElt]);
   LoadSDNode *LDBase = cast<LoadSDNode>(EltBase);
   EVT LDBaseVT = EltBase.getValueType();
 
   // Consecutive loads can contain UNDEFS but not ZERO elements.
+  // Consecutive loads with UNDEFs and ZEROs elements require a
+  // an additional shuffle stage to clear the ZERO elements.
   bool IsConsecutiveLoad = true;
+  bool IsConsecutiveLoadWithZeros = true;
   for (int i = FirstLoadedElt + 1; i <= LastLoadedElt; ++i) {
     if (LoadMask[i]) {
       SDValue Elt = PeekThroughBitcast(Elts[i]);
       LoadSDNode *LD = cast<LoadSDNode>(Elt);
       if (!DAG.isConsecutiveLoad(LD, LDBase,
                                  Elt.getValueType().getStoreSizeInBits() / 8,
                                  i - FirstLoadedElt)) {
         IsConsecutiveLoad = false;
+        IsConsecutiveLoadWithZeros = false;
         break;
       }
     } else if (ZeroMask[i]) {
       IsConsecutiveLoad = false;
       break;
     }
   }
 
auto CreateLoad = [&DAG, &DL](EVT VT, LoadSDNode *LDBase) {
     }
 
     return NewLd;
   };
 
   // LOAD - all consecutive load/undefs (must start/end with a load).
   // If we have found an entire vector of loads and undefs, then return a large
   // load of the entire vector width starting at the base pointer.
-  if (IsConsecutiveLoad && FirstLoadedElt == 0 &&
-      LastLoadedElt == (int)(NumElems - 1) && ZeroMask.none()) {
+  // If the vector contains zeros, then attempt to shuffle those elements.
+  if (FirstLoadedElt == 0 && LastLoadedElt == (int)(NumElems - 1) &&
+      (IsConsecutiveLoad || IsConsecutiveLoadWithZeros)) {
     assert(LDBase && "Did not find base load for merging consecutive loads");
     EVT EltVT = LDBase->getValueType(0);
     // Ensure that the input vector size for the merged loads matches the
     // cumulative size of the input elements.
     if (VT.getSizeInBits() != EltVT.getSizeInBits() * NumElems)
       return SDValue();
 
     if (isAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
       return SDValue();
 
+    if (IsConsecutiveLoad)
       return CreateLoad(VT, LDBase);
+
+    // IsConsecutiveLoadWithZeros - we need to create a shuffle of the loaded
+    // vector and a zero vector to clear out the zero elements.
+    if (!isAfterLegalize && NumElems == VT.getVectorNumElements()) {
+      SmallVector<int, 4> ClearMask(NumElems, -1);
+      for (unsigned i = 0; i < NumElems; ++i) {
+        if (ZeroMask[i])
+          ClearMask[i] = i + NumElems;
+        else if (LoadMask[i])
+          ClearMask[i] = i;
+      }
+      SDValue V = CreateLoad(VT, LDBase);
+      SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
+                                 : DAG.getConstantFP(0.0, DL, VT);
+      return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
+    }
   }
 
   int LoadSize =
       (1 + LastLoadedElt - FirstLoadedElt) * LDBaseVT.getStoreSizeInBits();
 
   // VZEXT_LOAD - consecutive load/undefs followed by zeros/undefs.
   if (IsConsecutiveLoad && FirstLoadedElt == 0 && LoadSize == 64 &&
       ((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))) {

llvm/trunk/test/CodeGen/X86/merge-consecutive-loads-128.ll

; X32-SSE-NEXT:    retl
   %val0 = load float, float* %ptr0
   %val1 = load float, float* %ptr1
   %res0 = insertelement <4 x float> undef, float %val0, i32 0
   %res1 = insertelement <4 x float> %res0, float %val1, i32 1
   ret <4 x float> %res1
 }
 
 define <4 x float> @merge_4f32_f32_34z6(float* %ptr) nounwind uwtable noinline ssp {
-; SSE-LABEL: merge_4f32_f32_34z6:
-; SSE:       # BB#0:
-; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[1,0]
-; SSE-NEXT:    retq
+; SSE2-LABEL: merge_4f32_f32_34z6:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movups 12(%rdi), %xmm0
+; SSE2-NEXT:    xorps %xmm1, %xmm1
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: merge_4f32_f32_34z6:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    movups 12(%rdi), %xmm1
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: merge_4f32_f32_34z6:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,1],xmm0[1,0]
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = mem[0,1],xmm0[2],mem[3]
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE-LABEL: merge_4f32_f32_34z6:
 ; X32-SSE:       # BB#0:
 ; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[1,0]
+; X32-SSE-NEXT:    movups 12(%eax), %xmm1
+; X32-SSE-NEXT:    xorps %xmm0, %xmm0
+; X32-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
 ; X32-SSE-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 3
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 4
   %ptr3 = getelementptr inbounds float, float* %ptr, i64 6
   %val0 = load float, float* %ptr0
   %val1 = load float, float* %ptr1
   %val3 = load float, float* %ptr3
   %res0 = insertelement <4 x float> zeroinitializer, float %val0, i32 0

llvm/trunk/test/CodeGen/X86/merge-consecutive-loads-256.ll

; X32-AVX-NEXT:    retl
   %val0 = load double, double* %ptr0
   %val1 = load double, double* %ptr1
   %res0 = insertelement <4 x double> zeroinitializer, double %val0, i32 0
   %res1 = insertelement <4 x double> %res0, double %val1, i32 1
   ret <4 x double> %res1
 }
 
 define <4 x double> @merge_4f64_f64_34z6(double* %ptr) nounwind uwtable noinline ssp {
-; AVX1-LABEL: merge_4f64_f64_34z6:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vmovups 24(%rdi), %xmm0
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX1-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: merge_4f64_f64_34z6:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovdqu 24(%rdi), %xmm0
-; AVX2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX2-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512F-LABEL: merge_4f64_f64_34z6:
-; AVX512F:       # BB#0:
-; AVX512F-NEXT:    vmovdqu 24(%rdi), %xmm0
-; AVX512F-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX512F-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    retq
+; AVX-LABEL: merge_4f64_f64_34z6:
+; AVX:       # BB#0:
+; AVX-NEXT:    vxorpd %ymm0, %ymm0, %ymm0
+; AVX-NEXT:    vblendpd {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3]
+; AVX-NEXT:    retq
 ;
 ; X32-AVX-LABEL: merge_4f64_f64_34z6:
 ; X32-AVX:       # BB#0:
 ; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups 24(%eax), %xmm0
-; X32-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; X32-AVX-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X32-AVX-NEXT:    vxorpd %ymm0, %ymm0, %ymm0
+; X32-AVX-NEXT:    vblendpd {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3]
 ; X32-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 4
   %ptr3 = getelementptr inbounds double, double* %ptr, i64 6
   %val0 = load double, double* %ptr0
   %val1 = load double, double* %ptr1
   %val3 = load double, double* %ptr3
   %res0 = insertelement <4 x double> undef, double %val0, i32 0

 ; AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX512F-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
 ; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; X32-AVX-LABEL: merge_8f32_2f32_23z5:
 ; X32-AVX:       # BB#0:
 ; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovupd 16(%eax), %xmm0
-; X32-AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; X32-AVX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; X32-AVX-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X32-AVX-NEXT:    vxorpd %ymm0, %ymm0, %ymm0
+; X32-AVX-NEXT:    vblendpd {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3]
 ; X32-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds <2 x float>, <2 x float>* %ptr, i64 2
   %ptr1 = getelementptr inbounds <2 x float>, <2 x float>* %ptr, i64 3
   %ptr3 = getelementptr inbounds <2 x float>, <2 x float>* %ptr, i64 5
   %val0 = load <2 x float>, <2 x float>* %ptr0
   %val1 = load <2 x float>, <2 x float>* %ptr1
   %val3 = load <2 x float>, <2 x float>* %ptr3
   %res01 = shufflevector <2 x float> %val0, <2 x float> %val1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; X32-AVX-NEXT:    retl
   %res2 = insertelement <8 x float> %res1, float   0.0, i32 2
   %res3 = insertelement <8 x float> %res2, float   0.0, i32 3
   %res6 = insertelement <8 x float> %res3, float   0.0, i32 6
   %res7 = insertelement <8 x float> %res6, float   0.0, i32 7
   ret <8 x float> %res7
 }
 
 define <8 x float> @merge_8f32_f32_1u3u5zu8(float* %ptr) nounwind uwtable noinline ssp {
-; AVX1-LABEL: merge_8f32_f32_1u3u5zu8:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm2[0,1],xmm0[1,0]
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: merge_8f32_f32_1u3u5zu8:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm2[0,1],xmm0[1,0]
-; AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; AVX2-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512F-LABEL: merge_8f32_f32_1u3u5zu8:
-; AVX512F:       # BB#0:
-; AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512F-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,1],xmm0[1,0]
-; AVX512F-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512F-NEXT:    retq
+; AVX-LABEL: merge_8f32_f32_1u3u5zu8:
+; AVX:       # BB#0:
+; AVX-NEXT:    vxorps %ymm0, %ymm0, %ymm0
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
+; AVX-NEXT:    retq
 ;
 ; X32-AVX-LABEL: merge_8f32_f32_1u3u5zu8:
 ; X32-AVX:       # BB#0:
 ; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm2[0,1],xmm0[1,0]
-; X32-AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-AVX-NEXT:    vxorps %ymm0, %ymm0, %ymm0
+; X32-AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
 ; X32-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 1
   %ptr2 = getelementptr inbounds float, float* %ptr, i64 3
   %ptr4 = getelementptr inbounds float, float* %ptr, i64 5
   %ptr7 = getelementptr inbounds float, float* %ptr, i64 8
   %val0 = load float, float* %ptr0
   %val2 = load float, float* %ptr2
   %val4 = load float, float* %ptr4
; X32-AVX-NEXT:    retl
   %res6 = insertelement <8 x i32> %res4, i32     0, i32 6
   %res7 = insertelement <8 x i32> %res6, i32     0, i32 7
   ret <8 x i32> %res7
 }
 
 define <8 x i32> @merge_8i32_i32_1u3u5zu8(i32* %ptr) nounwind uwtable noinline ssp {
 ; AVX1-LABEL: merge_8i32_i32_1u3u5zu8:
 ; AVX1:       # BB#0:
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,0]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpinsrd $2, 12(%rdi), %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: merge_8i32_i32_1u3u5zu8:
 ; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,0]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:    vpinsrd $2, 12(%rdi), %xmm1, %xmm1
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: merge_8i32_i32_1u3u5zu8:
 ; AVX512F:       # BB#0:
-; AVX512F-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,0]
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512F-NEXT:    vpinsrd $2, 12(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vpxor %ymm0, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
 ; AVX512F-NEXT:    retq
 ;
 ; X32-AVX-LABEL: merge_8i32_i32_1u3u5zu8:
 ; X32-AVX:       # BB#0:
 ; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,1,0]
-; X32-AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; X32-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    vpinsrd $2, 12(%eax), %xmm1, %xmm1
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X32-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X32-AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
 ; X32-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 1
   %ptr2 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptr4 = getelementptr inbounds i32, i32* %ptr, i64 5
   %ptr7 = getelementptr inbounds i32, i32* %ptr, i64 8
   %val0 = load i32, i32* %ptr0
   %val2 = load i32, i32* %ptr2
   %val4 = load i32, i32* %ptr4
; X32-AVX-NEXT:    retl
   %resE = insertelement <16 x i16> %resC, i16 %valE, i16 14
   %resF = insertelement <16 x i16> %resE, i16 %valF, i16 15
   ret <16 x i16> %resF
 }
 
 define <16 x i16> @merge_16i16_i16_0uu3zzuuuuuzCuEF(i16* %ptr) nounwind uwtable noinline ssp {
 ; AVX1-LABEL: merge_16i16_i16_0uu3zzuuuuuzCuEF:
 ; AVX1:       # BB#0:
-; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpinsrw $4, 24(%rdi), %xmm1, %xmm1
-; AVX1-NEXT:    vpinsrw $6, 28(%rdi), %xmm1, %xmm1
-; AVX1-NEXT:    vpinsrw $7, 30(%rdi), %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [65535,0,0,65535,0,0,0,0,0,0,0,0,65535,0,65535,65535]
+; AVX1-NEXT:    vandps (%rdi), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: merge_16i16_i16_0uu3zzuuuuuzCuEF:
 ; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrw $4, 24(%rdi), %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrw $6, 28(%rdi), %xmm1, %xmm1
-; AVX2-NEXT:    vpinsrw $7, 30(%rdi), %xmm1, %xmm1
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vmovups (%rdi), %ymm0
+; AVX2-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: merge_16i16_i16_0uu3zzuuuuuzCuEF:
 ; AVX512F:       # BB#0:
-; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpinsrw $4, 24(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:    vpinsrw $6, 28(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:    vpinsrw $7, 30(%rdi), %xmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovups (%rdi), %ymm0
+; AVX512F-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; X32-AVX-LABEL: merge_16i16_i16_0uu3zzuuuuuzCuEF:
 ; X32-AVX:       # BB#0:
 ; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X32-AVX-NEXT:    vpinsrw $4, 24(%eax), %xmm1, %xmm1
-; X32-AVX-NEXT:    vpinsrw $6, 28(%eax), %xmm1, %xmm1
-; X32-AVX-NEXT:    vpinsrw $7, 30(%eax), %xmm1, %xmm1
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X32-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [65535,0,0,65535,0,0,0,0,0,0,0,0,65535,0,65535,65535]
+; X32-AVX-NEXT:    vandps (%eax), %ymm0, %ymm0
 ; X32-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 0
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 3
   %ptrC = getelementptr inbounds i16, i16* %ptr, i64 12
   %ptrE = getelementptr inbounds i16, i16* %ptr, i64 14
   %ptrF = getelementptr inbounds i16, i16* %ptr, i64 15
   %val0 = load i16, i16* %ptr0
   %val3 = load i16, i16* %ptr3

llvm/trunk/test/CodeGen/X86/merge-consecutive-loads-512.ll

; X32-AVX512F-NEXT:    retl
   %res6 = insertelement <8 x double> %res3, double   0.0, i32 6
   %res7 = insertelement <8 x double> %res6, double   0.0, i32 7
   ret <8 x double> %res7
 }
 
 define <8 x double> @merge_8f64_f64_1u3u5zu8(double* %ptr) nounwind uwtable noinline ssp {
 ; ALL-LABEL: merge_8f64_f64_1u3u5zu8:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; ALL-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; ALL-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; ALL-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
-; ALL-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovupd 8(%rdi), %zmm0
+; ALL-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; ALL-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,u,2,u,4,13,u,7>
+; ALL-NEXT:    vpermt2pd %zmm1, %zmm2, %zmm0
 ; ALL-NEXT:    retq
 ;
 ; X32-AVX512F-LABEL: merge_8f64_f64_1u3u5zu8:
 ; X32-AVX512F:       # BB#0:
 ; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; X32-AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
-; X32-AVX512F-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; X32-AVX512F-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-AVX512F-NEXT:    vmovupd 8(%eax), %zmm0
+; X32-AVX512F-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; X32-AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,0,u,u,2,0,u,u,4,0,13,0,u,u,7,0>
+; X32-AVX512F-NEXT:    vpermt2pd %zmm1, %zmm2, %zmm0
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 1
   %ptr2 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr4 = getelementptr inbounds double, double* %ptr, i64 5
   %ptr7 = getelementptr inbounds double, double* %ptr, i64 8
   %val0 = load double, double* %ptr0
   %val2 = load double, double* %ptr2
   %val4 = load double, double* %ptr4
; X32-AVX512F-NEXT:    retl
   %res6 = insertelement <8 x i64> %res4, i64     0, i32 6
   %res7 = insertelement <8 x i64> %res6, i64     0, i32 7
   ret <8 x i64> %res7
 }
 
 define <8 x i64> @merge_8i64_i64_1u3u5zu8(i64* %ptr) nounwind uwtable noinline ssp {
 ; ALL-LABEL: merge_8i64_i64_1u3u5zu8:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; ALL-NEXT:    vpbroadcastq 64(%rdi), %xmm1
-; ALL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; ALL-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; ALL-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
-; ALL-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqu64 8(%rdi), %zmm0
+; ALL-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; ALL-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,u,2,u,4,13,u,7>
+; ALL-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
 ; ALL-NEXT:    retq
 ;
 ; X32-AVX512F-LABEL: merge_8i64_i64_1u3u5zu8:
 ; X32-AVX512F:       # BB#0:
 ; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vpinsrd $2, 64(%eax), %xmm0, %xmm0
-; X32-AVX512F-NEXT:    vpinsrd $3, 68(%eax), %xmm0, %xmm0
-; X32-AVX512F-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; X32-AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX512F-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; X32-AVX512F-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
-; X32-AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; X32-AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-AVX512F-NEXT:    vmovdqu32 8(%eax), %zmm0
+; X32-AVX512F-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; X32-AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,0,u,u,2,0,u,u,4,0,13,0,u,u,7,0>
+; X32-AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 1
   %ptr2 = getelementptr inbounds i64, i64* %ptr, i64 3
   %ptr4 = getelementptr inbounds i64, i64* %ptr, i64 5
   %ptr7 = getelementptr inbounds i64, i64* %ptr, i64 8
   %val0 = load i64, i64* %ptr0
   %val2 = load i64, i64* %ptr2
   %val4 = load i64, i64* %ptr4
; X32-AVX512F-NEXT:    retl
   %resE = insertelement <16 x float> %resC, float %valE, i32 14
   %resF = insertelement <16 x float> %resE, float %valF, i32 15
   ret <16 x float> %resF
 }
 
 define <16 x float> @merge_16f32_f32_0uu3zzuuuuuzCuEF(float* %ptr) nounwind uwtable noinline ssp {
 ; ALL-LABEL: merge_16f32_f32_0uu3zzuuuuuzCuEF:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; ALL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; ALL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; ALL-NEXT:    vmovupd (%rdi), %xmm1
-; ALL-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; ALL-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; ALL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovups (%rdi), %zmm0
+; ALL-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; ALL-NEXT:    vmovdqa32 {{.*#+}} zmm2 = <0,u,u,3,20,21,u,u,u,u,u,u,12,29,14,15>
+; ALL-NEXT:    vpermt2ps %zmm1, %zmm2, %zmm0
 ; ALL-NEXT:    retq
 ;
 ; X32-AVX512F-LABEL: merge_16f32_f32_0uu3zzuuuuuzCuEF:
 ; X32-AVX512F:       # BB#0:
 ; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-AVX512F-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; X32-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512F-NEXT:    vmovupd (%eax), %xmm1
-; X32-AVX512F-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; X32-AVX512F-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; X32-AVX512F-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-AVX512F-NEXT:    vmovups (%eax), %zmm0
+; X32-AVX512F-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; X32-AVX512F-NEXT:    vmovdqa32 {{.*#+}} zmm2 = <0,u,u,3,20,21,u,u,u,u,u,u,12,29,14,15>
+; X32-AVX512F-NEXT:    vpermt2ps %zmm1, %zmm2, %zmm0
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 0
   %ptr3 = getelementptr inbounds float, float* %ptr, i64 3
   %ptrC = getelementptr inbounds float, float* %ptr, i64 12
   %ptrE = getelementptr inbounds float, float* %ptr, i64 14
   %ptrF = getelementptr inbounds float, float* %ptr, i64 15
   %val0 = load float, float* %ptr0
   %val3 = load float, float* %ptr3
; X32-AVX512F-NEXT:    retl
   %resE = insertelement <16 x i32> %resC, i32 %valE, i32 14
   %resF = insertelement <16 x i32> %resE, i32 %valF, i32 15
   ret <16 x i32> %resF
 }
 
 define <16 x i32> @merge_16i32_i32_0uu3zzuuuuuzCuEF(i32* %ptr) nounwind uwtable noinline ssp {
 ; ALL-LABEL: merge_16i32_i32_0uu3zzuuuuuzCuEF:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; ALL-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; ALL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; ALL-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; ALL-NEXT:    vmovdqu (%rdi), %xmm1
-; ALL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; ALL-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; ALL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; ALL-NEXT:    vmovdqu32 (%rdi), %zmm0
+; ALL-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; ALL-NEXT:    vmovdqa32 {{.*#+}} zmm2 = <0,u,u,3,20,21,u,u,u,u,u,u,12,29,14,15>
+; ALL-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
 ; ALL-NEXT:    retq
 ;
 ; X32-AVX512F-LABEL: merge_16i32_i32_0uu3zzuuuuuzCuEF:
 ; X32-AVX512F:       # BB#0:
 ; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; X32-AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512F-NEXT:    vmovdqu (%eax), %xmm1
-; X32-AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X32-AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; X32-AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-AVX512F-NEXT:    vmovdqu32 (%eax), %zmm0
+; X32-AVX512F-NEXT:    vpxord %zmm1, %zmm1, %zmm1
+; X32-AVX512F-NEXT:    vmovdqa32 {{.*#+}} zmm2 = <0,u,u,3,20,21,u,u,u,u,u,u,12,29,14,15>
+; X32-AVX512F-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 0
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptrC = getelementptr inbounds i32, i32* %ptr, i64 12
   %ptrE = getelementptr inbounds i32, i32* %ptr, i64 14
   %ptrF = getelementptr inbounds i32, i32* %ptr, i64 15
   %val0 = load i32, i32* %ptr0
   %val3 = load i32, i32* %ptr3