[X86][AVX] scalar_to_vector(load_scalar()) -> load_vector() for fast dereferencable loads

[RKSimon]

Summary

As reported on PR51075, we fail to make use of dereferencable 128-bit vector loads for float2 loads which were then being widened for float4 operations, preventing a useful load-fold.

We already do a similar fold for insert_subvector patterns of 128-bit loads with 256-bit dereferencable pointers.

[lebedev.ri]

llvm/lib/Target/X86/X86ISelLowering.cpp

8611

Please can you precommit this case change

[RKSimon]

rebase after rG142e60f40b50

[efriedma]

llvm/test/CodeGen/X86/load-partial-dot-product.ll

183

Even if we're allowed to do this, it doesn't seem wise; having zero in the high bits of the register is better than random junk. Can we mark up the loads somehow?

[RKSimon]

llvm/test/CodeGen/X86/load-partial-dot-product.ll

183

Isn't that what the dereferenceable(16) tag is doing?

[pengfei]

llvm/test/CodeGen/X86/load-partial-dot-product.ll

183

I have the same doubt. dereferenceable(16) tells the memory of the high bits is available. But shouldn't we always prefer to loading less bytes for performance?

[RKSimon]

I'll have a think about possible alternatives for now

[lebedev.ri]

Please note that this patch is very partial.
The actual assembly diff should be as follows:
https://godbolt.org/z/W47nvzc4e

I think from it is it clear that the wide load is unquestionably better.

llvm/test/CodeGen/X86/load-partial-dot-product.ll

183

You are comparing apples to oranges here.
The problem here is that vinsertps is (obviously) redundant and should go away.
Then it's obviously better - we have one less memory access.

[pengfei]

llvm/test/CodeGen/X86/load-partial-dot-product.ll

183

I see it now. It makes sense if it wants to turn

vmovsd {{.*#+}} xmm0 = mem[0],zero
vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]

into
vmovups (%rdi), %xmm0

[lebedev.ri]

Took a stab at VectorCombine side of the puzzle: D106399

[RKSimon]

Should this wait for D106447?

[lebedev.ri]

In D106280#2925306, @RKSimon wrote:

Should this wait for D106447?

No, probably not.

llvm/lib/Target/X86/X86ISelLowering.cpp

 /// Given the initializing elements 'Elts' of a vector of type 'VT', see if the
 /// elements can be replaced by a single large load which has the same value as
 /// a build_vector or insert_subvector whose loaded operands are 'Elts'.
 ///
 /// Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
                                         const SDLoc &DL, SelectionDAG &DAG,
                                         const X86Subtarget &Subtarget,
-                                        bool isAfterLegalize) {
+                                        bool IsAfterLegalize,
+                                        bool VectorLoadsOnly = false) {
   if ((VT.getScalarSizeInBits() % 8) != 0)
     return SDValue();
 
   unsigned NumElems = Elts.size();
 
   int LastLoadedElt = -1;
   APInt LoadMask = APInt::getNullValue(NumElems);
   APInt ZeroMask = APInt::getNullValue(NumElems);

   // LOAD - all consecutive load/undefs (must start/end with a load or be
   // entirely dereferenceable). If we have found an entire vector of loads and
   // undefs, then return a large load of the entire vector width starting at the
   // base pointer. If the vector contains zeros, then attempt to shuffle those
   // elements.
   if (FirstLoadedElt == 0 &&
       (NumLoadedElts == (int)NumElems || IsDereferenceable) &&
       (IsConsecutiveLoad || IsConsecutiveLoadWithZeros)) {
-    if (isAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
+    if (IsAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
       return SDValue();
 
     // Don't create 256-bit non-temporal aligned loads without AVX2 as these
     // will lower to regular temporal loads and use the cache.
     if (LDBase->isNonTemporal() && LDBase->getAlignment() >= 32 &&
         VT.is256BitVector() && !Subtarget.hasInt256())
       return SDValue();
 
     if (NumElems == 1)
       return DAG.getBitcast(VT, Elts[FirstLoadedElt]);
 
     if (!ZeroMask)
       return CreateLoad(VT, LDBase);
 
     // IsConsecutiveLoadWithZeros - we need to create a shuffle of the loaded
     // vector and a zero vector to clear out the zero elements.
-    if (!isAfterLegalize && VT.isVector()) {
+    if (!VectorLoadsOnly && !IsAfterLegalize && VT.isVector()) {
       unsigned NumMaskElts = VT.getVectorNumElements();
       if ((NumMaskElts % NumElems) == 0) {
         unsigned Scale = NumMaskElts / NumElems;
         SmallVector<int, 4> ClearMask(NumMaskElts, -1);
         for (unsigned i = 0; i < NumElems; ++i) {
           if (UndefMask[i])
             continue;
           int Offset = ZeroMask[i] ? NumMaskElts : 0;
           for (unsigned j = 0; j != Scale; ++j)
             ClearMask[(i * Scale) + j] = (i * Scale) + j + Offset;
         }
         SDValue V = CreateLoad(VT, LDBase);
         SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
                                    : DAG.getConstantFP(0.0, DL, VT);
         return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
       }
     }
   }
 
+  if (VectorLoadsOnly)
+    return SDValue();
+
   // If the upper half of a ymm/zmm load is undef then just load the lower half.
   if (VT.is256BitVector() || VT.is512BitVector()) {
     unsigned HalfNumElems = NumElems / 2;
     if (UndefMask.extractBits(HalfNumElems, HalfNumElems).isAllOnesValue()) {
       EVT HalfVT =
           EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems);
       SDValue HalfLD =
           EltsFromConsecutiveLoads(HalfVT, Elts.drop_back(HalfNumElems), DL,
-                                   DAG, Subtarget, isAfterLegalize);
+                                   DAG, Subtarget, IsAfterLegalize);
       if (HalfLD)
         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
                            HalfLD, DAG.getIntPtrConstant(0, DL));
     }
   }
 
   // VZEXT_LOAD - consecutive 32/64-bit load/undefs followed by zeros/undefs.
   if (IsConsecutiveLoad && FirstLoadedElt == 0 &&

       if (RepeatSize > ScalarSize)
         RepeatVT = EVT::getVectorVT(*DAG.getContext(), RepeatVT,
                                     RepeatSize / ScalarSize);
       EVT BroadcastVT =
           EVT::getVectorVT(*DAG.getContext(), RepeatVT.getScalarType(),
                            VT.getSizeInBits() / ScalarSize);
       if (TLI.isTypeLegal(BroadcastVT)) {
         if (SDValue RepeatLoad = EltsFromConsecutiveLoads(
-                RepeatVT, RepeatedLoads, DL, DAG, Subtarget, isAfterLegalize)) {
+                RepeatVT, RepeatedLoads, DL, DAG, Subtarget, IsAfterLegalize)) {
           SDValue Broadcast = RepeatLoad;
           if (RepeatSize > ScalarSize) {
             while (Broadcast.getValueSizeInBits() < VT.getSizeInBits())
               Broadcast = concatSubVectors(Broadcast, Broadcast, DAG, DL);
           } else {
             Broadcast =
                 DAG.getNode(X86ISD::VBROADCAST, DL, BroadcastVT, RepeatLoad);
           }
           return DAG.getBitcast(VT, Broadcast);
         }
       }
     }
   }
 
   return SDValue();
 }
 
 // Combine a vector ops (shuffles etc.) that is equal to build_vector load1,
 // load2, load3, load4, <0, 1, 2, 3> into a vector load if the load addresses
 // are consecutive, non-overlapping, and in the right order.
 static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL,
                                          SelectionDAG &DAG,
                                          const X86Subtarget &Subtarget,
-                                         bool isAfterLegalize) {
+                                         bool IsAfterLegalize,
+                                         bool VectorLoadsOnly = false) {
   SmallVector<SDValue, 64> Elts;
   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
     if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) {
       Elts.push_back(Elt);
       continue;
     }
     return SDValue();
   }
   assert(Elts.size() == VT.getVectorNumElements());
   return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget,
-                                  isAfterLegalize);
+                                  IsAfterLegalize, VectorLoadsOnly);
 }
 
 static Constant *getConstantVector(MVT VT, const APInt &SplatValue,
                                    unsigned SplatBitSize, LLVMContext &C) {
   unsigned ScalarSize = VT.getScalarSizeInBits();
   unsigned NumElm = SplatBitSize / ScalarSize;
 
   SmallVector<Constant *, 32> ConstantVec;

   SDLoc dl(N);
   EVT VT = N->getValueType(0);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (TLI.isTypeLegal(VT))
     if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
       return AddSub;
 
   // Attempt to combine into a vector load/broadcast.
-  if (SDValue LD = combineToConsecutiveLoads(VT, SDValue(N, 0), dl, DAG,
-                                             Subtarget, true))
+  if (SDValue LD = combineToConsecutiveLoads(
+          VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true))
     return LD;
 
   // For AVX2, we sometimes want to combine
   // (vector_shuffle <mask> (concat_vectors t1, undef)
   //                        (concat_vectors t2, undef))
   // Into:
   // (vector_shuffle <mask> (concat_vectors t1, t2), undef)
   // Since the latter can be efficiently lowered with VPERMD/VPERMQ

     SDValue Ext =
         extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits);
     return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1));
   }
 
   return SDValue();
 }
 
-static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
+static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG,
+                                     TargetLowering::DAGCombinerInfo &DCI,
+                                     const X86Subtarget &Subtarget) {
   EVT VT = N->getValueType(0);
   SDValue Src = N->getOperand(0);
   SDLoc DL(N);
 
   // If this is a scalar to vector to v1i1 from an AND with 1, bypass the and.
   // This occurs frequently in our masked scalar intrinsic code and our
   // floating point select lowering with AVX512.
   // TODO: SimplifyDemandedBits instead?

                           DAG.getAnyExtOrTrunc(ExtSrc, DL, MVT::i32)));
   }
 
   // Combine (v2i64 (scalar_to_vector (i64 (bitconvert (mmx))))) to MOVQ2DQ.
   if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST &&
       Src.getOperand(0).getValueType() == MVT::x86mmx)
     return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0));
 
-  // See if we're broadcasting the scalar value, in which case just reuse that.
-  // Ensure the same SDValue from the SDNode use is being used.
-  if (VT.getScalarType() == Src.getValueType())
-    for (SDNode *User : Src->uses())
+  // Ensure we don't have any implicit truncation.
+  if (VT.getScalarType() == Src.getValueType()) {
+    // See if we're broadcasting the scalar value, in which case just reuse
+    // that. Ensure the same SDValue from the SDNode use is being used.
+    for (SDNode *User : Src->uses()) {
       if (User->getOpcode() == X86ISD::VBROADCAST &&
           Src == User->getOperand(0)) {
         unsigned SizeInBits = VT.getFixedSizeInBits();
         unsigned BroadcastSizeInBits =
             User->getValueSizeInBits(0).getFixedSize();
         if (BroadcastSizeInBits == SizeInBits)
           return SDValue(User, 0);
         if (BroadcastSizeInBits > SizeInBits)
           return extractSubVector(SDValue(User, 0), 0, DAG, DL, SizeInBits);
         // TODO: Handle BroadcastSizeInBits < SizeInBits when we have test
         // coverage.
       }
+    }
+
+    // Attempt to combine into a vector load.
+    if (auto *Ld = dyn_cast<LoadSDNode>(peekThroughBitcasts(Src))) {
+      bool Fast;
+      const X86TargetLowering *TLI = Subtarget.getTargetLowering();
+      if (N->isOnlyUserOf(Src.getNode()) &&
+          TLI->allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
+                                  *Ld->getMemOperand(), &Fast) && Fast)
+        if (SDValue LD = combineToConsecutiveLoads(
+                VT, SDValue(N, 0), DL, DAG, Subtarget, DCI.isAfterLegalizeDAG(),
+                /*VectorLoadsOnly*/ true))
+          return LD;
+    }
+  }
 
   return SDValue();
 }
 
 // Simplify PMULDQ and PMULUDQ operations.
 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
                              TargetLowering::DAGCombinerInfo &DCI,
                              const X86Subtarget &Subtarget) {

 }
 
 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   switch (N->getOpcode()) {
   default: break;
   case ISD::SCALAR_TO_VECTOR:
-    return combineScalarToVector(N, DAG);
+    return combineScalarToVector(N, DAG, DCI, Subtarget);
   case ISD::EXTRACT_VECTOR_ELT:
   case X86ISD::PEXTRW:
   case X86ISD::PEXTRB:
     return combineExtractVectorElt(N, DAG, DCI, Subtarget);
   case ISD::CONCAT_VECTORS:
     return combineConcatVectors(N, DAG, DCI, Subtarget);
   case ISD::INSERT_SUBVECTOR:
     return combineInsertSubvector(N, DAG, DCI, Subtarget);

llvm/test/CodeGen/X86/load-partial-dot-product.ll

 ; SSE41-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: dot3_float3:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovups (%rdi), %xmm0
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vmovups (%rsi), %xmm1
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
 ; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %bcx012 = bitcast float* %a0 to <3 x float>*

 ; SSE41-NEXT:    mulss 8(%rsi), %xmm2
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: dot3_float2_float:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovups (%rdi), %xmm0
 ; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vmulps (%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vmulss 8(%rsi), %xmm1, %xmm1
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %bcx01 = bitcast float* %a0 to <2 x float>*
   %bcy01 = bitcast float* %a1 to <2 x float>*
   %x01 = load <2 x float>, <2 x float>* %bcx01, align 4

 ; SSE41-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; SSE41-NEXT:    mulps %xmm0, %xmm1
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: dot2_float2:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovups (%rdi), %xmm0
+; AVX-NEXT:    vmulps (%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %bcx01 = bitcast float* %a0 to <2 x float>*
   %bcy01 = bitcast float* %a1 to <2 x float>*
   %x01 = load <2 x float>, <2 x float>* %bcx01, align 4
   %y01 = load <2 x float>, <2 x float>* %bcy01, align 4
   %mul01 = fmul <2 x float> %x01, %y01
   %mul0 = extractelement <2 x float> %mul01, i32 0
   %mul1 = extractelement <2 x float> %mul01, i32 1
   %dot01 = fadd float %mul0, %mul1
   ret float %dot01
 }

llvm/test/CodeGen/X86/load-partial.ll

 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: load_float4_float3_as_float2_float_0122:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovups (%rdi), %xmm0
 ; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
 ; AVX-NEXT:    retq
   %2 = bitcast <4 x float>* %0 to <2 x float>*
   %3 = load <2 x float>, <2 x float>* %2, align 4
   %4 = extractelement <2 x float> %3, i32 0
   %5 = insertelement <4 x float> undef, float %4, i32 0
   %6 = extractelement <2 x float> %3, i32 1

llvm/test/CodeGen/X86/masked_gather.ll

 ; SSE-NEXT:    pcmpeqd %xmm0, %xmm1
 ; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
 ; SSE-NEXT:    packssdw %xmm1, %xmm0
 ; SSE-NEXT:    packsswb %xmm0, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testb $1, %al
 ; SSE-NEXT:    je .LBB4_1
 ; SSE-NEXT:  # %bb.2: # %cond.load
-; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    movdqu c+12(%rip), %xmm0
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    jne .LBB4_4
 ; SSE-NEXT:    jmp .LBB4_5
 ; SSE-NEXT:  .LBB4_1:
 ; SSE-NEXT:    # implicit-def: $xmm0
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB4_5
 ; SSE-NEXT:  .LBB4_4: # %cond.load1

 ; AVX1-NEXT:  .LBB4_48: # %else102
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ; AVX1-NEXT:  .LBB4_1: # %cond.load
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovdqu c+12(%rip), %xmm1
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB4_4
 ; AVX1-NEXT:  .LBB4_3: # %cond.load1
 ; AVX1-NEXT:    vpinsrd $1, c+12(%rip), %xmm1, %xmm3
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
 ; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    je .LBB4_6
 ; AVX1-NEXT:  .LBB4_5: # %cond.load4

 ; AVX2-NEXT:  # %bb.47: # %cond.load99
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX2-NEXT:    vpinsrd $3, c+28(%rip), %xmm2, %xmm2
 ; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; AVX2-NEXT:  .LBB4_48: # %else102
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ; AVX2-NEXT:  .LBB4_1: # %cond.load
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vmovdqu c+12(%rip), %xmm1
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB4_4
 ; AVX2-NEXT:  .LBB4_3: # %cond.load1
 ; AVX2-NEXT:    vpinsrd $1, c+12(%rip), %xmm1, %xmm2
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB4_6
 ; AVX2-NEXT:  .LBB4_5: # %cond.load4