[DAGCombine] reduceBuildVecToShuffle(): sort input vectors by decreasing size

[lebedev.ri]

Summary

The sorting, obviously, must be stable, else we will have random assembly fluctuations.

Apparently there was no test coverage that would benefit from that,
so i've added one test.

The sorting consists of two parts - just sort the input vectors,
and recompute the shuffle mask -> input vector mapping.
I don't believe we need to do anything else.

[RKSimon]

A few nits but the premise seems sound - did you encounter real world cases that led you to this?

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19124

assert message

19275

Move this inside reduceBuildVecToShuffle?

19421

assert message (or merge asserts)

[lebedev.ri]

In D104187#2815568, @RKSimon wrote:

A few nits but the premise seems sound

did you encounter real world cases that led you to this?

Not particularly, was just looking at the shuffle handling code and saw this.

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19275

I actually want to pull this into STLExtras or something,
this seems like a common patter.

[RKSimon]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19275

SGTM - but until then moving it inside reduceBuildVecToShuffle is still probably cleaner - maybe add a TODO comment?

[lebedev.ri]

@RKSimon thank you for taking a look!
Addressing review notes.

[RKSimon]

LGTM - cheers

[lebedev.ri]

Thank you for the review!
I should hopefully have a few more follow-ups for createBuildVecShuffle() soon..

[jpienaar]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19277

Seems to be failing to build for some compiler configs: https://lab.llvm.org/buildbot/#/builders/13/builds/8869/steps/6/logs/stdio

[lebedev.ri]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19277

Yes, already fixed.

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N,
   // If we artificially split a vector in two already, then the offsets in the
   // operands will all be based off of VecIn1, even those in VecIn2.
   unsigned Vec2Offset = DidSplitVec ? 0 : InVT1.getVectorNumElements();
 
   uint64_t VTSize = VT.getFixedSizeInBits();
   uint64_t InVT1Size = InVT1.getFixedSizeInBits();
   uint64_t InVT2Size = InVT2.getFixedSizeInBits();
 
+  assert(InVT2Size <= InVT1Size &&
+         "Inputs must be sorted to be in non-increasing vector size order.");
+
   // We can't generate a shuffle node with mismatched input and output types.
   // Try to make the types match the type of the output.
   if (InVT1 != VT || InVT2 != VT) {
     if ((VTSize % InVT1Size == 0) && InVT1 == InVT2) {
       // If the output vector length is a multiple of both input lengths,
       // we can concatenate them and pad the rest with undefs.
       unsigned NumConcats = VTSize / InVT1Size;
       assert(NumConcats >= 2 && "Concat needs at least two inputs!");
if ((VTSize % InVT1Size == 0) && InVT1 == InVT2) {
         // If we only have one input vector, and it's twice the size of the
         // output, split it in two.
         VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1,
                              DAG.getVectorIdxConstant(NumElems, DL));
         VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx);
         // Since we now have shorter input vectors, adjust the offset of the
         // second vector's start.
         Vec2Offset = NumElems;
-      } else if (InVT2Size <= InVT1Size) {
+      } else {
+        assert(InVT2Size <= InVT1Size &&
+               "Second input is not going to be larger than the first one.");
+
         // VecIn1 is wider than the output, and we have another, possibly
         // smaller input. Pad the smaller input with undefs, shuffle at the
         // input vector width, and extract the output.
         // The shuffle type is different than VT, so check legality again.
         if (LegalOperations &&
             !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, InVT1))
           return SDValue();
 
         // Legalizing INSERT_SUBVECTOR is tricky - you basically have to
         // lower it back into a BUILD_VECTOR. So if the inserted type is
         // illegal, don't even try.
         if (InVT1 != InVT2) {
           if (!TLI.isTypeLegal(InVT2))
             return SDValue();
           VecIn2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT1,
                                DAG.getUNDEF(InVT1), VecIn2, ZeroIdx);
         }
         ShuffleNumElems = NumElems * 2;
-      } else {
-        // Both VecIn1 and VecIn2 are wider than the output, and VecIn2 is wider
-        // than VecIn1. We can't handle this for now - this case will disappear
-        // when we start sorting the vectors by type.
-        return SDValue();
       }
     } else if (InVT2Size * 2 == VTSize && InVT1Size == VTSize) {
       SmallVector<SDValue, 2> ConcatOps(2, DAG.getUNDEF(InVT2));
       ConcatOps[0] = VecIn2;
       VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
     } else {
       // TODO: Support cases where the length mismatch isn't exactly by a
       // factor of 2.

 }
 
 // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
 // operations. If the types of the vectors we're extracting from allow it,
 // turn this into a vector_shuffle node.
 SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) {
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
 
+  // FIXME: promote to STLExtras.
+  auto getFirstIndexOf = [](auto &&Range, const auto &Val) {
+    auto I = find(Range, Val);
+    if (I == Range.end())
+      return -1L;
+    return std::distance(Range.begin(), I);
+  };
+
   // Only type-legal BUILD_VECTOR nodes are converted to shuffle nodes.
   if (!isTypeLegal(VT))
     return SDValue();
 
   if (SDValue V = reduceBuildVecToShuffleWithZero(N, DAG))
     return V;
 
   // May only combine to shuffle after legalize if shuffle is legal.
for (unsigned i = 0; i != NumElems; ++i) {
     // All inputs must have the same element type as the output.
     if (VT.getVectorElementType() !=
         ExtractedFromVec.getValueType().getVectorElementType())
       return SDValue();
 
     // Have we seen this input vector before?
     // The vectors are expected to be tiny (usually 1 or 2 elements), so using
     // a map back from SDValues to numbers isn't worth it.
-    unsigned Idx = std::distance(VecIn.begin(), find(VecIn, ExtractedFromVec));
-    if (Idx == VecIn.size())
+    int Idx = getFirstIndexOf(VecIn, ExtractedFromVec);
+    if (Idx == -1) { // A new source vector?
+      Idx = VecIn.size();
       VecIn.push_back(ExtractedFromVec);
+    }
 
     VectorMask[i] = Idx;
   }
 
   // If we didn't find at least one input vector, bail out.
   if (VecIn.size() < 2)
     return SDValue();
 
if (InVT.isSimple() && NearestPow2 > 2 && MaxIndex < NearestPow2 &&
           if (VectorMask[i] <= 0)
             continue;
           VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2;
         }
       }
     }
   }
 
-  // TODO: We want to sort the vectors by descending length, so that adjacent
-  // pairs have similar length, and the longer vector is always first in the
-  // pair.
+  // Sort input vectors by decreasing vector element count,
+  // while preserving the relative order of equally-sized vectors.
+  // Note that we keep the first "implicit zero vector as-is.
+  SmallVector<SDValue, 8> SortedVecIn(VecIn);
+  llvm::stable_sort(MutableArrayRef<SDValue>(SortedVecIn).drop_front(),
+                    [](const SDValue &a, const SDValue &b) {
+                      return a.getValueType().getVectorNumElements() >
+                             b.getValueType().getVectorNumElements();
+                    });
+
+  // We now also need to rebuild the VectorMask, because it referenced element
+  // order in VecIn, and we just sorted them.
+  for (int &SourceVectorIndex : VectorMask) {
+    if (SourceVectorIndex <= 0)
+      continue;
+    unsigned Idx = getFirstIndexOf(SortedVecIn, VecIn[SourceVectorIndex]);
+    assert(Idx > 0 && Idx < SortedVecIn.size() &&
+           VecIn[SourceVectorIndex] == SortedVecIn[Idx] && "Remapping failure");
+    SourceVectorIndex = Idx;
+  }
+
+  VecIn = std::move(SortedVecIn);
 
   // TODO: Should this fire if some of the input vectors has illegal type (like
   // it does now), or should we let legalization run its course first?
 
   // Shuffle phase:
   // Take pairs of vectors, and shuffle them so that the result has elements
   // from these vectors in the correct places.
   // For example, given:

llvm/test/CodeGen/X86/oddshuffles.ll

 ; SSE42-NEXT:    movdqu %xmm1, (%rcx)
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: interleave_24i16_out_reverse:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqu (%rdi), %xmm0
 ; AVX1-NEXT:    vmovdqu 16(%rdi), %xmm1
 ; AVX1-NEXT:    vmovdqu 32(%rdi), %xmm2
-; AVX1-NEXT:    vpextrw $6, %xmm1, %eax
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[14,15,8,9,2,3,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
-; AVX1-NEXT:    vpextrw $3, %xmm1, %eax
-; AVX1-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
-; AVX1-NEXT:    vmovd %xmm1, %eax
-; AVX1-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
-; AVX1-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX1-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
-; AVX1-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX1-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm3
-; AVX1-NEXT:    vpextrw $5, %xmm1, %eax
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,12,13,6,7,0,1,10,11,4,5]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[3,4,5,6,7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[12,13,6,7,0,1,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT:    vpinsrw $3, %eax, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrw $2, %xmm1, %eax
-; AVX1-NEXT:    vpinsrw $4, %eax, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX1-NEXT:    vpinsrw $5, %eax, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX1-NEXT:    vpinsrw $6, %eax, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX1-NEXT:    vpinsrw $7, %eax, %xmm4, %xmm4
-; AVX1-NEXT:    vpextrw $2, %xmm2, %eax
-; AVX1-NEXT:    vpextrw $5, %xmm2, %edi
-; AVX1-NEXT:    vmovd %edi, %xmm2
-; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrw $7, %xmm1, %eax
-; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrw $4, %xmm1, %eax
-; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
-; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm1
-; AVX1-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX1-NEXT:    vpinsrw $5, %eax, %xmm1, %xmm1
-; AVX1-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX1-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,10,11,4,5,14,15,8,9,2,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm5[3,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,14,15,8,9,2,3,12,13,6,7,0,1]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3,4,5,6,7]
 ; AVX1-NEXT:    vmovdqu %xmm3, (%rsi)
 ; AVX1-NEXT:    vmovdqu %xmm4, (%rdx)
 ; AVX1-NEXT:    vmovdqu %xmm0, (%rcx)
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: interleave_24i16_out_reverse:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX2-SLOW-NEXT:    vmovdqu 16(%rdi), %xmm1
-; AVX2-SLOW-NEXT:    vmovdqu 32(%rdi), %xmm2
-; AVX2-SLOW-NEXT:    vpextrw $6, %xmm1, %eax
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[14,15,8,9,2,3,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
-; AVX2-SLOW-NEXT:    vpextrw $3, %xmm1, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
-; AVX2-SLOW-NEXT:    vmovd %xmm1, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
-; AVX2-SLOW-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
-; AVX2-SLOW-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm3
-; AVX2-SLOW-NEXT:    vpextrw $5, %xmm1, %eax
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[12,13,6,7,0,1,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpinsrw $3, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT:    vpextrw $2, %xmm1, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $4, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $5, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $6, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $7, %eax, %xmm4, %xmm4
-; AVX2-SLOW-NEXT:    vpextrw $2, %xmm2, %eax
-; AVX2-SLOW-NEXT:    vpextrw $5, %xmm2, %edi
-; AVX2-SLOW-NEXT:    vmovd %edi, %xmm2
-; AVX2-SLOW-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vpextrw $7, %xmm1, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vpextrw $4, %xmm1, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vpextrw $1, %xmm1, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm1
-; AVX2-SLOW-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $5, %eax, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vmovd %xmm0, %eax
-; AVX2-SLOW-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    vmovdqu %xmm3, (%rsi)
-; AVX2-SLOW-NEXT:    vmovdqu %xmm4, (%rdx)
-; AVX2-SLOW-NEXT:    vmovdqu %xmm0, (%rcx)
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: interleave_24i16_out_reverse:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX2-FAST-NEXT:    vmovdqu 16(%rdi), %xmm1
-; AVX2-FAST-NEXT:    vmovdqu 32(%rdi), %xmm2
-; AVX2-FAST-NEXT:    vpextrw $6, %xmm1, %eax
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[14,15,8,9,2,3,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
-; AVX2-FAST-NEXT:    vpextrw $3, %xmm1, %eax
-; AVX2-FAST-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
-; AVX2-FAST-NEXT:    vmovd %xmm1, %eax
-; AVX2-FAST-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
-; AVX2-FAST-NEXT:    vpextrw $5, %xmm0, %eax
-; AVX2-FAST-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
-; AVX2-FAST-NEXT:    vpextrw $2, %xmm0, %eax
-; AVX2-FAST-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm3
-; AVX2-FAST-NEXT:    vpextrw $5, %xmm1, %eax
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[12,13,6,7,0,1,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpinsrw $3, %eax, %xmm4, %xmm4
-; AVX2-FAST-NEXT:    vpextrw $2, %xmm1, %eax
-; AVX2-FAST-NEXT:    vpinsrw $4, %eax, %xmm4, %xmm4
-; AVX2-FAST-NEXT:    vpextrw $7, %xmm0, %eax
-; AVX2-FAST-NEXT:    vpinsrw $5, %eax, %xmm4, %xmm4
-; AVX2-FAST-NEXT:    vpextrw $4, %xmm0, %eax
-; AVX2-FAST-NEXT:    vpinsrw $6, %eax, %xmm4, %xmm4
-; AVX2-FAST-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX2-FAST-NEXT:    vpinsrw $7, %eax, %xmm4, %xmm4
-; AVX2-FAST-NEXT:    vpextrw $7, %xmm1, %eax
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[10,11,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpextrw $4, %xmm1, %eax
-; AVX2-FAST-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpextrw $1, %xmm1, %eax
-; AVX2-FAST-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm1
-; AVX2-FAST-NEXT:    vpextrw $6, %xmm0, %eax
-; AVX2-FAST-NEXT:    vpinsrw $5, %eax, %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vpextrw $3, %xmm0, %eax
-; AVX2-FAST-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
-; AVX2-FAST-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm0
-; AVX2-FAST-NEXT:    vmovdqu %xmm3, (%rsi)
-; AVX2-FAST-NEXT:    vmovdqu %xmm4, (%rdx)
-; AVX2-FAST-NEXT:    vmovdqu %xmm0, (%rcx)
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: interleave_24i16_out_reverse:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX2-NEXT:    vmovdqu 16(%rdi), %xmm1
+; AVX2-NEXT:    vmovdqu 32(%rdi), %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm3 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0],xmm3[1,2],xmm1[3],xmm3[4,5],xmm1[6],xmm3[7]
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[14,15,8,9,2,3,12,13,6,7,0,1,10,11,4,5]
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[12,13,6,7,0,1,10,11,4,5,14,15,8,9,2,3]
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[10,11,4,5,14,15,8,9,2,3,12,13,6,7,0,1]
+; AVX2-NEXT:    vmovdqu %xmm3, (%rsi)
+; AVX2-NEXT:    vmovdqu %xmm4, (%rdx)
+; AVX2-NEXT:    vmovdqu %xmm0, (%rcx)
+; AVX2-NEXT:    retq
 ;
 ; XOP-LABEL: interleave_24i16_out_reverse:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vmovdqu (%rdi), %xmm0
 ; XOP-NEXT:    vmovdqu 16(%rdi), %xmm1
 ; XOP-NEXT:    vmovdqu 32(%rdi), %xmm2
-; XOP-NEXT:    vpextrw $5, %xmm0, %eax
-; XOP-NEXT:    vpperm {{.*#+}} xmm3 = xmm2[14,15,8,9,2,3],xmm1[12,13,6,7,0,1],xmm2[u,u,u,u]
-; XOP-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
-; XOP-NEXT:    vpextrw $2, %xmm0, %eax
-; XOP-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm3
-; XOP-NEXT:    vpperm {{.*#+}} xmm4 = xmm2[12,13,6,7,0,1],xmm1[10,11,4,5],xmm2[u,u,u,u,u,u]
-; XOP-NEXT:    vpperm {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,7,8,9],xmm0[14,15,8,9,2,3]
-; XOP-NEXT:    vpperm {{.*#+}} xmm1 = xmm2[10,11,4,5],xmm1[14,15,8,9,2,3,u,u,u,u,u,u]
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5,6,7,8,9],xmm0[12,13,6,7,0,1]
+; XOP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
+; XOP-NEXT:    vpperm {{.*#+}} xmm3 = xmm2[14,15,8,9,2,3],xmm3[12,13,6,7,0,1,10,11,4,5]
+; XOP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
+; XOP-NEXT:    vpperm {{.*#+}} xmm4 = xmm2[12,13,6,7,0,1],xmm4[10,11,4,5,14,15,8,9,2,3]
+; XOP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm2[10,11,4,5],xmm0[14,15,8,9,2,3,12,13,6,7,0,1]
 ; XOP-NEXT:    vmovdqu %xmm3, (%rsi)
 ; XOP-NEXT:    vmovdqu %xmm4, (%rdx)
 ; XOP-NEXT:    vmovdqu %xmm0, (%rcx)
 ; XOP-NEXT:    retq
   %wide.vec.reverse = load <24 x i16>, <24 x i16>* %p, align 4
   %wide.vec = shufflevector <24 x i16> %wide.vec.reverse, <24 x i16> undef, <24 x i32> <i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   %s1 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
   %s2 = shufflevector <24 x i16> %wide.vec, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>