[DAGCombiner] recognize shuffle (shuffle X, Mask0), Mask --> splat X

[spatel]

Summary

We get the simple cases of this via demanded elements and other folds, but that doesn't work if the values have >1 use, so add a dedicated match for the pattern.

We already have this transform in IR, but it doesn't help the motivating x86 tests (based on PR42024) because the shuffles don't exist until after legalization and other combines have happened. The AArch64 test shows a minimal IR example of the problem.

[RKSimon]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19298

Check for isShuffleMaskLegal() ?

[lebedev.ri]

Sounds good to me modulo legality check.

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19264

I'd suggest OuterShuf and InnerShuf

19277

CombinedMask? Not sure.

19282–19286

Might it be more obvious to do

int MaskElt = Mask[i];
if (MaskElt == -1)
  continue;
// Peek through the shuffle masks to get the underlying source element.
int MaskOp0Elt = MaskOp0[MaskElt];
if (MaskOp0Elt == -1)
  continue;

?

[spatel]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19298

That's the conservatively correct approach, but I'm not sure if we want that limit. We know that the shuffle that we are creating is the same type that we started with, and it's a splat, so we treat that as a special-case that any target should handle efficiently:
https://llvm.org/docs/CodeGenerator.html#selectiondag-legalize-phase

I don't have a real-world example of this, but it's easy to hack an x86 test to show an improvement without the check:

define void @disguised_splat(<8 x i32> %x, <8 x i32>* %p0, <8 x i32>* %p1) {
  %shuf0 = shufflevector <8 x i32> %x, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 0, i32 0, i32 3, i32 3, i32 4>
  store <8 x i32> %shuf0, <8 x i32>* %p0
  %shuf1 = shufflevector <8 x i32> %shuf0, <8 x i32> undef, <8 x i32> <i32 4, i32 3, i32 4, i32 3, i32 4, i32 4, i32 3, i32 3>
  store <8 x i32> %shuf1, <8 x i32>* %p1
  ret void
}

With mask legality check, we'll fail to detect the splat before the op gets split:

$ llc -o - shuf.ll
	shufps	$48, %xmm0, %xmm1       ## xmm1 = xmm1[0,0],xmm0[3,0]
	movaps	%xmm0, %xmm2
	shufps	$44, %xmm1, %xmm2       ## xmm2 = xmm2[0,3],xmm1[2,0]
	pshufd	$36, %xmm0, %xmm1       ## xmm1 = xmm0[0,1,2,0]
	movdqa	%xmm1, (%rdi)
	movaps	%xmm2, 16(%rdi)
	pshufd	$0, %xmm0, %xmm2        ## xmm2 = xmm0[0,0,0,0]
	shufps	$240, %xmm1, %xmm0      ## xmm0 = xmm0[0,0],xmm1[3,3]
	movdqa	%xmm2, (%rsi)
	movaps	%xmm0, 16(%rsi)

Without mask legality check, we can simplify before splitting:

$ llc -o - shuf.ll 
	shufps	$48, %xmm0, %xmm1       ## xmm1 = xmm1[0,0],xmm0[3,0]
	pshufd	$36, %xmm0, %xmm2       ## xmm2 = xmm0[0,1,2,0]
	pshufd	$0, %xmm0, %xmm3        ## xmm3 = xmm0[0,0,0,0]
	shufps	$44, %xmm1, %xmm0       ## xmm0 = xmm0[0,3],xmm1[2,0]
	movdqa	%xmm2, (%rdi)
	movaps	%xmm0, 16(%rdi)
	movdqa	%xmm3, (%rsi)
	movdqa	%xmm3, 16(%rsi)

The safe thing would be to add the mask legality predicate here, then remove it as a follow-up (after adding a test like above to show a diff)...then wait to see if there are any complaints. :)

[spatel]

Patch updated:

1. Added shuffle mask legality check.
2. Improved variable names/logic.
3. Added assert for types.

[lebedev.ri]

lg

[RKSimon]

LGTM - cheers

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/CodeGen/DAGCombine.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf,
   for (int Idx : ShufMask)
     NewMask.push_back(Idx == -1 ? -1 : SplatMask[Idx]);
 
   return DAG.getVectorShuffle(Splat->getValueType(0), SDLoc(Splat),
                               Splat->getOperand(0), Splat->getOperand(1),
                               NewMask);
 }
 
+/// Combine shuffle of shuffle of the form:
+/// shuf (shuf X, undef, InnerMask), undef, OuterMask --> splat X
+static SDValue formSplatFromShuffles(ShuffleVectorSDNode *OuterShuf,
+                                     SelectionDAG &DAG) {
+  if (!OuterShuf->getOperand(1).isUndef())
+    return SDValue();
+  auto *InnerShuf = dyn_cast<ShuffleVectorSDNode>(OuterShuf->getOperand(0));
+  if (!InnerShuf || !InnerShuf->getOperand(1).isUndef())
+    return SDValue();
+
+  ArrayRef<int> OuterMask = OuterShuf->getMask();
+  ArrayRef<int> InnerMask = InnerShuf->getMask();
+  unsigned NumElts = OuterMask.size();
+  assert(NumElts == InnerMask.size() && "Mask length mismatch");
+  SmallVector<int, 32> CombinedMask(NumElts, -1);
+  int SplatIndex = -1;
+  for (unsigned i = 0; i != NumElts; ++i) {
+    // Undef lanes remain undef.
+    int OuterMaskElt = OuterMask[i];
+    if (OuterMaskElt == -1)
+      continue;
+
+    // Peek through the shuffle masks to get the underlying source element.
+    int InnerMaskElt = InnerMask[OuterMaskElt];
+    if (InnerMaskElt == -1)
+      continue;
+
+    // Initialize the splatted element.
+    if (SplatIndex == -1)
+      SplatIndex = InnerMaskElt;
+
+    // Non-matching index - this is not a splat.
+    if (SplatIndex != InnerMaskElt)
+      return SDValue();
+
+    CombinedMask[i] = InnerMaskElt;
+  }
+  assert(all_of(CombinedMask, [](int M) { return M == -1; }) ||
+         getSplatIndex(CombinedMask) != -1 && "Expected a splat mask");
+
+  // TODO: The transform may be a win even if the mask is not legal.
+  EVT VT = OuterShuf->getValueType(0);
+  assert(VT == InnerShuf->getValueType(0) && "Expected matching shuffle types");
+  if (!DAG.getTargetLoweringInfo().isShuffleMaskLegal(CombinedMask, VT))
+    return SDValue();
+
+  return DAG.getVectorShuffle(VT, SDLoc(OuterShuf), InnerShuf->getOperand(0),
+                              InnerShuf->getOperand(1), CombinedMask);
+}
+
 /// If the shuffle mask is taking exactly one element from the first vector
 /// operand and passing through all other elements from the second vector
 /// operand, return the index of the mask element that is choosing an element
 /// from the first operand. Otherwise, return -1.
 static int getShuffleMaskIndexOfOneElementFromOp0IntoOp1(ArrayRef<int> Mask) {
   int MaskSize = Mask.size();
   int EltFromOp0 = -1;
   // TODO: This does not match if there are undef elements in the shuffle mask.
SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
 
   if (SDValue InsElt = replaceShuffleOfInsert(SVN, DAG))
     return InsElt;
 
   // A shuffle of a single vector that is a splatted value can always be folded.
   if (SDValue V = combineShuffleOfSplatVal(SVN, DAG))
     return V;
 
+  if (SDValue V = formSplatFromShuffles(SVN, DAG))
+    return V;
+
   // If it is a splat, check if the argument vector is another splat or a
   // build_vector.
   if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) {
     int SplatIndex = SVN->getSplatIndex();
     if (N0.hasOneUse() && TLI.isExtractVecEltCheap(VT, SplatIndex) &&
         TLI.isBinOp(N0.getOpcode()) && N0.getNode()->getNumValues() == 1) {
       // splat (vector_bo L, R), Index -->
       // splat (scalar_bo (extelt L, Index), (extelt R, Index))

llvm/test/CodeGen/AArch64/arm64-dup.ll

 }
 
 define void @disguised_dup(<4 x float> %x, <4 x float>* %p1, <4 x float>* %p2) {
 ; CHECK-LABEL: disguised_dup:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    dup.4s v1, v0[0]
 ; CHECK-NEXT:    ext.16b v0, v0, v0, #12
 ; CHECK-NEXT:    ext.16b v0, v0, v1, #8
-; CHECK-NEXT:    zip2.4s v1, v0, v0
-; CHECK-NEXT:    ext.16b v1, v0, v1, #12
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    str q1, [x1]
 ; CHECK-NEXT:    ret
   %shuf = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 1, i32 2, i32 0, i32 0>
   %dup = shufflevector <4 x float> %shuf, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 3>
   store <4 x float> %shuf, <4 x float>* %p1, align 8
   store <4 x float> %dup, <4 x float>* %p2, align 8
   ret void
 }

llvm/test/CodeGen/X86/vector-reduce-mul.ll

; AVX512-NEXT:    retq
   %1 = call i32 @llvm.experimental.vector.reduce.mul.v2i32(<2 x i32> %a0)
   ret i32 %1
 }
 
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-LABEL: test_v4i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,1,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v4i32:

 ; SSE2-LABEL: test_v8i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,2,0,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,0,2,2]
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm0
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0

 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm4, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm5, %xmm2
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,0,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,0,2,2]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm0
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v16i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulld %xmm3, %xmm1

 ; SSE2-NEXT:    pmuludq %xmm4, %xmm0
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm0
 ; SSE2-NEXT:    pmuludq %xmm7, %xmm3
 ; SSE2-NEXT:    pmuludq %xmm5, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[2,2,0,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[2,0,2,2]
 ; SSE2-NEXT:    pmuludq %xmm11, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v32i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulld %xmm6, %xmm2

llvm/test/CodeGen/X86/x86-interleaved-access.ll

; AVX512-NEXT:    retq
   %r = shufflevector <32 x i32> %x4, <32 x i32> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
   store <32 x i32> %r, <32 x i32>* %d, align 4
   ret void
 }
 
 define void @splat4_v4f64_load_store(<4 x double>* %s, <16 x double>* %d) {
 ; AVX1-LABEL: splat4_v4f64_load_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovupd (%rdi), %ymm0
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[0,1,0,1]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX1-NEXT:    vmovddup {{.*#+}} ymm2 = ymm1[0,0,2,2]
-; AVX1-NEXT:    vmovddup {{.*#+}} ymm3 = ymm0[0,0,2,2]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[1,1,3,3]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,1,3,3]
-; AVX1-NEXT:    vmovupd %ymm0, 96(%rsi)
-; AVX1-NEXT:    vmovupd %ymm3, 64(%rsi)
-; AVX1-NEXT:    vmovupd %ymm1, 32(%rsi)
-; AVX1-NEXT:    vmovupd %ymm2, (%rsi)
+; AVX1-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX1-NEXT:    vbroadcastsd 16(%rdi), %ymm1
+; AVX1-NEXT:    vbroadcastsd 8(%rdi), %ymm2
+; AVX1-NEXT:    vbroadcastsd 24(%rdi), %ymm3
+; AVX1-NEXT:    vmovups %ymm3, 96(%rsi)
+; AVX1-NEXT:    vmovups %ymm1, 64(%rsi)
+; AVX1-NEXT:    vmovups %ymm2, 32(%rsi)
+; AVX1-NEXT:    vmovups %ymm0, (%rsi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splat4_v4f64_load_store:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovups (%rdi), %ymm0
-; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm1
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm0[2,2,2,2]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm3 = ymm0[1,1,1,1]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3]
-; AVX2-NEXT:    vmovups %ymm0, 96(%rsi)
-; AVX2-NEXT:    vmovups %ymm2, 64(%rsi)
-; AVX2-NEXT:    vmovups %ymm3, 32(%rsi)
-; AVX2-NEXT:    vmovups %ymm1, (%rsi)
+; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX2-NEXT:    vbroadcastsd 16(%rdi), %ymm1
+; AVX2-NEXT:    vbroadcastsd 8(%rdi), %ymm2
+; AVX2-NEXT:    vbroadcastsd 24(%rdi), %ymm3
+; AVX2-NEXT:    vmovups %ymm3, 96(%rsi)
+; AVX2-NEXT:    vmovups %ymm1, 64(%rsi)
+; AVX2-NEXT:    vmovups %ymm2, 32(%rsi)
+; AVX2-NEXT:    vmovups %ymm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splat4_v4f64_load_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovups (%rdi), %ymm0
-; AVX512-NEXT:    vbroadcastsd (%rdi), %ymm1
-; AVX512-NEXT:    vpermpd {{.*#+}} ymm2 = ymm0[2,2,2,2]
-; AVX512-NEXT:    vpermpd {{.*#+}} ymm3 = ymm0[1,1,1,1]
-; AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3]
+; AVX512-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX512-NEXT:    vbroadcastsd 16(%rdi), %ymm1
+; AVX512-NEXT:    vbroadcastsd 8(%rdi), %ymm2
+; AVX512-NEXT:    vbroadcastsd 24(%rdi), %ymm3
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512-NEXT:    vinsertf64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512-NEXT:    vmovups %zmm0, 64(%rsi)
-; AVX512-NEXT:    vmovups %zmm1, (%rsi)
+; AVX512-NEXT:    vmovups %zmm1, 64(%rsi)
+; AVX512-NEXT:    vmovups %zmm0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %x = load <4 x double>, <4 x double>* %s, align 8
   %x2 = shufflevector <4 x double> %x, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   %x4 = shufflevector <8 x double> %x2, <8 x double> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %r = shufflevector <16 x double> %x4, <16 x double> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
   store <16 x double> %r, <16 x double>* %d, align 8
   ret void
 }
 
 define void @splat4_v4i64_load_store(<4 x i64>* %s, <16 x i64>* %d) {
 ; AVX1-LABEL: splat4_v4i64_load_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovupd (%rdi), %ymm0
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[0,1,0,1]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX1-NEXT:    vmovddup {{.*#+}} ymm2 = ymm1[0,0,2,2]
-; AVX1-NEXT:    vmovddup {{.*#+}} ymm3 = ymm0[0,0,2,2]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[1,1,3,3]
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,1,3,3]
-; AVX1-NEXT:    vmovupd %ymm0, 96(%rsi)
-; AVX1-NEXT:    vmovupd %ymm3, 64(%rsi)
-; AVX1-NEXT:    vmovupd %ymm1, 32(%rsi)
-; AVX1-NEXT:    vmovupd %ymm2, (%rsi)
+; AVX1-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX1-NEXT:    vbroadcastsd 16(%rdi), %ymm1
+; AVX1-NEXT:    vbroadcastsd 8(%rdi), %ymm2
+; AVX1-NEXT:    vbroadcastsd 24(%rdi), %ymm3
+; AVX1-NEXT:    vmovups %ymm3, 96(%rsi)
+; AVX1-NEXT:    vmovups %ymm1, 64(%rsi)
+; AVX1-NEXT:    vmovups %ymm2, 32(%rsi)
+; AVX1-NEXT:    vmovups %ymm0, (%rsi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splat4_v4i64_load_store:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovups (%rdi), %ymm0
-; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm1
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm0[2,2,2,2]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm3 = ymm0[1,1,1,1]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3]
-; AVX2-NEXT:    vmovups %ymm0, 96(%rsi)
-; AVX2-NEXT:    vmovups %ymm2, 64(%rsi)
-; AVX2-NEXT:    vmovups %ymm3, 32(%rsi)
-; AVX2-NEXT:    vmovups %ymm1, (%rsi)
+; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX2-NEXT:    vbroadcastsd 16(%rdi), %ymm1
+; AVX2-NEXT:    vbroadcastsd 8(%rdi), %ymm2
+; AVX2-NEXT:    vbroadcastsd 24(%rdi), %ymm3
+; AVX2-NEXT:    vmovups %ymm3, 96(%rsi)
+; AVX2-NEXT:    vmovups %ymm1, 64(%rsi)
+; AVX2-NEXT:    vmovups %ymm2, 32(%rsi)
+; AVX2-NEXT:    vmovups %ymm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splat4_v4i64_load_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovups (%rdi), %ymm0
-; AVX512-NEXT:    vbroadcastsd (%rdi), %ymm1
-; AVX512-NEXT:    vpermpd {{.*#+}} ymm2 = ymm0[2,2,2,2]
-; AVX512-NEXT:    vpermpd {{.*#+}} ymm3 = ymm0[1,1,1,1]
-; AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3]
+; AVX512-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX512-NEXT:    vbroadcastsd 16(%rdi), %ymm1
+; AVX512-NEXT:    vbroadcastsd 8(%rdi), %ymm2
+; AVX512-NEXT:    vbroadcastsd 24(%rdi), %ymm3
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512-NEXT:    vinsertf64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT:    vinsertf64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512-NEXT:    vmovups %zmm0, 64(%rsi)
-; AVX512-NEXT:    vmovups %zmm1, (%rsi)
+; AVX512-NEXT:    vmovups %zmm1, 64(%rsi)
+; AVX512-NEXT:    vmovups %zmm0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %x = load <4 x i64>, <4 x i64>* %s, align 8
   %x2 = shufflevector <4 x i64> %x, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   %x4 = shufflevector <8 x i64> %x2, <8 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %r = shufflevector <16 x i64> %x4, <16 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
   store <16 x i64> %r, <16 x i64>* %d, align 8
   ret void
 }