[X86] Limit X86InterleavedAccessGroup to handle the same type case only

[pengfei]

Summary

The current implementation assumes the destination type of shuffle is the same as the decomposed ones. Add the check to avoid crush when the condition is not satisfied.

This fixes PR37616.

[RKSimon]

LGTM - cheers

[RKSimon]

Sorry @pengfei but I'm seeing this after this patch (possibly only on EXPENSIVE_CHECKS?)

Pass modifies its input and doesn't report it: Interleaved Access Pass
Pass modifies its input and doesn't report it
UNREACHABLE executed at E:\llvm\llvm-project\llvm\lib\IR\LegacyPassManager.cpp:1445!
PLEASE submit a bug report to https://bugs.llvm.org/ and include the crash backtrace.

[pengfei]

Thanks @RKSimon , I will investigate it.

[RKSimon]

That's great - thanks @pengfei !

llvm/lib/Target/X86/X86InterleavedAccess.cpp

 // Lowers this interleaved access group into X86-specific
 // instructions/intrinsics.
 bool X86InterleavedAccessGroup::lowerIntoOptimizedSequence() {
   SmallVector<Instruction *, 4> DecomposedVectors;
   SmallVector<Value *, 4> TransposedVectors;
   auto *ShuffleTy = cast<FixedVectorType>(Shuffles[0]->getType());
 
   if (isa<LoadInst>(Inst)) {
-    // Try to generate target-sized register(/instruction).
-    decompose(Inst, Factor, ShuffleTy, DecomposedVectors);
-
     auto *ShuffleEltTy = cast<FixedVectorType>(Inst->getType());
     unsigned NumSubVecElems = ShuffleEltTy->getNumElements() / Factor;
-    // Perform matrix-transposition in order to compute interleaved
-    // results by generating some sort of (optimized) target-specific
-    // instructions.
-
     switch (NumSubVecElems) {
     default:
       return false;
     case 4:
-      transpose_4x4(DecomposedVectors, TransposedVectors);
-      break;
     case 8:
     case 16:
     case 32:
     case 64:
-      deinterleave8bitStride3(DecomposedVectors, TransposedVectors,
-                              NumSubVecElems);
+      if (ShuffleTy->getNumElements() != NumSubVecElems)
+        return false;
       break;
     }
 
+    // Try to generate target-sized register(/instruction).
+    decompose(Inst, Factor, ShuffleTy, DecomposedVectors);
+
+    // Perform matrix-transposition in order to compute interleaved
+    // results by generating some sort of (optimized) target-specific
+    // instructions.
+
+    if (NumSubVecElems == 4)
+      transpose_4x4(DecomposedVectors, TransposedVectors);
+    else
+      deinterleave8bitStride3(DecomposedVectors, TransposedVectors,
+                              NumSubVecElems);
+
     // Now replace the unoptimized-interleaved-vectors with the
     // transposed-interleaved vectors.
     for (unsigned i = 0, e = Shuffles.size(); i < e; ++i)
       Shuffles[i]->replaceAllUsesWith(TransposedVectors[Indices[i]]);
 
     return true;
   }
 

llvm/test/CodeGen/X86/x86-interleaved-access.ll

 ; AVX512-NEXT:    retq
   %x = load <4 x i64>, <4 x i64>* %s, align 8
   %x2 = shufflevector <4 x i64> %x, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   %x4 = shufflevector <8 x i64> %x2, <8 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %r = shufflevector <16 x i64> %x4, <16 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
   store <16 x i64> %r, <16 x i64>* %d, align 8
   ret void
 }
+
+define <2 x i64> @PR37616(<16 x i64>* %a0) {
+; AVX1-LABEL: PR37616:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps 16(%rdi), %xmm0
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX1-NEXT:    retq
+;
+; AVX2OR512-LABEL: PR37616:
+; AVX2OR512:       # %bb.0:
+; AVX2OR512-NEXT:    vmovaps (%rdi), %ymm0
+; AVX2OR512-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2]
+; AVX2OR512-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX2OR512-NEXT:    vzeroupper
+; AVX2OR512-NEXT:    retq
+  %load = load <16 x i64>, <16 x i64>* %a0, align 128
+  %shuffle = shufflevector <16 x i64> %load, <16 x i64> undef, <2 x i32> <i32 2, i32 6>
+  ret <2 x i64> %shuffle
+}