[GISel][CombinerHelper] Add a combine turning shuffle_vector into concat_vectors

[qcolombet]

Summary

Teach the CombinerHelper how to turn shuffle_vectors, that
concatenate vectors, into concat_vectors and add this combine
to the AArch64 pre-legalizer combiner.

[arsenm]

lib/CodeGen/GlobalISel/CombinerHelper.cpp

190 ↗

(On Diff #225543)

s/unsigned/Register/

235 ↗

(On Diff #225543)

s/unsigned/Register

[qcolombet]

lib/CodeGen/GlobalISel/CombinerHelper.cpp

190 ↗

(On Diff #225543)

Good catch!
(Habits are hard to kill :))

[qcolombet]

• Replace some unsigned into Register

[arsenm]

LGTM

[qcolombet]

Thanks for the quick review @arsenm!

llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

public:
   /// \pre MI.getOpcode() == G_CONCAT_VECTORS.
   bool matchCombineConcatVectors(MachineInstr &MI, bool &IsUndef,
                                  SmallVectorImpl<Register> &Ops);
   /// Replace \p MI with a flattened build_vector with \p Ops or an
   /// implicit_def if IsUndef is true.
   void applyCombineConcatVectors(MachineInstr &MI, bool IsUndef,
                                  const ArrayRef<Register> Ops);
 
+  /// Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS.
+  /// Returns true if MI changed.
+  ///
+  /// \pre MI.getOpcode() == G_SHUFFLE_VECTOR.
+  bool tryCombineShuffleVector(MachineInstr &MI);
+  /// Check if the G_SHUFFLE_VECTOR \p MI can be replaced by a
+  /// concat_vectors.
+  /// \p Ops will contain the operands needed to produce the flattened
+  /// concat_vectors.
+  ///
+  /// \pre MI.getOpcode() == G_SHUFFLE_VECTOR.
+  bool matchCombineShuffleVector(MachineInstr &MI,
+                                 SmallVectorImpl<Register> &Ops);
+  /// Replace \p MI with a concat_vectors with \p Ops.
+  void applyCombineShuffleVector(MachineInstr &MI,
+                                 const ArrayRef<Register> Ops);
+
   /// Optimize memcpy intrinsics et al, e.g. constant len calls.
   /// /p MaxLen if non-zero specifies the max length of a mem libcall to inline.
   ///
   /// For example (pre-indexed):
   ///
   ///     $addr = G_GEP $base, $offset
   ///     [...]
   ///     $val = G_LOAD $addr

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

void CombinerHelper::applyCombineConcatVectors(
   if (IsUndef)
     Builder.buildUndef(NewDstReg);
   else
     Builder.buildBuildVector(NewDstReg, Ops);
   MI.eraseFromParent();
   replaceRegWith(MRI, DstReg, NewDstReg);
 }
 
+bool CombinerHelper::tryCombineShuffleVector(MachineInstr &MI) {
+  SmallVector<Register, 4> Ops;
+  if (matchCombineShuffleVector(MI, Ops)) {
+    applyCombineShuffleVector(MI, Ops);
+    return true;
+  }
+  return false;
+}
+
+bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI,
+                                               SmallVectorImpl<Register> &Ops) {
+  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
+         "Invalid instruction kind");
+  LLT DstType = MRI.getType(MI.getOperand(0).getReg());
+  Register Src1 = MI.getOperand(1).getReg();
+  LLT SrcType = MRI.getType(Src1);
+  unsigned DstNumElts = DstType.getNumElements();
+  unsigned SrcNumElts = SrcType.getNumElements();
+
+  // If the resulting vector is smaller than the size of the source
+  // vectors being concatenated, we won't be able to replace the
+  // shuffle vector into a concat_vectors.
+  //
+  // Note: We may still be able to produce a concat_vectors fed by
+  //       extract_vector_elt and so on. It is less clear that would
+  //       be better though, so don't bother for now.
+  if (DstNumElts < 2 * SrcNumElts)
+    return false;
+
+  // Check that the shuffle mask can be broken evenly between the
+  // different sources.
+  if (DstNumElts % SrcNumElts != 0)
+    return false;
+
+  // Mask length is a multiple of the source vector length.
+  // Check if the shuffle is some kind of concatenation of the input
+  // vectors.
+  unsigned NumConcat = DstNumElts / SrcNumElts;
+  SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
+  SmallVector<int, 8> Mask;
+  ShuffleVectorInst::getShuffleMask(MI.getOperand(3).getShuffleMask(), Mask);
+  for (unsigned i = 0; i != DstNumElts; ++i) {
+    int Idx = Mask[i];
+    // Undef value.
+    if (Idx < 0)
+      continue;
+    // Ensure the indices in each SrcType sized piece are sequential and that
+    // the same source is used for the whole piece.
+    if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
+        (ConcatSrcs[i / SrcNumElts] >= 0 &&
+         ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts)))
+      return false;
+    // Remember which source this index came from.
+    ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
+  }
+
+  // The shuffle is concatenating multiple vectors together.
+  // Collect the different operands for that.
+  Register UndefReg;
+  Register Src2 = MI.getOperand(2).getReg();
+  for (auto Src : ConcatSrcs) {
+    if (Src < 0) {
+      if (!UndefReg) {
+        Builder.setInsertPt(*MI.getParent(), MI);
+        UndefReg = Builder.buildUndef(SrcType).getReg(0);
+      }
+      Ops.push_back(UndefReg);
+    } else if (Src == 0)
+      Ops.push_back(Src1);
+    else
+      Ops.push_back(Src2);
+  }
+  return true;
+}
+
+void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI,
+                                               const ArrayRef<Register> Ops) {
+  Register DstReg = MI.getOperand(0).getReg();
+  Builder.setInsertPt(*MI.getParent(), MI);
+  Register NewDstReg = MRI.cloneVirtualRegister(DstReg);
+
+  Builder.buildConcatVectors(NewDstReg, Ops);
+
+  MI.eraseFromParent();
+  replaceRegWith(MRI, DstReg, NewDstReg);
+}
+
 namespace {
 
 /// Select a preference between two uses. CurrentUse is the current preference
 /// while *ForCandidate is attributes of the candidate under consideration.
 PreferredTuple ChoosePreferredUse(PreferredTuple &CurrentUse,
                                   const LLT &TyForCandidate,
                                   unsigned OpcodeForCandidate,
                                   MachineInstr *MIForCandidate) {

llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp

 bool AArch64PreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
                                               MachineInstr &MI,
                                               MachineIRBuilder &B) const {
   CombinerHelper Helper(Observer, B, KB, MDT);
 
   switch (MI.getOpcode()) {
   case TargetOpcode::G_CONCAT_VECTORS:
     return Helper.tryCombineConcatVectors(MI);
+  case TargetOpcode::G_SHUFFLE_VECTOR:
+    return Helper.tryCombineShuffleVector(MI);
   case TargetOpcode::G_LOAD:
   case TargetOpcode::G_SEXTLOAD:
   case TargetOpcode::G_ZEXTLOAD: {
     bool Changed = false;
     Changed |= Helper.tryCombineExtendingLoads(MI);
     Changed |= Helper.tryCombineIndexedLoadStore(MI);
     return Changed;
   }

llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-apple-ios  -run-pass=aarch64-prelegalizer-combiner %s -o - | FileCheck %s
+
+# Check that we canonicalize shuffle_vector(Src1, Src2, mask(0,1,2,3))
+# into concat_vector(Src1, Src2).
+---
+name: shuffle_vector_to_concat_vector_0123
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_0123
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[COPY]](<2 x s32>), [[COPY1]](<2 x s32>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<4 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<4 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(0, 1, 2, 3)
+    RET_ReallyLR implicit %2
+...
+# Check that we canonicalize shuffle_vector(Src1, Src2, mask(2,3,0,1,2,3))
+# into concat_vector(Src2, Src1, Src2).
+---
+name: shuffle_vector_to_concat_vector_230123
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_230123
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s32>) = G_CONCAT_VECTORS [[COPY1]](<2 x s32>), [[COPY]](<2 x s32>), [[COPY1]](<2 x s32>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<6 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,3,0,1,2,3)
+    RET_ReallyLR implicit %2
+...
+# Check that we canonicalize shuffle_vector(Src1, Src2, mask(2,undef,0,1,undef,3))
+# into concat_vector(Src2, Src1, Src2).
+---
+name: shuffle_vector_to_concat_vector_2undef01undef3
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_2undef01undef3
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s32>) = G_CONCAT_VECTORS [[COPY1]](<2 x s32>), [[COPY]](<2 x s32>), [[COPY1]](<2 x s32>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<6 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,-1,0,1,-1,3)
+    RET_ReallyLR implicit %2
+...
+# Check that we don't canonicalize shuffle_vector into concat_vectors
+# when the sources get half mixed.
+---
+name: shuffle_vector_to_concat_vector_mixed_src_200123_neg
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_mixed_src_200123_neg
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 0, 0, 1, 2, 3)
+    ; CHECK: RET_ReallyLR implicit [[SHUF]](<6 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,0,0,1,2,3)
+    RET_ReallyLR implicit %2
+...
+# Check that we produce implicit_def while canonicalizing shuffle_vector(Src1, Src2, mask(2,undef,undef,undef,undef,1))
+# into concat_vector(Src2, undef, Src1).
+---
+name: shuffle_vector_to_concat_vector_2undef1
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_2undef1
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s32>) = G_CONCAT_VECTORS [[COPY1]](<2 x s32>), [[DEF]](<2 x s32>), [[COPY]](<2 x s32>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<6 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,-1,-1,-1,-1,1)
+    RET_ReallyLR implicit %2
+...
+# Check that we don't canonicalize shuffle_vector into concat_vectors
+# if a source is flipped.
+---
+name: shuffle_vector_to_concat_vector_src_flipped_230132_neg
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_src_flipped_230132_neg
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 3, 0, 1, 3, 2)
+    ; CHECK: RET_ReallyLR implicit [[SHUF]](<6 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,3,0,1,3,2)
+    RET_ReallyLR implicit %2
+...
+# Check that we don't canonicalize shuffle_vector into concat_vectors
+# if a source is flipped and the other half is undef.
+---
+name: shuffle_vector_to_concat_vector_src_flipped_23013undef_neg
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $d0, $d1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_src_flipped_23013undef_neg
+    ; CHECK: liveins: $d0, $d1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 3, 0, 1, 3, -1)
+    ; CHECK: RET_ReallyLR implicit [[SHUF]](<6 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,3,0,1,3,-1)
+    RET_ReallyLR implicit %2
+...
+
+# Check that we canonicalize shuffle_vector(Src1, Src2, mask(0,1,2,3,4,5,6,7))
+# into concat_vector(Src1, Src2) with bigger vector type.
+---
+name: shuffle_vector_to_concat_vector_01234567
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_01234567
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<8 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<8 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(0, 1, 2, 3, 4, 5, 6, 7)
+    RET_ReallyLR implicit %2
+...
+# Check that we canonicalize shuffle_vector(Src1, Src2, mask(4,5,6,7,0,1,2,3))
+# into concat_vector(Src2, Src1, Src2) with bigger vector type.
+---
+name: shuffle_vector_to_concat_vector_45670123
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_45670123
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[COPY1]](<4 x s32>), [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<12 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<12 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,6,7,0,1,2,3,4,5,6,7)
+    RET_ReallyLR implicit %2
+...
+# Check that we canonicalize shuffle_vector(Src1, Src2, mask(4,5,undef,undef,0,1,2,3,undef,undef,6,7))
+# into concat_vector(Src2, Src1, Src2) with bigger vector type.
+---
+name: shuffle_vector_to_concat_vector_45undefundef0123undefundef67
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_45undefundef0123undefundef67
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[COPY1]](<4 x s32>), [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<12 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<12 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,-1,-1,0,1,2,3,-1,-1,6,7)
+    RET_ReallyLR implicit %2
+...
+# Check that we don't canonicalize shuffle_vector into concat_vectors
+# when the sources get half mixed with bigger vector type.
+---
+name: shuffle_vector_to_concat_vector_mixed_src_456000123_neg
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_mixed_src_456000123_neg
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<8 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 6, 0, 0, 1, 2, 3)
+    ; CHECK: RET_ReallyLR implicit [[SHUF]](<8 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<8 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,6,0,0,1,2,3)
+    RET_ReallyLR implicit %2
+...
+# Check that we produce implicit_def while canonicalizing shuffle_vector(Src1, Src2, mask(2,undef,undef,undef,undef,1))
+# into concat_vector(Src2, undef, Src1) with bigger vector type.
+---
+name: shuffle_vector_to_concat_vector_45undefundefundefundefundefundefundefundef23
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_45undefundefundefundefundefundefundefundef23
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[COPY1]](<4 x s32>), [[DEF]](<4 x s32>), [[COPY]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<12 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<12 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,-1,-1,-1,-1,-1,-1,-1,-1,2,3)
+    RET_ReallyLR implicit %2
+...
+# Check that we don't canonicalize shuffle_vector into concat_vectors
+# when we grab less than the full vectors.
+---
+name: shuffle_vector_to_concat_vector_4501_neg
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_4501_neg
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<4 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 0, 1)
+    ; CHECK: RET_ReallyLR implicit [[SHUF]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<4 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,0,1)
+    RET_ReallyLR implicit %2
+...
+# Check that we don't canonicalize shuffle_vector into concat_vectors
+# if a source is flipped and the other half is undef with bigger vector.
+---
+name: shuffle_vector_to_concat_vector_src_flipped_4567012367undefundef_neg
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_src_flipped_4567012367undefundef_neg
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<12 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 6, 7, 0, 1, 2, 3, 6, 7, -1, -1)
+    ; CHECK: RET_ReallyLR implicit [[SHUF]](<12 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<12 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,6,7,0,1,2,3,6,7,-1,-1)
+    RET_ReallyLR implicit %2
+...
+# Check that we canonicalize shuffle_vector(Src1, Src2, mask(4,5,undef,undef,0,1,2,3,undef,undef,6,7))
+# into concat_vector(Src2, Src1, Src2) with vector of pointers.
+---
+name: shuffle_vector_to_concat_vector_45undefundef0123undefundef67_ptr
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0_q1, $q2_q3
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_45undefundef0123undefundef67_ptr
+    ; CHECK: liveins: $q0_q1, $q2_q3
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x p0>) = COPY $q0_q1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x p0>) = COPY $q2_q3
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x p0>) = G_CONCAT_VECTORS [[COPY1]](<4 x p0>), [[COPY]](<4 x p0>), [[COPY1]](<4 x p0>)
+    ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<12 x p0>)
+    %0:_(<4 x p0>) = COPY $q0_q1
+    %1:_(<4 x p0>) = COPY $q2_q3
+    %2:_(<12 x p0>) = G_SHUFFLE_VECTOR %0(<4 x p0>), %1(<4 x p0>), shufflemask(4,5,-1,-1,0,1,2,3,-1,-1,6,7)
+    RET_ReallyLR implicit %2
+...
+# Check that we don't canonicalize shuffle_vector into concat_vectors
+# when the sources get half mixed with vector of pointers.
+---
+name: shuffle_vector_to_concat_vector_mixed_src_456000123_neg_ptr
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0_q1, $q2_q3
+
+    ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_mixed_src_456000123_neg_ptr
+    ; CHECK: liveins: $q0_q1, $q2_q3
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x p0>) = COPY $q0_q1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x p0>) = COPY $q2_q3
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<8 x p0>) = G_SHUFFLE_VECTOR [[COPY]](<4 x p0>), [[COPY1]], shufflemask(4, 5, 6, 0, 0, 1, 2, 3)
+    ; CHECK: RET_ReallyLR implicit [[SHUF]](<8 x p0>)
+    %0:_(<4 x p0>) = COPY $q0_q1
+    %1:_(<4 x p0>) = COPY $q2_q3
+    %2:_(<8 x p0>) = G_SHUFFLE_VECTOR %0(<4 x p0>), %1(<4 x p0>), shufflemask(4,5,6,0,0,1,2,3)
+    RET_ReallyLR implicit %2
+...
+
+# Check that shuffle_vector gets combined into concat_vectors then the
+# concat_vectors gets combined into build_vector.
+---
+name: shuffle_vector_to_build_vector_ptr
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $x0, $x1, $x2, $x3
+
+    ; CHECK-LABEL: name: shuffle_vector_to_build_vector_ptr
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(p0) = COPY $x3
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x p0>) = G_BUILD_VECTOR [[COPY]](p0), [[COPY1]](p0), [[COPY2]](p0), [[COPY3]](p0)
+    ; CHECK: RET_ReallyLR implicit [[BUILD_VECTOR]](<4 x p0>)
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(p0) = COPY $x2
+    %3:_(p0) = COPY $x3
+    %4:_(<2 x p0>) = G_BUILD_VECTOR %0(p0), %1
+    %5:_(<2 x p0>) = G_BUILD_VECTOR %2(p0), %3
+    %6:_(<4 x p0>) = G_SHUFFLE_VECTOR %4(<2 x p0>), %5(<2 x p0>), shufflemask(0,1,2,3)
+    RET_ReallyLR implicit %6
+...