diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -112,6 +112,23 @@ void applyCombineConcatVectors(MachineInstr &MI, bool IsUndef, const ArrayRef Ops); + /// Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS. + /// Returns true if MI changed. + /// + /// \pre MI.getOpcode() == G_SHUFFLE_VECTOR. + bool tryCombineShuffleVector(MachineInstr &MI); + /// Check if the G_SHUFFLE_VECTOR \p MI can be replaced by a + /// concat_vectors. + /// \p Ops will contain the operands needed to produce the flattened + /// concat_vectors. + /// + /// \pre MI.getOpcode() == G_SHUFFLE_VECTOR. + bool matchCombineShuffleVector(MachineInstr &MI, + SmallVectorImpl &Ops); + /// Replace \p MI with a concat_vectors with \p Ops. + void applyCombineShuffleVector(MachineInstr &MI, + const ArrayRef Ops); + /// Optimize memcpy intrinsics et al, e.g. constant len calls. /// /p MaxLen if non-zero specifies the max length of a mem libcall to inline. /// diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -173,6 +173,93 @@ replaceRegWith(MRI, DstReg, NewDstReg); } +bool CombinerHelper::tryCombineShuffleVector(MachineInstr &MI) { + SmallVector Ops; + if (matchCombineShuffleVector(MI, Ops)) { + applyCombineShuffleVector(MI, Ops); + return true; + } + return false; +} + +bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI, + SmallVectorImpl &Ops) { + assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR && + "Invalid instruction kind"); + LLT DstType = MRI.getType(MI.getOperand(0).getReg()); + Register Src1 = MI.getOperand(1).getReg(); + LLT SrcType = MRI.getType(Src1); + unsigned DstNumElts = DstType.getNumElements(); + unsigned SrcNumElts = SrcType.getNumElements(); + + // If the resulting vector is smaller than the size of the source + // vectors being concatenated, we won't be able to replace the + // shuffle vector into a concat_vectors. + // + // Note: We may still be able to produce a concat_vectors fed by + // extract_vector_elt and so on. It is less clear that would + // be better though, so don't bother for now. + if (DstNumElts < 2 * SrcNumElts) + return false; + + // Check that the shuffle mask can be broken evenly between the + // different sources. + if (DstNumElts % SrcNumElts != 0) + return false; + + // Mask length is a multiple of the source vector length. + // Check if the shuffle is some kind of concatenation of the input + // vectors. + unsigned NumConcat = DstNumElts / SrcNumElts; + SmallVector ConcatSrcs(NumConcat, -1); + SmallVector Mask; + ShuffleVectorInst::getShuffleMask(MI.getOperand(3).getShuffleMask(), Mask); + for (unsigned i = 0; i != DstNumElts; ++i) { + int Idx = Mask[i]; + // Undef value. + if (Idx < 0) + continue; + // Ensure the indices in each SrcType sized piece are sequential and that + // the same source is used for the whole piece. + if ((Idx % SrcNumElts != (i % SrcNumElts)) || + (ConcatSrcs[i / SrcNumElts] >= 0 && + ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) + return false; + // Remember which source this index came from. + ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts; + } + + // The shuffle is concatenating multiple vectors together. + // Collect the different operands for that. + Register UndefReg; + Register Src2 = MI.getOperand(2).getReg(); + for (auto Src : ConcatSrcs) { + if (Src < 0) { + if (!UndefReg) { + Builder.setInsertPt(*MI.getParent(), MI); + UndefReg = Builder.buildUndef(SrcType).getReg(0); + } + Ops.push_back(UndefReg); + } else if (Src == 0) + Ops.push_back(Src1); + else + Ops.push_back(Src2); + } + return true; +} + +void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI, + const ArrayRef Ops) { + Register DstReg = MI.getOperand(0).getReg(); + Builder.setInsertPt(*MI.getParent(), MI); + Register NewDstReg = MRI.cloneVirtualRegister(DstReg); + + Builder.buildConcatVectors(NewDstReg, Ops); + + MI.eraseFromParent(); + replaceRegWith(MRI, DstReg, NewDstReg); +} + namespace { /// Select a preference between two uses. CurrentUse is the current preference diff --git a/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp --- a/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp +++ b/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp @@ -64,6 +64,8 @@ switch (MI.getOpcode()) { case TargetOpcode::G_CONCAT_VECTORS: return Helper.tryCombineConcatVectors(MI); + case TargetOpcode::G_SHUFFLE_VECTOR: + return Helper.tryCombineShuffleVector(MI); case TargetOpcode::G_LOAD: case TargetOpcode::G_SEXTLOAD: case TargetOpcode::G_ZEXTLOAD: { diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-shuffle-vector.mir @@ -0,0 +1,353 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple aarch64-apple-ios -run-pass=aarch64-prelegalizer-combiner %s -o - | FileCheck %s + +# Check that we canonicalize shuffle_vector(Src1, Src2, mask(0,1,2,3)) +# into concat_vector(Src1, Src2). +--- +name: shuffle_vector_to_concat_vector_0123 +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $d1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_0123 + ; CHECK: liveins: $d0, $d1 + ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[COPY]](<2 x s32>), [[COPY1]](<2 x s32>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<4 x s32>) + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = COPY $d1 + %2:_(<4 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(0, 1, 2, 3) + RET_ReallyLR implicit %2 +... +# Check that we canonicalize shuffle_vector(Src1, Src2, mask(2,3,0,1,2,3)) +# into concat_vector(Src2, Src1, Src2). +--- +name: shuffle_vector_to_concat_vector_230123 +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $d1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_230123 + ; CHECK: liveins: $d0, $d1 + ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s32>) = G_CONCAT_VECTORS [[COPY1]](<2 x s32>), [[COPY]](<2 x s32>), [[COPY1]](<2 x s32>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<6 x s32>) + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = COPY $d1 + %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,3,0,1,2,3) + RET_ReallyLR implicit %2 +... +# Check that we canonicalize shuffle_vector(Src1, Src2, mask(2,undef,0,1,undef,3)) +# into concat_vector(Src2, Src1, Src2). +--- +name: shuffle_vector_to_concat_vector_2undef01undef3 +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $d1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_2undef01undef3 + ; CHECK: liveins: $d0, $d1 + ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s32>) = G_CONCAT_VECTORS [[COPY1]](<2 x s32>), [[COPY]](<2 x s32>), [[COPY1]](<2 x s32>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<6 x s32>) + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = COPY $d1 + %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,-1,0,1,-1,3) + RET_ReallyLR implicit %2 +... +# Check that we don't canonicalize shuffle_vector into concat_vectors +# when the sources get half mixed. +--- +name: shuffle_vector_to_concat_vector_mixed_src_200123_neg +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $d1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_mixed_src_200123_neg + ; CHECK: liveins: $d0, $d1 + ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 + ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 0, 0, 1, 2, 3) + ; CHECK: RET_ReallyLR implicit [[SHUF]](<6 x s32>) + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = COPY $d1 + %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,0,0,1,2,3) + RET_ReallyLR implicit %2 +... +# Check that we produce implicit_def while canonicalizing shuffle_vector(Src1, Src2, mask(2,undef,undef,undef,undef,1)) +# into concat_vector(Src2, undef, Src1). +--- +name: shuffle_vector_to_concat_vector_2undef1 +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $d1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_2undef1 + ; CHECK: liveins: $d0, $d1 + ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 + ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s32>) = G_CONCAT_VECTORS [[COPY1]](<2 x s32>), [[DEF]](<2 x s32>), [[COPY]](<2 x s32>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<6 x s32>) + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = COPY $d1 + %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,-1,-1,-1,-1,1) + RET_ReallyLR implicit %2 +... +# Check that we don't canonicalize shuffle_vector into concat_vectors +# if a source is flipped. +--- +name: shuffle_vector_to_concat_vector_src_flipped_230132_neg +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $d1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_src_flipped_230132_neg + ; CHECK: liveins: $d0, $d1 + ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 + ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 3, 0, 1, 3, 2) + ; CHECK: RET_ReallyLR implicit [[SHUF]](<6 x s32>) + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = COPY $d1 + %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,3,0,1,3,2) + RET_ReallyLR implicit %2 +... +# Check that we don't canonicalize shuffle_vector into concat_vectors +# if a source is flipped and the other half is undef. +--- +name: shuffle_vector_to_concat_vector_src_flipped_23013undef_neg +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $d1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_src_flipped_23013undef_neg + ; CHECK: liveins: $d0, $d1 + ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 + ; CHECK: [[SHUF:%[0-9]+]]:_(<6 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<2 x s32>), [[COPY1]], shufflemask(2, 3, 0, 1, 3, -1) + ; CHECK: RET_ReallyLR implicit [[SHUF]](<6 x s32>) + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = COPY $d1 + %2:_(<6 x s32>) = G_SHUFFLE_VECTOR %0(<2 x s32>), %1(<2 x s32>), shufflemask(2,3,0,1,3,-1) + RET_ReallyLR implicit %2 +... + +# Check that we canonicalize shuffle_vector(Src1, Src2, mask(0,1,2,3,4,5,6,7)) +# into concat_vector(Src1, Src2) with bigger vector type. +--- +name: shuffle_vector_to_concat_vector_01234567 +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_01234567 + ; CHECK: liveins: $q0, $q1 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<8 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(<8 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(0, 1, 2, 3, 4, 5, 6, 7) + RET_ReallyLR implicit %2 +... +# Check that we canonicalize shuffle_vector(Src1, Src2, mask(4,5,6,7,0,1,2,3)) +# into concat_vector(Src2, Src1, Src2) with bigger vector type. +--- +name: shuffle_vector_to_concat_vector_45670123 +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_45670123 + ; CHECK: liveins: $q0, $q1 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[COPY1]](<4 x s32>), [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<12 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(<12 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,6,7,0,1,2,3,4,5,6,7) + RET_ReallyLR implicit %2 +... +# Check that we canonicalize shuffle_vector(Src1, Src2, mask(4,5,undef,undef,0,1,2,3,undef,undef,6,7)) +# into concat_vector(Src2, Src1, Src2) with bigger vector type. +--- +name: shuffle_vector_to_concat_vector_45undefundef0123undefundef67 +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_45undefundef0123undefundef67 + ; CHECK: liveins: $q0, $q1 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[COPY1]](<4 x s32>), [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<12 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(<12 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,-1,-1,0,1,2,3,-1,-1,6,7) + RET_ReallyLR implicit %2 +... +# Check that we don't canonicalize shuffle_vector into concat_vectors +# when the sources get half mixed with bigger vector type. +--- +name: shuffle_vector_to_concat_vector_mixed_src_456000123_neg +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_mixed_src_456000123_neg + ; CHECK: liveins: $q0, $q1 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK: [[SHUF:%[0-9]+]]:_(<8 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 6, 0, 0, 1, 2, 3) + ; CHECK: RET_ReallyLR implicit [[SHUF]](<8 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(<8 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,6,0,0,1,2,3) + RET_ReallyLR implicit %2 +... +# Check that we produce implicit_def while canonicalizing shuffle_vector(Src1, Src2, mask(2,undef,undef,undef,undef,1)) +# into concat_vector(Src2, undef, Src1) with bigger vector type. +--- +name: shuffle_vector_to_concat_vector_45undefundefundefundefundefundefundefundef23 +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_45undefundefundefundefundefundefundefundef23 + ; CHECK: liveins: $q0, $q1 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[COPY1]](<4 x s32>), [[DEF]](<4 x s32>), [[COPY]](<4 x s32>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<12 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(<12 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,-1,-1,-1,-1,-1,-1,-1,-1,2,3) + RET_ReallyLR implicit %2 +... +# Check that we don't canonicalize shuffle_vector into concat_vectors +# when we grab less than the full vectors. +--- +name: shuffle_vector_to_concat_vector_4501_neg +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_4501_neg + ; CHECK: liveins: $q0, $q1 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK: [[SHUF:%[0-9]+]]:_(<4 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 0, 1) + ; CHECK: RET_ReallyLR implicit [[SHUF]](<4 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(<4 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,0,1) + RET_ReallyLR implicit %2 +... +# Check that we don't canonicalize shuffle_vector into concat_vectors +# if a source is flipped and the other half is undef with bigger vector. +--- +name: shuffle_vector_to_concat_vector_src_flipped_4567012367undefundef_neg +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_src_flipped_4567012367undefundef_neg + ; CHECK: liveins: $q0, $q1 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK: [[SHUF:%[0-9]+]]:_(<12 x s32>) = G_SHUFFLE_VECTOR [[COPY]](<4 x s32>), [[COPY1]], shufflemask(4, 5, 6, 7, 0, 1, 2, 3, 6, 7, -1, -1) + ; CHECK: RET_ReallyLR implicit [[SHUF]](<12 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(<12 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %1(<4 x s32>), shufflemask(4,5,6,7,0,1,2,3,6,7,-1,-1) + RET_ReallyLR implicit %2 +... +# Check that we canonicalize shuffle_vector(Src1, Src2, mask(4,5,undef,undef,0,1,2,3,undef,undef,6,7)) +# into concat_vector(Src2, Src1, Src2) with vector of pointers. +--- +name: shuffle_vector_to_concat_vector_45undefundef0123undefundef67_ptr +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0_q1, $q2_q3 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_45undefundef0123undefundef67_ptr + ; CHECK: liveins: $q0_q1, $q2_q3 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x p0>) = COPY $q0_q1 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x p0>) = COPY $q2_q3 + ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x p0>) = G_CONCAT_VECTORS [[COPY1]](<4 x p0>), [[COPY]](<4 x p0>), [[COPY1]](<4 x p0>) + ; CHECK: RET_ReallyLR implicit [[CONCAT_VECTORS]](<12 x p0>) + %0:_(<4 x p0>) = COPY $q0_q1 + %1:_(<4 x p0>) = COPY $q2_q3 + %2:_(<12 x p0>) = G_SHUFFLE_VECTOR %0(<4 x p0>), %1(<4 x p0>), shufflemask(4,5,-1,-1,0,1,2,3,-1,-1,6,7) + RET_ReallyLR implicit %2 +... +# Check that we don't canonicalize shuffle_vector into concat_vectors +# when the sources get half mixed with vector of pointers. +--- +name: shuffle_vector_to_concat_vector_mixed_src_456000123_neg_ptr +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0_q1, $q2_q3 + + ; CHECK-LABEL: name: shuffle_vector_to_concat_vector_mixed_src_456000123_neg_ptr + ; CHECK: liveins: $q0_q1, $q2_q3 + ; CHECK: [[COPY:%[0-9]+]]:_(<4 x p0>) = COPY $q0_q1 + ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x p0>) = COPY $q2_q3 + ; CHECK: [[SHUF:%[0-9]+]]:_(<8 x p0>) = G_SHUFFLE_VECTOR [[COPY]](<4 x p0>), [[COPY1]], shufflemask(4, 5, 6, 0, 0, 1, 2, 3) + ; CHECK: RET_ReallyLR implicit [[SHUF]](<8 x p0>) + %0:_(<4 x p0>) = COPY $q0_q1 + %1:_(<4 x p0>) = COPY $q2_q3 + %2:_(<8 x p0>) = G_SHUFFLE_VECTOR %0(<4 x p0>), %1(<4 x p0>), shufflemask(4,5,6,0,0,1,2,3) + RET_ReallyLR implicit %2 +... + +# Check that shuffle_vector gets combined into concat_vectors then the +# concat_vectors gets combined into build_vector. +--- +name: shuffle_vector_to_build_vector_ptr +tracksRegLiveness: true +body: | + bb.1: + liveins: $x0, $x1, $x2, $x3 + + ; CHECK-LABEL: name: shuffle_vector_to_build_vector_ptr + ; CHECK: liveins: $x0, $x1, $x2, $x3 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1 + ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $x2 + ; CHECK: [[COPY3:%[0-9]+]]:_(p0) = COPY $x3 + ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x p0>) = G_BUILD_VECTOR [[COPY]](p0), [[COPY1]](p0), [[COPY2]](p0), [[COPY3]](p0) + ; CHECK: RET_ReallyLR implicit [[BUILD_VECTOR]](<4 x p0>) + %0:_(p0) = COPY $x0 + %1:_(p0) = COPY $x1 + %2:_(p0) = COPY $x2 + %3:_(p0) = COPY $x3 + %4:_(<2 x p0>) = G_BUILD_VECTOR %0(p0), %1 + %5:_(<2 x p0>) = G_BUILD_VECTOR %2(p0), %3 + %6:_(<4 x p0>) = G_SHUFFLE_VECTOR %4(<2 x p0>), %5(<2 x p0>), shufflemask(0,1,2,3) + RET_ReallyLR implicit %6 +...