[SVE][CodeGen] Add support for scalable vectors in AArch64TargetLowering::ReconstructShuffle

[david-arm]

Summary

At the moment if AArch64TargetLowering::ReconstructShuffle encounters
a scalable vector it just bails out of the optimisation entirely.
However, there are cases when we can make the transformation succeed
and produce far better code than if we just immediately bail out.

You can see the codegen improvement in the following tests:

CodeGen/AArch64/sve-fixed-length-reshuffle.ll

I have tried to highlight in sve-fixed-length-reshuffle.ll those tests
that exercise the AArch64TargetLowering::ReconstructShuffle code path
when the transformation succeeds or fails.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

9048

When protecting code can you check the positive rather than the negative. So if (SrcVTSize.isFixed() && ...). Looking at TypeSize suggests there's no such function, but I think that's an omission.

llvm/include/llvm/Support/TypeSize.h

return static_cast<LeafTy>(
         LinearPolySize(MinVal, Scalable ? ScalableDim : FixedDim));
   }
   static LeafTy getNull() { return get(0, false); }
 
   /// Returns the minimum value this size can represent.
   ScalarTy getKnownMinValue() const { return this->Value; }
   /// Returns whether the size is scaled by a runtime quantity (vscale).
   bool isScalable() const { return this->UnivariateDim == ScalableDim; }
+  /// Returns true if the size is known at compile time.
+  bool isFixed() const { return this->UnivariateDim == FixedDim; }
   /// A return value of true indicates we know at compile time that the number
   /// of elements (vscale * Min) is definitely even. However, returning false
   /// does not guarantee that the total number of elements is odd.
   bool isKnownEven() const { return (getKnownMinValue() & 0x1) == 0; }
   /// This function tells the caller whether the element count is known at
   /// compile time to be a multiple of the scalar value RHS.
   bool isKnownMultipleOf(ScalarTy RHS) const {
     return getKnownMinValue() % RHS == 0;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
   // First gather all vectors used as an immediate source for this BUILD_VECTOR
   // node.
   SmallVector<ShuffleSourceInfo, 2> Sources;
   for (unsigned i = 0; i < NumElts; ++i) {
     SDValue V = Op.getOperand(i);
     if (V.isUndef())
       continue;
     else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
-             !isa<ConstantSDNode>(V.getOperand(1)) ||
-             V.getOperand(0).getValueType().isScalableVector()) {
+             !isa<ConstantSDNode>(V.getOperand(1))) {
       LLVM_DEBUG(
           dbgs() << "Reshuffle failed: "
                     "a shuffle can only come from building a vector from "
-                    "various elements of other fixed-width vectors, provided "
-                    "their indices are constant\n");
+                    "various elements of other vectors, provided their"
+                    "indices are constant\n");
       return SDValue();
     }
 
     // Add this element source to the list if it's not already there.
     SDValue SourceVec = V.getOperand(0);
     auto Source = find(Sources, SourceVec);
     if (Source == Sources.end())
       Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
if (SrcVTSize == TypeSize::Fixed(VTSize))
       continue;
 
     // This stage of the search produces a source with the same element type as
     // the original, but with a total width matching the BUILD_VECTOR output.
     EVT EltVT = SrcVT.getVectorElementType();
     unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
     EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
 
-    if (SrcVTSize.getFixedValue() < VTSize) {
+    if (SrcVTSize.isFixed() && SrcVTSize.getFixedValue() < VTSize) {
       assert(2 * SrcVTSize == VTSize);
       // We can pad out the smaller vector for free, so if it's part of a
       // shuffle...
       Src.ShuffleVec =
           DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
                       DAG.getUNDEF(Src.ShuffleVec.getValueType()));
       continue;
     }
 
-    if (SrcVTSize.getFixedValue() != 2 * VTSize) {
+    if ((SrcVTSize.isFixed() && SrcVTSize.getFixedValue() != 2 * VTSize) ||
+        (SrcVTSize.isScalable() && SrcVTSize.getKnownMinValue() != VTSize)) {
       LLVM_DEBUG(
           dbgs() << "Reshuffle failed: result vector too small to extract\n");
       return SDValue();
     }
 
     if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
       LLVM_DEBUG(
           dbgs() << "Reshuffle failed: span too large for a VEXT to cope\n");
if (SrcEltTy == SmallestEltTy)
       continue;
     assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
     Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec);
     Src.WindowScale =
         SrcEltTy.getFixedSizeInBits() / SmallestEltTy.getFixedSizeInBits();
     Src.WindowBase *= Src.WindowScale;
   }
 
-  // Final check before we try to actually produce a shuffle.
   LLVM_DEBUG(for (auto Src
-                  : Sources)
-                 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
+                  : Sources) assert(Src.ShuffleVec.getValueType()
+                                        .getSizeInBits()
+                                        .getKnownMinValue() ==
+                                    ShuffleVT.getFixedSizeInBits()););
 
   // The stars all align, our next step is to produce the mask for the shuffle.
   SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
   int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
   for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
     SDValue Entry = Op.getOperand(i);
     if (Entry.isUndef())
       continue;

llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll

; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i1> %a, i32 0
   %el1 = extractelement <vscale x 4 x i1> %a, i32 1
   %v0 = insertelement <2 x i1> undef, i1 %el0, i32 0
   %v1 = insertelement <2 x i1> %v0, i1 %el1, i32 1
   ret <2 x i1> %v1
 }
 
 
+; ReconstructShuffle fails
 define <4 x i1> @reshuffle_v4i1_nxv4i1(<vscale x 4 x i1> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i1_nxv4i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z1.s, p0/z, #1 // =0x1
 ; CHECK-NEXT:    mov w8, v1.s[1]
 ; CHECK-NEXT:    mov w9, v1.s[2]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    mov v0.h[1], w8
; CHECK-NEXT:    ret
   %v0 = insertelement <4 x i1> undef, i1 %el0, i32 0
   %v1 = insertelement <4 x i1> %v0, i1 %el1, i32 1
   %v2 = insertelement <4 x i1> %v1, i1 %el2, i32 2
   %v3 = insertelement <4 x i1> %v2, i1 %el3, i32 3
   ret <4 x i1> %v3
 }
 
 
+; ReconstructShuffle fails
 define <4 x i1> @reshuffle_v4i1_nxv8i1(<vscale x 8 x i1> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i1_nxv8i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z1.h, p0/z, #1 // =0x1
 ; CHECK-NEXT:    umov w8, v1.h[1]
 ; CHECK-NEXT:    umov w9, v1.h[2]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    mov v0.h[1], w8
; CHECK-NEXT:    ret
   %v0 = insertelement <4 x i1> undef, i1 %el0, i32 0
   %v1 = insertelement <4 x i1> %v0, i1 %el1, i32 1
   %v2 = insertelement <4 x i1> %v1, i1 %el2, i32 2
   %v3 = insertelement <4 x i1> %v2, i1 %el3, i32 3
   ret <4 x i1> %v3
 }
 
 
+; ReconstructShuffle fails
 define <8 x i1> @reshuffle_v8i1_nxv8i1(<vscale x 8 x i1> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i1_nxv8i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z1.h, p0/z, #1 // =0x1
 ; CHECK-NEXT:    umov w8, v1.h[1]
 ; CHECK-NEXT:    umov w9, v1.h[2]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    mov v0.b[1], w8
; CHECK-NEXT:    ret
   %v4 = insertelement <8 x i1> %v3, i1 %el4, i32 4
   %v5 = insertelement <8 x i1> %v4, i1 %el5, i32 5
   %v6 = insertelement <8 x i1> %v5, i1 %el6, i32 6
   %v7 = insertelement <8 x i1> %v6, i1 %el7, i32 7
   ret <8 x i1> %v7
 }
 
 
+; ReconstructShuffle fails
 define <16 x i1> @reshuffle_v16i1_nxv16i1(<vscale x 16 x i1> %a) #0 {
 ; CHECK-LABEL: reshuffle_v16i1_nxv16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z1.b, p0/z, #1 // =0x1
 ; CHECK-NEXT:    umov w8, v1.b[1]
 ; CHECK-NEXT:    umov w9, v1.b[2]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    mov v0.b[1], w8
; CHECK-NEXT:    ret
   %v2 = insertelement <4 x float> %v1, float %el2, i32 2
   %v3 = insertelement <4 x float> %v2, float %el3, i32 3
   ret <4 x float> %v3
 }
 
 
 ; == Reversed first N elements ==
 
+; ReconstructShuffle - fails
 define <4 x i16> @reshuffle_v4i16_nxv4i16_reverse(<vscale x 4 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i16_nxv4i16_reverse:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, v0.s[3]
 ; CHECK-NEXT:    mov w9, v0.s[2]
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    mov v1.h[1], w9
; CHECK-NEXT:    ret
   %v0 = insertelement <4 x i16> undef, i16 %el3, i32 0
   %v1 = insertelement <4 x i16> %v0, i16 %el2, i32 1
   %v2 = insertelement <4 x i16> %v1, i16 %el1, i32 2
   %v3 = insertelement <4 x i16> %v2, i16 %el0, i32 3
   ret <4 x i16> %v3
 }
 
 
+; ReconstructShuffle - fails
 define <8 x i16> @reshuffle_v8i16_nxv8i16_reverse(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_reverse:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[7]
 ; CHECK-NEXT:    umov w9, v0.h[6]
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    umov w8, v0.h[5]
 ; CHECK-NEXT:    mov v1.h[1], w9

 ; CHECK-NEXT:    dup v0.2s, w8
 ; CHECK-NEXT:    ret
   %el1 = extractelement <vscale x 4 x i32> %a, i32 1
   %v0 = insertelement <2 x i32> undef, i32 %el1, i32 1
   ret <2 x i32> %v0
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_nxv4i32_undef(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_nxv4i32_undef:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    mov v0.s[1], w8
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i32> %a, i32 0
   %el1 = extractelement <vscale x 4 x i32> %a, i32 1
   %v0 = insertelement <4 x i32> undef, i32 %el0, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el1, i32 1
   ret <4 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_nxv4i32_undef2(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_nxv4i32_undef2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[2]
-; CHECK-NEXT:    mov w9, v0.s[3]
-; CHECK-NEXT:    mov v0.s[2], w8
-; CHECK-NEXT:    mov v0.s[3], w9
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el2 = extractelement <vscale x 4 x i32> %a, i32 2
   %el3 = extractelement <vscale x 4 x i32> %a, i32 3
   %v0 = insertelement <4 x i32> undef, i32 %el2, i32 2
   %v1 = insertelement <4 x i32> %v0, i32 %el3, i32 3
   ret <4 x i32> %v1
 }
 

 ; CHECK-NEXT:    dup v0.2s, w8
 ; CHECK-NEXT:    ret
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %v0 = insertelement <2 x i16> undef, i16 %el1, i32 1
   ret <2 x i16> %v0
 }
 
 
+; ReconstructShuffle - fails
 define <4 x i16> @reshuffle_v4i16_nxv8i16_undef(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i16_nxv8i16_undef:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[1]
 ; CHECK-NEXT:    mov v0.h[1], w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %v0 = insertelement <4 x i16> undef, i16 %el0, i32 0
   %v1 = insertelement <4 x i16> %v0, i16 %el1, i32 1
   ret <4 x i16> %v1
 }
 
+; ReconstructShuffle - fails
 define <4 x i16> @reshuffle_v4i16_nxv8i16_undef2(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i16_nxv8i16_undef2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[2]
 ; CHECK-NEXT:    umov w9, v0.h[3]
 ; CHECK-NEXT:    mov v0.h[2], w8
 ; CHECK-NEXT:    mov v0.h[3], w9
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %el2 = extractelement <vscale x 8 x i16> %a, i32 2
   %el3 = extractelement <vscale x 8 x i16> %a, i32 3
   %v0 = insertelement <4 x i16> undef, i16 %el2, i32 2
   %v1 = insertelement <4 x i16> %v0, i16 %el3, i32 3
   ret <4 x i16> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_nxv8i16_undef(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_undef:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[3]
-; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    mov v0.h[2], w9
-; CHECK-NEXT:    mov v0.h[3], w10
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %el2 = extractelement <vscale x 8 x i16> %a, i32 2
   %el3 = extractelement <vscale x 8 x i16> %a, i32 3
   %v0 = insertelement <8 x i16> undef, i16 %el0, i32 0
   %v1 = insertelement <8 x i16> %v0, i16 %el1, i32 1
   %v2 = insertelement <8 x i16> %v1, i16 %el2, i32 2
   %v3 = insertelement <8 x i16> %v2, i16 %el3, i32 3
   ret <8 x i16> %v3
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_nxv8i16_undef2(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_undef2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umov w8, v0.h[4]
-; CHECK-NEXT:    umov w9, v0.h[5]
-; CHECK-NEXT:    mov v1.h[4], w8
-; CHECK-NEXT:    umov w8, v0.h[6]
-; CHECK-NEXT:    mov v1.h[5], w9
-; CHECK-NEXT:    umov w9, v0.h[7]
-; CHECK-NEXT:    mov v1.h[6], w8
-; CHECK-NEXT:    mov v1.h[7], w9
-; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el4 = extractelement <vscale x 8 x i16> %a, i32 4
   %el5 = extractelement <vscale x 8 x i16> %a, i32 5
   %el6 = extractelement <vscale x 8 x i16> %a, i32 6
   %el7 = extractelement <vscale x 8 x i16> %a, i32 7
   %v0 = insertelement <8 x i16> undef, i16 %el4, i32 4
   %v1 = insertelement <8 x i16> %v0, i16 %el5, i32 5
   %v2 = insertelement <8 x i16> %v1, i16 %el6, i32 6
   %v3 = insertelement <8 x i16> %v2, i16 %el7, i32 7
   ret <8 x i16> %v3
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_nxv4i32_reverse_undef(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_nxv4i32_reverse_undef:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i32> %a, i32 0
   %el1 = extractelement <vscale x 4 x i32> %a, i32 1
   %v0 = insertelement <4 x i32> undef, i32 %el1, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el0, i32 1
   ret <4 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_nxv4i32_reverse_undef2(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_nxv4i32_reverse_undef2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[3]
-; CHECK-NEXT:    mov w9, v0.s[2]
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
 ; CHECK-NEXT:    ret
   %el2 = extractelement <vscale x 4 x i32> %a, i32 2
   %el3 = extractelement <vscale x 4 x i32> %a, i32 3
   %v0 = insertelement <4 x i32> undef, i32 %el3, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el2, i32 1
   ret <4 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_nxv8i16_reverse_undef(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_reverse_undef:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umov w8, v0.h[3]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    mov v1.h[1], w9
-; CHECK-NEXT:    mov v1.h[2], w8
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    mov v1.h[3], w8
-; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    rev64 v0.8h, v0.8h
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %el2 = extractelement <vscale x 8 x i16> %a, i32 2
   %el3 = extractelement <vscale x 8 x i16> %a, i32 3
   %v0 = insertelement <8 x i16> undef, i16 %el3, i32 0
   %v1 = insertelement <8 x i16> %v0, i16 %el2, i32 1
   %v2 = insertelement <8 x i16> %v1, i16 %el1, i32 2
   %v3 = insertelement <8 x i16> %v2, i16 %el0, i32 3
   ret <8 x i16> %v3
 }
 
 
+; ReconstructShuffle - fails
 define <8 x i16> @reshuffle_v8i16_nxv8i16_reverse_undef2(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_reverse_undef2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[7]
 ; CHECK-NEXT:    umov w9, v0.h[6]
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    umov w8, v0.h[5]
 ; CHECK-NEXT:    mov v1.h[1], w9
; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 2 x i32> %a, i32 0
   %el1 = extractelement <vscale x 2 x i32> %b, i32 0
   %v0 = insertelement <2 x i32> undef, i32 %el0, i32 0
   %v1 = insertelement <2 x i32> %v0, i32 %el1, i32 1
   ret <2 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_2x_nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_2x_nxv4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    mov w9, v1.s[1]
-; CHECK-NEXT:    mov v0.s[1], w8
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov v0.s[2], w8
-; CHECK-NEXT:    mov v0.s[3], w9
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i32> %a, i32 0
   %el1 = extractelement <vscale x 4 x i32> %a, i32 1
   %el2 = extractelement <vscale x 4 x i32> %b, i32 0
   %el3 = extractelement <vscale x 4 x i32> %b, i32 1
   %v0 = insertelement <4 x i32> undef, i32 %el0, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el1, i32 1
   %v2 = insertelement <4 x i32> %v1, i32 %el2, i32 2
   %v3 = insertelement <4 x i32> %v2, i32 %el3, i32 3
   ret <4 x i32> %v3
 }
 
 
+; ReconstructShuffle - fails
 define <4 x i16> @reshuffle_v4i16_2x_nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: reshuffle_v4i16_2x_nxv8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[1]
 ; CHECK-NEXT:    umov w9, v1.h[1]
 ; CHECK-NEXT:    mov v0.h[1], w8
 ; CHECK-NEXT:    fmov w8, s1
 ; CHECK-NEXT:    mov v0.h[2], w8
 ; CHECK-NEXT:    mov v0.h[3], w9
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %el2 = extractelement <vscale x 8 x i16> %b, i32 0
   %el3 = extractelement <vscale x 8 x i16> %b, i32 1
   %v0 = insertelement <4 x i16> undef, i16 %el0, i32 0
   %v1 = insertelement <4 x i16> %v0, i16 %el1, i32 1
   %v2 = insertelement <4 x i16> %v1, i16 %el2, i32 2
   %v3 = insertelement <4 x i16> %v2, i16 %el3, i32 3
   ret <4 x i16> %v3
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_2x_nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_2x_nxv8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[3]
-; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov v0.h[2], w9
-; CHECK-NEXT:    umov w9, v1.h[1]
-; CHECK-NEXT:    mov v0.h[3], w10
-; CHECK-NEXT:    mov v0.h[4], w8
-; CHECK-NEXT:    umov w8, v1.h[2]
-; CHECK-NEXT:    mov v0.h[5], w9
-; CHECK-NEXT:    umov w9, v1.h[3]
-; CHECK-NEXT:    mov v0.h[6], w8
-; CHECK-NEXT:    mov v0.h[7], w9
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %el2 = extractelement <vscale x 8 x i16> %a, i32 2
   %el3 = extractelement <vscale x 8 x i16> %a, i32 3
   %el4 = extractelement <vscale x 8 x i16> %b, i32 0
   %el5 = extractelement <vscale x 8 x i16> %b, i32 1
; CHECK-NEXT:    ret
   %v7 = insertelement <8 x i16> %v6, i16 %el7, i32 7
   ret <8 x i16> %v7
 }
 
 
 ; == Shuffle comes from two input sources and result requires padding with undef ==
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_2x_nxv4i32_undef1(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_2x_nxv4i32_undef1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov v0.s[1], w8
-; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    zip1 v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i32> %a, i32 0
   %el1 = extractelement <vscale x 4 x i32> %b, i32 0
   %v0 = insertelement <4 x i32> undef, i32 %el0, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el1, i32 1
   ret <4 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_2x_nxv8i16_undef1(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_2x_nxv8i16_undef1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    zip1 v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %b, i32 0
   %v0 = insertelement <8 x i16> undef, i16 %el0, i32 0
   %v1 = insertelement <8 x i16> %v0, i16 %el1, i32 1
   ret <8 x i16> %v1
 }
 
 
+; ReconstructShuffle - fails
 define <8 x i16> @reshuffle_v8i16_2x_nxv8i16_undef2(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_2x_nxv8i16_undef2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[1]
 ; CHECK-NEXT:    umov w9, v1.h[1]
 ; CHECK-NEXT:    mov v0.h[1], w8
 ; CHECK-NEXT:    fmov w8, s1
 ; CHECK-NEXT:    mov v0.h[2], w8