[SVE][CodeGen] Add support for scalable vectors in AArch64TargetLowering::ReconstructShuffle

[david-arm]

Summary

At the moment if AArch64TargetLowering::ReconstructShuffle encounters
a scalable vector it just bails out of the optimisation entirely.
However, there are cases when we can make the transformation succeed
and produce far better code than if we just immediately bail out.

You can see the codegen improvement in the following tests:

CodeGen/AArch64/sve-fixed-length-reshuffle.ll

I have tried to highlight in sve-fixed-length-reshuffle.ll those tests
that exercise the AArch64TargetLowering::ReconstructShuffle code path
when the transformation succeeds or fails.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

9048

When protecting code can you check the positive rather than the negative. So if (SrcVTSize.isFixed() && ...). Looking at TypeSize suggests there's no such function, but I think that's an omission.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
   // extracting a suitable range of elements.
   for (auto &Src : Sources) {
     EVT SrcVT = Src.ShuffleVec.getValueType();
 
     TypeSize SrcVTSize = SrcVT.getSizeInBits();
     if (SrcVTSize == TypeSize::Fixed(VTSize))
       continue;
 
-    if (SrcVTSize.isScalable()) {
-      LLVM_DEBUG(
-          dbgs() << "Reshuffle failed: scalable vectors not yet supported\n");
-      return SDValue();
-    }
-
     // This stage of the search produces a source with the same element type as
     // the original, but with a total width matching the BUILD_VECTOR output.
     EVT EltVT = SrcVT.getVectorElementType();
     unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
     EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
 
-    if (SrcVTSize.getFixedValue() < VTSize) {
+    if (!SrcVTSize.isScalable() && SrcVTSize.getFixedValue() < VTSize) {
       assert(2 * SrcVTSize == VTSize);
       // We can pad out the smaller vector for free, so if it's part of a
       // shuffle...
       Src.ShuffleVec =
           DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
                       DAG.getUNDEF(Src.ShuffleVec.getValueType()));
       continue;
     }
 
-    if (SrcVTSize.getFixedValue() != 2 * VTSize) {
+    if ((!SrcVTSize.isScalable() && SrcVTSize.getFixedValue() != 2 * VTSize) ||
+        (SrcVTSize.isScalable() && SrcVTSize.getKnownMinValue() != VTSize)) {
       LLVM_DEBUG(
           dbgs() << "Reshuffle failed: result vector too small to extract\n");
       return SDValue();
     }
 
     if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
       LLVM_DEBUG(
           dbgs() << "Reshuffle failed: span too large for a VEXT to cope\n");
if (SrcEltTy == SmallestEltTy)
       continue;
     assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
     Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec);
     Src.WindowScale =
         SrcEltTy.getFixedSizeInBits() / SmallestEltTy.getFixedSizeInBits();
     Src.WindowBase *= Src.WindowScale;
   }
 
-  // Final check before we try to actually produce a shuffle.
   LLVM_DEBUG(for (auto Src
-                  : Sources)
-                 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
+                  : Sources) assert(Src.ShuffleVec.getValueType()
+                                        .getSizeInBits()
+                                        .getKnownMinValue() ==
+                                    ShuffleVT.getFixedSizeInBits()););
 
   // The stars all align, our next step is to produce the mask for the shuffle.
   SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
   int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
   for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
     SDValue Entry = Op.getOperand(i);
     if (Entry.isUndef())
       continue;

llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll

; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i1> %a, i32 0
   %el1 = extractelement <vscale x 4 x i1> %a, i32 1
   %v0 = insertelement <2 x i1> undef, i1 %el0, i32 0
   %v1 = insertelement <2 x i1> %v0, i1 %el1, i32 1
   ret <2 x i1> %v1
 }
 
 
+; ReconstructShuffle fails
 define <4 x i1> @reshuffle_v4i1_nxv4i1(<vscale x 4 x i1> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i1_nxv4i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z1.s, p0/z, #1 // =0x1
 ; CHECK-NEXT:    mov w8, v1.s[1]
 ; CHECK-NEXT:    mov w9, v1.s[2]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    mov v0.h[1], w8
; CHECK-NEXT:    ret
   %v0 = insertelement <4 x i1> undef, i1 %el0, i32 0
   %v1 = insertelement <4 x i1> %v0, i1 %el1, i32 1
   %v2 = insertelement <4 x i1> %v1, i1 %el2, i32 2
   %v3 = insertelement <4 x i1> %v2, i1 %el3, i32 3
   ret <4 x i1> %v3
 }
 
 
+; ReconstructShuffle fails
 define <4 x i1> @reshuffle_v4i1_nxv8i1(<vscale x 8 x i1> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i1_nxv8i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z1.h, p0/z, #1 // =0x1
 ; CHECK-NEXT:    umov w8, v1.h[1]
 ; CHECK-NEXT:    umov w9, v1.h[2]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    mov v0.h[1], w8
; CHECK-NEXT:    ret
   %v0 = insertelement <4 x i1> undef, i1 %el0, i32 0
   %v1 = insertelement <4 x i1> %v0, i1 %el1, i32 1
   %v2 = insertelement <4 x i1> %v1, i1 %el2, i32 2
   %v3 = insertelement <4 x i1> %v2, i1 %el3, i32 3
   ret <4 x i1> %v3
 }
 
 
+; ReconstructShuffle fails
 define <8 x i1> @reshuffle_v8i1_nxv8i1(<vscale x 8 x i1> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i1_nxv8i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z1.h, p0/z, #1 // =0x1
 ; CHECK-NEXT:    umov w8, v1.h[1]
 ; CHECK-NEXT:    umov w9, v1.h[2]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    mov v0.b[1], w8
; CHECK-NEXT:    ret
   %v4 = insertelement <8 x i1> %v3, i1 %el4, i32 4
   %v5 = insertelement <8 x i1> %v4, i1 %el5, i32 5
   %v6 = insertelement <8 x i1> %v5, i1 %el6, i32 6
   %v7 = insertelement <8 x i1> %v6, i1 %el7, i32 7
   ret <8 x i1> %v7
 }
 
 
+; ReconstructShuffle fails
 define <16 x i1> @reshuffle_v16i1_nxv16i1(<vscale x 16 x i1> %a) #0 {
 ; CHECK-LABEL: reshuffle_v16i1_nxv16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z1.b, p0/z, #1 // =0x1
 ; CHECK-NEXT:    umov w8, v1.b[1]
 ; CHECK-NEXT:    umov w9, v1.b[2]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    mov v0.b[1], w8
; CHECK-NEXT:    ret
   %v2 = insertelement <4 x float> %v1, float %el2, i32 2
   %v3 = insertelement <4 x float> %v2, float %el3, i32 3
   ret <4 x float> %v3
 }
 
 
 ; == Reversed first N elements ==
 
+; ReconstructShuffle - fails
 define <4 x i16> @reshuffle_v4i16_nxv4i16_reverse(<vscale x 4 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i16_nxv4i16_reverse:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, v0.s[3]
 ; CHECK-NEXT:    mov w9, v0.s[2]
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov w8, v0.s[1]
 ; CHECK-NEXT:    mov v1.h[1], w9
; CHECK-NEXT:    ret
   %v0 = insertelement <4 x i16> undef, i16 %el3, i32 0
   %v1 = insertelement <4 x i16> %v0, i16 %el2, i32 1
   %v2 = insertelement <4 x i16> %v1, i16 %el1, i32 2
   %v3 = insertelement <4 x i16> %v2, i16 %el0, i32 3
   ret <4 x i16> %v3
 }
 
 
+; ReconstructShuffle - fails
 define <8 x i16> @reshuffle_v8i16_nxv8i16_reverse(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_reverse:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[7]
 ; CHECK-NEXT:    umov w9, v0.h[6]
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    umov w8, v0.h[5]
 ; CHECK-NEXT:    mov v1.h[1], w9

 ; CHECK-NEXT:    dup v0.2s, w8
 ; CHECK-NEXT:    ret
   %el1 = extractelement <vscale x 4 x i32> %a, i32 1
   %v0 = insertelement <2 x i32> undef, i32 %el1, i32 1
   ret <2 x i32> %v0
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_nxv4i32_undef(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_nxv4i32_undef:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    mov v0.s[1], w8
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i32> %a, i32 0
   %el1 = extractelement <vscale x 4 x i32> %a, i32 1
   %v0 = insertelement <4 x i32> undef, i32 %el0, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el1, i32 1
   ret <4 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_nxv4i32_undef2(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_nxv4i32_undef2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[2]
-; CHECK-NEXT:    mov w9, v0.s[3]
-; CHECK-NEXT:    mov v0.s[2], w8
-; CHECK-NEXT:    mov v0.s[3], w9
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el2 = extractelement <vscale x 4 x i32> %a, i32 2
   %el3 = extractelement <vscale x 4 x i32> %a, i32 3
   %v0 = insertelement <4 x i32> undef, i32 %el2, i32 2
   %v1 = insertelement <4 x i32> %v0, i32 %el3, i32 3
   ret <4 x i32> %v1
 }
 

 ; CHECK-NEXT:    dup v0.2s, w8
 ; CHECK-NEXT:    ret
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %v0 = insertelement <2 x i16> undef, i16 %el1, i32 1
   ret <2 x i16> %v0
 }
 
 
+; ReconstructShuffle - fails
 define <4 x i16> @reshuffle_v4i16_nxv8i16_undef(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i16_nxv8i16_undef:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[1]
 ; CHECK-NEXT:    mov v0.h[1], w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %v0 = insertelement <4 x i16> undef, i16 %el0, i32 0
   %v1 = insertelement <4 x i16> %v0, i16 %el1, i32 1
   ret <4 x i16> %v1
 }
 
+; ReconstructShuffle - fails
 define <4 x i16> @reshuffle_v4i16_nxv8i16_undef2(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i16_nxv8i16_undef2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[2]
 ; CHECK-NEXT:    umov w9, v0.h[3]
 ; CHECK-NEXT:    mov v0.h[2], w8
 ; CHECK-NEXT:    mov v0.h[3], w9
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %el2 = extractelement <vscale x 8 x i16> %a, i32 2
   %el3 = extractelement <vscale x 8 x i16> %a, i32 3
   %v0 = insertelement <4 x i16> undef, i16 %el2, i32 2
   %v1 = insertelement <4 x i16> %v0, i16 %el3, i32 3
   ret <4 x i16> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_nxv8i16_undef(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_undef:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[3]
-; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    mov v0.h[2], w9
-; CHECK-NEXT:    mov v0.h[3], w10
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %el2 = extractelement <vscale x 8 x i16> %a, i32 2
   %el3 = extractelement <vscale x 8 x i16> %a, i32 3
   %v0 = insertelement <8 x i16> undef, i16 %el0, i32 0
   %v1 = insertelement <8 x i16> %v0, i16 %el1, i32 1
   %v2 = insertelement <8 x i16> %v1, i16 %el2, i32 2
   %v3 = insertelement <8 x i16> %v2, i16 %el3, i32 3
   ret <8 x i16> %v3
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_nxv8i16_undef2(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_undef2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umov w8, v0.h[4]
-; CHECK-NEXT:    umov w9, v0.h[5]
-; CHECK-NEXT:    mov v1.h[4], w8
-; CHECK-NEXT:    umov w8, v0.h[6]
-; CHECK-NEXT:    mov v1.h[5], w9
-; CHECK-NEXT:    umov w9, v0.h[7]
-; CHECK-NEXT:    mov v1.h[6], w8
-; CHECK-NEXT:    mov v1.h[7], w9
-; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el4 = extractelement <vscale x 8 x i16> %a, i32 4
   %el5 = extractelement <vscale x 8 x i16> %a, i32 5
   %el6 = extractelement <vscale x 8 x i16> %a, i32 6
   %el7 = extractelement <vscale x 8 x i16> %a, i32 7
   %v0 = insertelement <8 x i16> undef, i16 %el4, i32 4
   %v1 = insertelement <8 x i16> %v0, i16 %el5, i32 5
   %v2 = insertelement <8 x i16> %v1, i16 %el6, i32 6
   %v3 = insertelement <8 x i16> %v2, i16 %el7, i32 7
   ret <8 x i16> %v3
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_nxv4i32_reverse_undef(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_nxv4i32_reverse_undef:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i32> %a, i32 0
   %el1 = extractelement <vscale x 4 x i32> %a, i32 1
   %v0 = insertelement <4 x i32> undef, i32 %el1, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el0, i32 1
   ret <4 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_nxv4i32_reverse_undef2(<vscale x 4 x i32> %a) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_nxv4i32_reverse_undef2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[3]
-; CHECK-NEXT:    mov w9, v0.s[2]
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
 ; CHECK-NEXT:    ret
   %el2 = extractelement <vscale x 4 x i32> %a, i32 2
   %el3 = extractelement <vscale x 4 x i32> %a, i32 3
   %v0 = insertelement <4 x i32> undef, i32 %el3, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el2, i32 1
   ret <4 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_nxv8i16_reverse_undef(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_reverse_undef:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umov w8, v0.h[3]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    mov v1.h[1], w9
-; CHECK-NEXT:    mov v1.h[2], w8
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    mov v1.h[3], w8
-; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    rev64 v0.8h, v0.8h
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %el2 = extractelement <vscale x 8 x i16> %a, i32 2
   %el3 = extractelement <vscale x 8 x i16> %a, i32 3
   %v0 = insertelement <8 x i16> undef, i16 %el3, i32 0
   %v1 = insertelement <8 x i16> %v0, i16 %el2, i32 1
   %v2 = insertelement <8 x i16> %v1, i16 %el1, i32 2
   %v3 = insertelement <8 x i16> %v2, i16 %el0, i32 3
   ret <8 x i16> %v3
 }
 
 
+; ReconstructShuffle - fails
 define <8 x i16> @reshuffle_v8i16_nxv8i16_reverse_undef2(<vscale x 8 x i16> %a) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_nxv8i16_reverse_undef2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[7]
 ; CHECK-NEXT:    umov w9, v0.h[6]
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    umov w8, v0.h[5]
 ; CHECK-NEXT:    mov v1.h[1], w9
; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 2 x i32> %a, i32 0
   %el1 = extractelement <vscale x 2 x i32> %b, i32 0
   %v0 = insertelement <2 x i32> undef, i32 %el0, i32 0
   %v1 = insertelement <2 x i32> %v0, i32 %el1, i32 1
   ret <2 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_2x_nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_2x_nxv4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    mov w9, v1.s[1]
-; CHECK-NEXT:    mov v0.s[1], w8
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov v0.s[2], w8
-; CHECK-NEXT:    mov v0.s[3], w9
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i32> %a, i32 0
   %el1 = extractelement <vscale x 4 x i32> %a, i32 1
   %el2 = extractelement <vscale x 4 x i32> %b, i32 0
   %el3 = extractelement <vscale x 4 x i32> %b, i32 1
   %v0 = insertelement <4 x i32> undef, i32 %el0, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el1, i32 1
   %v2 = insertelement <4 x i32> %v1, i32 %el2, i32 2
   %v3 = insertelement <4 x i32> %v2, i32 %el3, i32 3
   ret <4 x i32> %v3
 }
 
 
+; ReconstructShuffle - fails
 define <4 x i16> @reshuffle_v4i16_2x_nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: reshuffle_v4i16_2x_nxv8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[1]
 ; CHECK-NEXT:    umov w9, v1.h[1]
 ; CHECK-NEXT:    mov v0.h[1], w8
 ; CHECK-NEXT:    fmov w8, s1
 ; CHECK-NEXT:    mov v0.h[2], w8
 ; CHECK-NEXT:    mov v0.h[3], w9
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %el2 = extractelement <vscale x 8 x i16> %b, i32 0
   %el3 = extractelement <vscale x 8 x i16> %b, i32 1
   %v0 = insertelement <4 x i16> undef, i16 %el0, i32 0
   %v1 = insertelement <4 x i16> %v0, i16 %el1, i32 1
   %v2 = insertelement <4 x i16> %v1, i16 %el2, i32 2
   %v3 = insertelement <4 x i16> %v2, i16 %el3, i32 3
   ret <4 x i16> %v3
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_2x_nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_2x_nxv8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[3]
-; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov v0.h[2], w9
-; CHECK-NEXT:    umov w9, v1.h[1]
-; CHECK-NEXT:    mov v0.h[3], w10
-; CHECK-NEXT:    mov v0.h[4], w8
-; CHECK-NEXT:    umov w8, v1.h[2]
-; CHECK-NEXT:    mov v0.h[5], w9
-; CHECK-NEXT:    umov w9, v1.h[3]
-; CHECK-NEXT:    mov v0.h[6], w8
-; CHECK-NEXT:    mov v0.h[7], w9
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %a, i32 1
   %el2 = extractelement <vscale x 8 x i16> %a, i32 2
   %el3 = extractelement <vscale x 8 x i16> %a, i32 3
   %el4 = extractelement <vscale x 8 x i16> %b, i32 0
   %el5 = extractelement <vscale x 8 x i16> %b, i32 1
; CHECK-NEXT:    ret
   %v7 = insertelement <8 x i16> %v6, i16 %el7, i32 7
   ret <8 x i16> %v7
 }
 
 
 ; == Shuffle comes from two input sources and result requires padding with undef ==
 
 
+; ReconstructShuffle - succeeds
 define <4 x i32> @reshuffle_v4i32_2x_nxv4i32_undef1(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
 ; CHECK-LABEL: reshuffle_v4i32_2x_nxv4i32_undef1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov v0.s[1], w8
-; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    zip1 v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 4 x i32> %a, i32 0
   %el1 = extractelement <vscale x 4 x i32> %b, i32 0
   %v0 = insertelement <4 x i32> undef, i32 %el0, i32 0
   %v1 = insertelement <4 x i32> %v0, i32 %el1, i32 1
   ret <4 x i32> %v1
 }
 
 
+; ReconstructShuffle - succeeds
 define <8 x i16> @reshuffle_v8i16_2x_nxv8i16_undef1(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_2x_nxv8i16_undef1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    zip1 v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %el0 = extractelement <vscale x 8 x i16> %a, i32 0
   %el1 = extractelement <vscale x 8 x i16> %b, i32 0
   %v0 = insertelement <8 x i16> undef, i16 %el0, i32 0
   %v1 = insertelement <8 x i16> %v0, i16 %el1, i32 1
   ret <8 x i16> %v1
 }
 
 
+; ReconstructShuffle - fails
 define <8 x i16> @reshuffle_v8i16_2x_nxv8i16_undef2(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: reshuffle_v8i16_2x_nxv8i16_undef2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umov w8, v0.h[1]
 ; CHECK-NEXT:    umov w9, v1.h[1]
 ; CHECK-NEXT:    mov v0.h[1], w8
 ; CHECK-NEXT:    fmov w8, s1
 ; CHECK-NEXT:    mov v0.h[2], w8