[AArch64][SVE] Custom ISelLowering for 256b `shuffle_vector v, undef, <1, 1, 0, 0>`

[cameron.mcinally]

Summary

This is more a bug report than earnest patch...

We've found a couple of inefficiently lowered shuffles when targeting neoverse-v1 and VLS. This patch covers:

%x = shufflevector <2 x double> %v, <2 x double> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>

It could be lowered in a number of ways, but I chose:

zip1 z0.d, z0.d, z0.d
ext z0.b, z0.b, z0.b, #16

The new lowering shows an 11% performance boost on 538.namd with our out-of-tree compiler.

[efriedma]

For NEON, we would use the PerfectShuffle tables for something like this... should we try to use those tables here? I mean, I guess it's kind of narrow to implement perfect shuffle tables specifically for <4 x double>, but it might make sense...

We should probably also consider implementing a general-purpose fallback for shuffling that doesn't involve the stack. For a shuffle with one source, we can use tbl; I guess for the general case we'd have to use tbl+tbl+orr. (Sort of messy, but almost certainly better than the default fallback of storing to the stack element by element.)

[cameron.mcinally]

@efriedma, I agree. There's a more general solution out there, but I'm too far removed from the AArch64 backend to see it.

The other interesting case from 538.namd is:

shufflevector <2 x double> %v, <2 x double> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>

And I suspect there are more that I haven't found yet, especially from Complex.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
   for (unsigned i = 0; i < NumElts; i += 2) {
     if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) ||
         (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + WhichResult))
       return false;
   }
   return true;
 }
 
+/// is256bRevDupMask - Special case for <2 x double> shuffles of the form:
+/// "vector_shuffle v, undef, <1, 1, 0, 0>"
+static bool is256bRevDupMask(ArrayRef<int> M, EVT VT) {
+  unsigned NumElts = VT.getVectorNumElements();
+  if (NumElts % 2 != 0)
+    return false;
+  if (VT.getSizeInBits() != 256 || VT.getScalarSizeInBits() != 64)
+    return false;
+  for (unsigned i = 0; i != NumElts; i ++) {
+    if (i < NumElts / 2) {
+      if (M[i] != 1)
+        return false;
+    } else {
+      if (M[i] != 0)
+        return false;
+    }
+  }
+  return true;
+}
+
 static bool isINSMask(ArrayRef<int> M, int NumInputElements,
                       bool &DstIsLeft, int &Anomaly) {
   if (M.size() != static_cast<size_t>(NumInputElements))
     return false;
 
   int NumLHSMatch = 0, NumRHSMatch = 0;
   int LastLHSMismatch = -1, LastRHSMismatch = -1;
 
if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult != 0)
       return convertFromScalableVector(
           DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op1));
 
     if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
       unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
       return convertFromScalableVector(
           DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
     }
+
+    if (is256bRevDupMask(ShuffleMask, VT)) {
+      Op = DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op1);
+      Op = DAG.getNode(ISD::VECTOR_SPLICE, DL, ContainerVT, Op, Op,
+                       DAG.getConstant(2, DL, MVT::i64));
+      return convertFromScalableVector(DAG, VT, Op);
+    }
   }
 
   return SDValue();
 }
 
 SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
                                                  SelectionDAG &DAG) const {
   SDLoc DL(Op);

llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle.ll

%ret = shufflevector <32 x i8> %op1, <32 x i8> %op2, <32 x i32> <i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38,
                                                                    i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62>
   store <32 x i8> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-LABEL: shuffle_ext_byone_v64i8:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov w8, #32
+; VBITS_GE_256-NEXT:    mov w8, #32 // =0x20
 ; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
 ; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x1, x8]
 ; VBITS_GE_256-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    mov z0.b, z0.b[31]
 ; VBITS_GE_256-NEXT:    mov z3.b, z2.b[31]
 ; VBITS_GE_256-NEXT:    fmov w9, s0
 ; VBITS_GE_256-NEXT:    fmov w10, s3
; VBITS_GE_512-NEXT:    ret
   store <64 x i8> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v128i8(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v128i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b, vl128
-; CHECK-NEXT:    mov w8, #127
+; CHECK-NEXT:    mov w8, #127 // =0x7f
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.b, xzr, x8
 ; CHECK-NEXT:    lastb w8, p1, z0.b
 ; CHECK-NEXT:    insr z1.b, w8
 ; CHECK-NEXT:    st1b { z1.b }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <128 x i8>, ptr %a
; CHECK-NEXT:    ret
   store <128 x i8> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v256i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b, vl256
-; CHECK-NEXT:    mov w8, #255
+; CHECK-NEXT:    mov w8, #255 // =0xff
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.b, xzr, x8
 ; CHECK-NEXT:    lastb w8, p1, z0.b
 ; CHECK-NEXT:    insr z1.b, w8
 ; CHECK-NEXT:    st1b { z1.b }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <256 x i8>, ptr %a
%ret = shufflevector <16 x i16> %op1, <16 x i16> %op2, <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
                                                                      i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
   store <16 x i16> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-LABEL: shuffle_ext_byone_v32i16:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #16
+; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    mov z0.h, z0.h[15]
 ; VBITS_GE_256-NEXT:    mov z3.h, z2.h[15]
 ; VBITS_GE_256-NEXT:    fmov w9, s0
 ; VBITS_GE_256-NEXT:    fmov w10, s3
; VBITS_GE_512-NEXT:    ret
   store <32 x i16> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v64i16(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v64i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl64
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mov w8, #63 // =0x3f
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.h, xzr, x8
 ; CHECK-NEXT:    lastb w8, p1, z0.h
 ; CHECK-NEXT:    insr z1.h, w8
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <64 x i16>, ptr %a
; CHECK-NEXT:    ret
   store <64 x i16> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v128i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl128
-; CHECK-NEXT:    mov w8, #127
+; CHECK-NEXT:    mov w8, #127 // =0x7f
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.h, xzr, x8
 ; CHECK-NEXT:    lastb w8, p1, z0.h
 ; CHECK-NEXT:    insr z1.h, w8
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <128 x i16>, ptr %a
; CHECK-NEXT:    ret
   %ret = shufflevector <8 x i32> %op1, <8 x i32> %op2, <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
   store <8 x i32> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-LABEL: shuffle_ext_byone_v16i32:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    mov z0.s, z0.s[7]
 ; VBITS_GE_256-NEXT:    mov z3.s, z2.s[7]
 ; VBITS_GE_256-NEXT:    fmov w9, s0
 ; VBITS_GE_256-NEXT:    fmov w10, s3
; VBITS_GE_512-NEXT:    ret
   store <16 x i32> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v32i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl32
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mov w8, #31 // =0x1f
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.s, xzr, x8
 ; CHECK-NEXT:    lastb w8, p1, z0.s
 ; CHECK-NEXT:    insr z1.s, w8
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x i32>, ptr %a
   %op2 = load <32 x i32>, ptr %b
   %ret = shufflevector <32 x i32> %op1, <32 x i32> %op2, <32 x i32> <i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38,
                                                                      i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46,
                                                                      i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54,
                                                                      i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62>
   store <32 x i32> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v64i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl64
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mov w8, #63 // =0x3f
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.s, xzr, x8
 ; CHECK-NEXT:    lastb w8, p1, z0.s
 ; CHECK-NEXT:    insr z1.s, w8
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <64 x i32>, ptr %a
; CHECK-NEXT:    ret
   %ret = shufflevector <4 x i64> %op1, <4 x i64> %op2, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
   store <4 x i64> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-LABEL: shuffle_ext_byone_v8i64:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    mov z0.d, z0.d[3]
 ; VBITS_GE_256-NEXT:    mov z3.d, z2.d[3]
 ; VBITS_GE_256-NEXT:    fmov x9, d0
 ; VBITS_GE_256-NEXT:    fmov x10, d3
; VBITS_GE_512-NEXT:    ret
   store <8 x i64> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v16i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl16
-; CHECK-NEXT:    mov w8, #15
+; CHECK-NEXT:    mov w8, #15 // =0xf
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.d, xzr, x8
 ; CHECK-NEXT:    lastb x8, p1, z0.d
 ; CHECK-NEXT:    insr z1.d, x8
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i64>, ptr %a
   %op2 = load <16 x i64>, ptr %b
   %ret = shufflevector <16 x i64> %op1, <16 x i64> %op2, <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
                                                                      i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
   store <16 x i64> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v32i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl32
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mov w8, #31 // =0x1f
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.d, xzr, x8
 ; CHECK-NEXT:    lastb x8, p1, z0.d
 ; CHECK-NEXT:    insr z1.d, x8
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x i64>, ptr %a
%ret = shufflevector <16 x half> %op1, <16 x half> %op2, <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
                                                                        i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
   store <16 x half> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v32f16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-LABEL: shuffle_ext_byone_v32f16:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #16
+; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x1, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    mov z0.h, z0.h[15]
 ; VBITS_GE_256-NEXT:    mov z3.h, z2.h[15]
 ; VBITS_GE_256-NEXT:    insr z2.h, h0
 ; VBITS_GE_256-NEXT:    insr z1.h, h3
; VBITS_GE_512-NEXT:    ret
   store <32 x half> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v64f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl64
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mov w8, #63 // =0x3f
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.h, xzr, x8
 ; CHECK-NEXT:    lastb h0, p1, z0.h
 ; CHECK-NEXT:    insr z1.h, h0
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <64 x half>, ptr %a
; CHECK-NEXT:    ret
   store <64 x half> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v128f16(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v128f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h, vl128
-; CHECK-NEXT:    mov w8, #127
+; CHECK-NEXT:    mov w8, #127 // =0x7f
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.h, xzr, x8
 ; CHECK-NEXT:    lastb h0, p1, z0.h
 ; CHECK-NEXT:    insr z1.h, h0
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <128 x half>, ptr %a
; CHECK-NEXT:    ret
   %ret = shufflevector <8 x float> %op1, <8 x float> %op2, <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
   store <8 x float> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-LABEL: shuffle_ext_byone_v16f32:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    mov z0.s, z0.s[7]
 ; VBITS_GE_256-NEXT:    mov z3.s, z2.s[7]
 ; VBITS_GE_256-NEXT:    insr z2.s, s0
 ; VBITS_GE_256-NEXT:    insr z1.s, s3
; VBITS_GE_512-NEXT:    ret
   store <16 x float> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v32f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl32
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mov w8, #31 // =0x1f
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.s, xzr, x8
 ; CHECK-NEXT:    lastb s0, p1, z0.s
 ; CHECK-NEXT:    insr z1.s, s0
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x float>, ptr %a
   %op2 = load <32 x float>, ptr %b
   %ret = shufflevector <32 x float> %op1, <32 x float> %op2, <32 x i32> <i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38,
                                                                          i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46,
                                                                          i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54,
                                                                          i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62>
   store <32 x float> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v64f32(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v64f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl64
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mov w8, #63 // =0x3f
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.s, xzr, x8
 ; CHECK-NEXT:    lastb s0, p1, z0.s
 ; CHECK-NEXT:    insr z1.s, s0
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <64 x float>, ptr %a
; CHECK-NEXT:    ret
   %ret = shufflevector <4 x double> %op1, <4 x double> %op2, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
   store <4 x double> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-LABEL: shuffle_ext_byone_v8f64:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    mov z0.d, z0.d[3]
 ; VBITS_GE_256-NEXT:    mov z3.d, z2.d[3]
 ; VBITS_GE_256-NEXT:    insr z2.d, d0
 ; VBITS_GE_256-NEXT:    insr z1.d, d3
; VBITS_GE_512-NEXT:    ret
   store <8 x double> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v16f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl16
-; CHECK-NEXT:    mov w8, #15
+; CHECK-NEXT:    mov w8, #15 // =0xf
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.d, xzr, x8
 ; CHECK-NEXT:    lastb d0, p1, z0.d
 ; CHECK-NEXT:    insr z1.d, d0
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x double>, ptr %a
   %op2 = load <16 x double>, ptr %b
   %ret = shufflevector <16 x double> %op1, <16 x double> %op2, <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
                                                                            i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
   store <16 x double> %ret, ptr %a
   ret void
 }
 
 define void @shuffle_ext_byone_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-LABEL: shuffle_ext_byone_v32f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl32
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mov w8, #31 // =0x1f
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    whilels p1.d, xzr, x8
 ; CHECK-NEXT:    lastb d0, p1, z0.d
 ; CHECK-NEXT:    insr z1.d, d0
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x double>, ptr %a

 ; CHECK-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %ret = shufflevector <4 x double> %op1, <4 x double> %op2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   store <4 x double> %ret, ptr %a
   ret void
 }
 
+define void @shuffle_256b_rev_and_dup(ptr %a) vscale_range(2,2) #0 {
+; CHECK-LABEL: shuffle_256b_rev_and_dup:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    zip1 z0.d, z0.d, z0.d
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+   %op = load <2 x double>, ptr %a
+   %ret = shufflevector <2 x double> %op, <2 x double> poison, <4 x i32> <i32 1, i32 1, i32 0, i32 0>
+   store <4 x double> %ret, ptr %a
+   ret void
+}
+
 attributes #0 = { "target-features"="+sve" }