[ARM,MVE] Add intrinsics for v[id]dupq and v[id]wdupq.

[simon_tatham]

Summary

These instructions generate a vector of consecutive elements starting
from a given base value and incrementing by 1, 2, 4 or 8. The wdup
versions also wrap the values back to zero when they reach a given
limit value. The instruction updates the scalar base register so that
another use of the same instruction will continue the sequence from
where the previous one left off.

At the IR level, I've represented these instructions as a family of
target-specific intrinsics with two return values (the constructed
vector and the updated base). The user-facing ACLE API provides a set
of intrinsics that throw away the written-back base and another set
that receive it as a pointer so they can update it, plus the usual
predicated versions.

Because the intrinsics return two values (as do the underlying
instructions), the isel has to be done in C++.

This is the first family of MVE intrinsics that use the imm_1248
immediate type in the clang Tablegen framework, so naturally, I found
I'd given it the wrong C integer type. Also added some tests of the
check that the immediate has a legal value, because this is the first
time those particular checks have been exercised.

Finally, I also had to fix a bug in MveEmitter which failed an
assertion when I nested two seq nodes (the inner one used to extract
the two values from the pair returned by the IR intrinsic, and the
outer one put on by the predication multiclass).

[dmgreen]

Looks good, from what I can tell.

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

2752

VT.getScalarSizeInBits()?

clang/include/clang/Basic/arm_mve.td

let params = T.Usual in {
   let pnt = PNT_None in
     def vdupq_n: Intrinsic<Vector, (args unpromoted<Scalar>:$s), (splat $s)>;
 
   defm vdupq: IntrinsicMX<
       Vector, (args unpromoted<Scalar>:$s, Predicate:$pred),
       (select $pred, (splat $s), $inactive), 1, "_n", PNT_NType, PNT_None>;
 }
 
+multiclass vxdup_mc<dag paramsIn, dag paramsOut> {
+  defvar UnpredInt     = IRInt<NAME, [Vector]>;
+  defvar PredInt       = IRInt<NAME # "_predicated", [Vector, Predicate]>;
+  defvar UnpredIntCall = !con((UnpredInt $base), paramsOut);
+  defvar PredIntCall   = !con((PredInt $inactive, $base), paramsOut, (? $pred));
+
+  // Straightforward case with neither writeback nor predication
+  let pnt = PNT_N in
+    def q_n: Intrinsic<Vector, !con((args u32:$base), paramsIn),
+                       (xval UnpredIntCall, 0)>;
+
+  // Predicated form without writeback
+  defm q: IntrinsicMX<
+      Vector, !con((args u32:$base), paramsIn, (? Predicate:$pred)),
+      (xval PredIntCall, 0), 1, "_n", PNT_NType, PNT_N>;
+
+  // Writeback without predication
+  let pnt = PNT_WB in
+    def q_wb: Intrinsic<
+      Vector, !con((args Ptr<u32>:$baseaddr), paramsIn),
+      (seq (load $baseaddr):$base,
+           UnpredIntCall:$pair,
+           (store (xval $pair, 1), $baseaddr),
+           (xval $pair, 0))>;
+
+  // Both writeback and predicated
+  defm q: IntrinsicMX<
+      Vector, !con((args Ptr<u32>:$baseaddr), paramsIn, (? Predicate:$pred)),
+      (seq (load $baseaddr):$base,
+           PredIntCall:$pair,
+           (store (xval $pair, 1), $baseaddr),
+           (xval $pair, 0)), 1, "_wb", PNT_WBType, PNT_WB>;
+}
+
+let params = T.Unsigned in {
+  defm vidup:  vxdup_mc<(?             imm_1248:$step), (?         $step)>;
+  defm vddup:  vxdup_mc<(?             imm_1248:$step), (?         $step)>;
+  defm viwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>;
+  defm vdwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>;
+}
+
 // The bitcasting below is not overcomplicating the IR because while
 // Vector and UVector may be different vector types at the C level i.e.
 // vectors of same size signed/unsigned ints. Once they're lowered
 // to IR, they are just bit vectors with no sign at all, so the
 // bitcasts will be automatically elided by IRBuilder.
 multiclass predicated_bit_op_fp<string int_op> {
 def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
                                 Predicate:$pred),

clang/include/clang/Basic/arm_mve_defs.td

 // it can range from 0 to (128 / size of scalar)-1 inclusive. (e.g. vgetq_lane)
 def imm_lane : Immediate<sint, IB_LaneIndex>;
 
 // imm_1to32 can be in the range 1 to 32, unconditionally. (e.g. scalar shift
 // intrinsics)
 def imm_1to32 : Immediate<sint, IB_ConstRange<1, 32>>;
 
 // imm_1248 can be 1, 2, 4 or 8. (e.g. vidupq)
-def imm_1248 : Immediate<u32, IB_ConstRange<1, 8>> {
+def imm_1248 : Immediate<sint, IB_ConstRange<1, 8>> {
   let extra = "Power2";
 }
 
 // imm_mem7bit<n> is a valid immediate offset for a load/store intrinsic whose
 // memory access size is n bytes (e.g. 1 for vldrb_[whatever], 2 for vldrh,
 // ...). The set of valid immediates for these is {-127*n, ..., -1*n, 0*n, 1*n,
 // ..., 127*n}.
 class imm_mem7bit<int membytes>

clang/test/CodeGen/arm-mve-intrinsics/idup.c

+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+
+#include <arm_mve.h>
+
+// CHECK-LABEL: @test_vidupq_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vidup.v16i8(i32 [[A:%.*]], i32 4)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+//
+uint8x16_t test_vidupq_n_u8(uint32_t a)
+{
+#ifdef POLYMORPHIC
+    return vidupq_u8(a, 4);
+#else /* POLYMORPHIC */
+    return vidupq_n_u8(a, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vidup.v8i16(i32 [[A:%.*]], i32 1)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_vidupq_n_u16(uint32_t a)
+{
+#ifdef POLYMORPHIC
+    return vidupq_u16(a, 1);
+#else /* POLYMORPHIC */
+    return vidupq_n_u16(a, 1);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 [[A:%.*]], i32 4)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_vidupq_n_u32(uint32_t a)
+{
+#ifdef POLYMORPHIC
+    return vidupq_u32(a, 4);
+#else /* POLYMORPHIC */
+    return vidupq_n_u32(a, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vddup.v16i8(i32 [[A:%.*]], i32 2)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+//
+uint8x16_t test_vddupq_n_u8(uint32_t a)
+{
+#ifdef POLYMORPHIC
+    return vddupq_u8(a, 2);
+#else /* POLYMORPHIC */
+    return vddupq_n_u8(a, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vddup.v8i16(i32 [[A:%.*]], i32 4)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_vddupq_n_u16(uint32_t a)
+{
+#ifdef POLYMORPHIC
+    return vddupq_u16(a, 4);
+#else /* POLYMORPHIC */
+    return vddupq_n_u16(a, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vddup.v4i32(i32 [[A:%.*]], i32 2)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_vddupq_n_u32(uint32_t a)
+{
+#ifdef POLYMORPHIC
+    return vddupq_u32(a, 2);
+#else /* POLYMORPHIC */
+    return vddupq_n_u32(a, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.v16i8(i32 [[A:%.*]], i32 [[B:%.*]], i32 4)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+//
+uint8x16_t test_viwdupq_n_u8(uint32_t a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_u8(a, b, 4);
+#else /* POLYMORPHIC */
+    return viwdupq_n_u8(a, b, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.v8i16(i32 [[A:%.*]], i32 [[B:%.*]], i32 2)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_viwdupq_n_u16(uint32_t a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_u16(a, b, 2);
+#else /* POLYMORPHIC */
+    return viwdupq_n_u16(a, b, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.v4i32(i32 [[A:%.*]], i32 [[B:%.*]], i32 8)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_viwdupq_n_u32(uint32_t a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_u32(a, b, 8);
+#else /* POLYMORPHIC */
+    return viwdupq_n_u32(a, b, 8);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.v16i8(i32 [[A:%.*]], i32 [[B:%.*]], i32 4)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+//
+uint8x16_t test_vdwdupq_n_u8(uint32_t a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_u8(a, b, 4);
+#else /* POLYMORPHIC */
+    return vdwdupq_n_u8(a, b, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.v8i16(i32 [[A:%.*]], i32 [[B:%.*]], i32 8)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_vdwdupq_n_u16(uint32_t a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_u16(a, b, 8);
+#else /* POLYMORPHIC */
+    return vdwdupq_n_u16(a, b, 8);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.v4i32(i32 [[A:%.*]], i32 [[B:%.*]], i32 1)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP0]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_vdwdupq_n_u32(uint32_t a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_u32(a, b, 1);
+#else /* POLYMORPHIC */
+    return vdwdupq_n_u32(a, b, 1);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vidup.v16i8(i32 [[TMP0]], i32 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vidupq_wb_u8(uint32_t *a)
+{
+#ifdef POLYMORPHIC
+    return vidupq_u8(a, 8);
+#else /* POLYMORPHIC */
+    return vidupq_wb_u8(a, 8);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vidup.v8i16(i32 [[TMP0]], i32 1)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vidupq_wb_u16(uint32_t *a)
+{
+#ifdef POLYMORPHIC
+    return vidupq_u16(a, 1);
+#else /* POLYMORPHIC */
+    return vidupq_wb_u16(a, 1);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 [[TMP0]], i32 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vidupq_wb_u32(uint32_t *a)
+{
+#ifdef POLYMORPHIC
+    return vidupq_u32(a, 4);
+#else /* POLYMORPHIC */
+    return vidupq_wb_u32(a, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vddup.v16i8(i32 [[TMP0]], i32 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vddupq_wb_u8(uint32_t *a)
+{
+#ifdef POLYMORPHIC
+    return vddupq_u8(a, 2);
+#else /* POLYMORPHIC */
+    return vddupq_wb_u8(a, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vddup.v8i16(i32 [[TMP0]], i32 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vddupq_wb_u16(uint32_t *a)
+{
+#ifdef POLYMORPHIC
+    return vddupq_u16(a, 8);
+#else /* POLYMORPHIC */
+    return vddupq_wb_u16(a, 8);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vddup.v4i32(i32 [[TMP0]], i32 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vddupq_wb_u32(uint32_t *a)
+{
+#ifdef POLYMORPHIC
+    return vddupq_u32(a, 2);
+#else /* POLYMORPHIC */
+    return vddupq_wb_u32(a, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.v16i8(i32 [[TMP0]], i32 [[B:%.*]], i32 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vdwdupq_wb_u8(uint32_t *a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_u8(a, b, 4);
+#else /* POLYMORPHIC */
+    return vdwdupq_wb_u8(a, b, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.v8i16(i32 [[TMP0]], i32 [[B:%.*]], i32 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vdwdupq_wb_u16(uint32_t *a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_u16(a, b, 4);
+#else /* POLYMORPHIC */
+    return vdwdupq_wb_u16(a, b, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.v16i8(i32 [[TMP0]], i32 [[B:%.*]], i32 1)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_viwdupq_wb_u8(uint32_t *a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_u8(a, b, 1);
+#else /* POLYMORPHIC */
+    return viwdupq_wb_u8(a, b, 1);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.v8i16(i32 [[TMP0]], i32 [[B:%.*]], i32 1)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_viwdupq_wb_u16(uint32_t *a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_u16(a, b, 1);
+#else /* POLYMORPHIC */
+    return viwdupq_wb_u16(a, b, 1);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.v4i32(i32 [[TMP0]], i32 [[B:%.*]], i32 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_viwdupq_wb_u32(uint32_t *a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_u32(a, b, 8);
+#else /* POLYMORPHIC */
+    return viwdupq_wb_u32(a, b, 8);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.v4i32(i32 [[TMP0]], i32 [[B:%.*]], i32 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP1]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vdwdupq_wb_u32(uint32_t *a, uint32_t b)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_u32(a, b, 2);
+#else /* POLYMORPHIC */
+    return vdwdupq_wb_u32(a, b, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_m_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vidup.predicated.v16i8.v16i1(<16 x i8> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 8, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vidupq_m_n_u8(uint8x16_t inactive, uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_m(inactive, a, 8, p);
+#else /* POLYMORPHIC */
+    return vidupq_m_n_u8(inactive, a, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_m_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vidup.predicated.v8i16.v8i1(<8 x i16> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 8, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vidupq_m_n_u16(uint16x8_t inactive, uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_m(inactive, a, 8, p);
+#else /* POLYMORPHIC */
+    return vidupq_m_n_u16(inactive, a, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_m_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vidup.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 2, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vidupq_m_n_u32(uint32x4_t inactive, uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_m(inactive, a, 2, p);
+#else /* POLYMORPHIC */
+    return vidupq_m_n_u32(inactive, a, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_m_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vddup.predicated.v16i8.v16i1(<16 x i8> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 8, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vddupq_m_n_u8(uint8x16_t inactive, uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_m(inactive, a, 8, p);
+#else /* POLYMORPHIC */
+    return vddupq_m_n_u8(inactive, a, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_m_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vddup.predicated.v8i16.v8i1(<8 x i16> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 2, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vddupq_m_n_u16(uint16x8_t inactive, uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_m(inactive, a, 2, p);
+#else /* POLYMORPHIC */
+    return vddupq_m_n_u16(inactive, a, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_m_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vddup.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 8, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vddupq_m_n_u32(uint32x4_t inactive, uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_m(inactive, a, 8, p);
+#else /* POLYMORPHIC */
+    return vddupq_m_n_u32(inactive, a, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_m_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.predicated.v16i8.v16i1(<16 x i8> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i32 8, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_viwdupq_m_n_u8(uint8x16_t inactive, uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_m(inactive, a, b, 8, p);
+#else /* POLYMORPHIC */
+    return viwdupq_m_n_u8(inactive, a, b, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_m_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.predicated.v8i16.v8i1(<8 x i16> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i32 8, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_viwdupq_m_n_u16(uint16x8_t inactive, uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_m(inactive, a, b, 8, p);
+#else /* POLYMORPHIC */
+    return viwdupq_m_n_u16(inactive, a, b, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_m_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i32 4, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_viwdupq_m_n_u32(uint32x4_t inactive, uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_m(inactive, a, b, 4, p);
+#else /* POLYMORPHIC */
+    return viwdupq_m_n_u32(inactive, a, b, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_m_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.predicated.v16i8.v16i1(<16 x i8> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i32 1, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vdwdupq_m_n_u8(uint8x16_t inactive, uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_m(inactive, a, b, 1, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_m_n_u8(inactive, a, b, 1, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_m_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.predicated.v8i16.v8i1(<8 x i16> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i32 2, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vdwdupq_m_n_u16(uint16x8_t inactive, uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_m(inactive, a, b, 2, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_m_n_u16(inactive, a, b, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_m_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i32 4, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vdwdupq_m_n_u32(uint32x4_t inactive, uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_m(inactive, a, b, 4, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_m_n_u32(inactive, a, b, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_m_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vidup.predicated.v16i8.v16i1(<16 x i8> [[INACTIVE:%.*]], i32 [[TMP0]], i32 8, <16 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP5]]
+//
+uint8x16_t test_vidupq_m_wb_u8(uint8x16_t inactive, uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_m(inactive, a, 8, p);
+#else /* POLYMORPHIC */
+    return vidupq_m_wb_u8(inactive, a, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_m_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vidup.predicated.v8i16.v8i1(<8 x i16> [[INACTIVE:%.*]], i32 [[TMP0]], i32 2, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP5]]
+//
+uint16x8_t test_vidupq_m_wb_u16(uint16x8_t inactive, uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_m(inactive, a, 2, p);
+#else /* POLYMORPHIC */
+    return vidupq_m_wb_u16(inactive, a, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_m_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vidup.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], i32 [[TMP0]], i32 8, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+//
+uint32x4_t test_vidupq_m_wb_u32(uint32x4_t inactive, uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_m(inactive, a, 8, p);
+#else /* POLYMORPHIC */
+    return vidupq_m_wb_u32(inactive, a, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_m_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vddup.predicated.v16i8.v16i1(<16 x i8> [[INACTIVE:%.*]], i32 [[TMP0]], i32 1, <16 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP5]]
+//
+uint8x16_t test_vddupq_m_wb_u8(uint8x16_t inactive, uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_m(inactive, a, 1, p);
+#else /* POLYMORPHIC */
+    return vddupq_m_wb_u8(inactive, a, 1, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_m_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vddup.predicated.v8i16.v8i1(<8 x i16> [[INACTIVE:%.*]], i32 [[TMP0]], i32 1, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP5]]
+//
+uint16x8_t test_vddupq_m_wb_u16(uint16x8_t inactive, uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_m(inactive, a, 1, p);
+#else /* POLYMORPHIC */
+    return vddupq_m_wb_u16(inactive, a, 1, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_m_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vddup.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], i32 [[TMP0]], i32 4, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+//
+uint32x4_t test_vddupq_m_wb_u32(uint32x4_t inactive, uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_m(inactive, a, 4, p);
+#else /* POLYMORPHIC */
+    return vddupq_m_wb_u32(inactive, a, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_m_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.predicated.v16i8.v16i1(<16 x i8> [[INACTIVE:%.*]], i32 [[TMP0]], i32 [[B:%.*]], i32 8, <16 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP5]]
+//
+uint8x16_t test_viwdupq_m_wb_u8(uint8x16_t inactive, uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_m(inactive, a, b, 8, p);
+#else /* POLYMORPHIC */
+    return viwdupq_m_wb_u8(inactive, a, b, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_m_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.predicated.v8i16.v8i1(<8 x i16> [[INACTIVE:%.*]], i32 [[TMP0]], i32 [[B:%.*]], i32 8, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP5]]
+//
+uint16x8_t test_viwdupq_m_wb_u16(uint16x8_t inactive, uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_m(inactive, a, b, 8, p);
+#else /* POLYMORPHIC */
+    return viwdupq_m_wb_u16(inactive, a, b, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_m_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], i32 [[TMP0]], i32 [[B:%.*]], i32 4, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+//
+uint32x4_t test_viwdupq_m_wb_u32(uint32x4_t inactive, uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_m(inactive, a, b, 4, p);
+#else /* POLYMORPHIC */
+    return viwdupq_m_wb_u32(inactive, a, b, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_m_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.predicated.v16i8.v16i1(<16 x i8> [[INACTIVE:%.*]], i32 [[TMP0]], i32 [[B:%.*]], i32 1, <16 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP5]]
+//
+uint8x16_t test_vdwdupq_m_wb_u8(uint8x16_t inactive, uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_m(inactive, a, b, 1, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_m_wb_u8(inactive, a, b, 1, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_m_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.predicated.v8i16.v8i1(<8 x i16> [[INACTIVE:%.*]], i32 [[TMP0]], i32 [[B:%.*]], i32 4, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP5]]
+//
+uint16x8_t test_vdwdupq_m_wb_u16(uint16x8_t inactive, uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_m(inactive, a, b, 4, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_m_wb_u16(inactive, a, b, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_m_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.predicated.v4i32.v4i1(<4 x i32> [[INACTIVE:%.*]], i32 [[TMP0]], i32 [[B:%.*]], i32 4, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+//
+uint32x4_t test_vdwdupq_m_wb_u32(uint32x4_t inactive, uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_m(inactive, a, b, 4, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_m_wb_u32(inactive, a, b, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_x_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vidup.predicated.v16i8.v16i1(<16 x i8> undef, i32 [[A:%.*]], i32 2, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vidupq_x_n_u8(uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_x_u8(a, 2, p);
+#else /* POLYMORPHIC */
+    return vidupq_x_n_u8(a, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_x_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vidup.predicated.v8i16.v8i1(<8 x i16> undef, i32 [[A:%.*]], i32 2, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vidupq_x_n_u16(uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_x_u16(a, 2, p);
+#else /* POLYMORPHIC */
+    return vidupq_x_n_u16(a, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_x_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vidup.predicated.v4i32.v4i1(<4 x i32> undef, i32 [[A:%.*]], i32 8, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vidupq_x_n_u32(uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_x_u32(a, 8, p);
+#else /* POLYMORPHIC */
+    return vidupq_x_n_u32(a, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_x_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vddup.predicated.v16i8.v16i1(<16 x i8> undef, i32 [[A:%.*]], i32 8, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vddupq_x_n_u8(uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_x_u8(a, 8, p);
+#else /* POLYMORPHIC */
+    return vddupq_x_n_u8(a, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_x_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vddup.predicated.v8i16.v8i1(<8 x i16> undef, i32 [[A:%.*]], i32 4, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vddupq_x_n_u16(uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_x_u16(a, 4, p);
+#else /* POLYMORPHIC */
+    return vddupq_x_n_u16(a, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_x_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vddup.predicated.v4i32.v4i1(<4 x i32> undef, i32 [[A:%.*]], i32 2, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vddupq_x_n_u32(uint32_t a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_x_u32(a, 2, p);
+#else /* POLYMORPHIC */
+    return vddupq_x_n_u32(a, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_x_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.predicated.v16i8.v16i1(<16 x i8> undef, i32 [[A:%.*]], i32 [[B:%.*]], i32 2, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_viwdupq_x_n_u8(uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_x_u8(a, b, 2, p);
+#else /* POLYMORPHIC */
+    return viwdupq_x_n_u8(a, b, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_x_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.predicated.v8i16.v8i1(<8 x i16> undef, i32 [[A:%.*]], i32 [[B:%.*]], i32 4, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_viwdupq_x_n_u16(uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_x_u16(a, b, 4, p);
+#else /* POLYMORPHIC */
+    return viwdupq_x_n_u16(a, b, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_x_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.predicated.v4i32.v4i1(<4 x i32> undef, i32 [[A:%.*]], i32 [[B:%.*]], i32 2, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_viwdupq_x_n_u32(uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_x_u32(a, b, 2, p);
+#else /* POLYMORPHIC */
+    return viwdupq_x_n_u32(a, b, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_x_n_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.predicated.v16i8.v16i1(<16 x i8> undef, i32 [[A:%.*]], i32 [[B:%.*]], i32 2, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+//
+uint8x16_t test_vdwdupq_x_n_u8(uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_x_u8(a, b, 2, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_x_n_u8(a, b, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_x_n_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.predicated.v8i16.v8i1(<8 x i16> undef, i32 [[A:%.*]], i32 [[B:%.*]], i32 2, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+//
+uint16x8_t test_vdwdupq_x_n_u16(uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_x_u16(a, b, 2, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_x_n_u16(a, b, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_x_n_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.predicated.v4i32.v4i1(<4 x i32> undef, i32 [[A:%.*]], i32 [[B:%.*]], i32 8, <4 x i1> [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP2]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+uint32x4_t test_vdwdupq_x_n_u32(uint32_t a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_x_u32(a, b, 8, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_x_n_u32(a, b, 8, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_x_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vidup.predicated.v16i8.v16i1(<16 x i8> undef, i32 [[TMP0]], i32 2, <16 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP5]]
+//
+uint8x16_t test_vidupq_x_wb_u8(uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_x_u8(a, 2, p);
+#else /* POLYMORPHIC */
+    return vidupq_x_wb_u8(a, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_x_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vidup.predicated.v8i16.v8i1(<8 x i16> undef, i32 [[TMP0]], i32 4, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP5]]
+//
+uint16x8_t test_vidupq_x_wb_u16(uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_x_u16(a, 4, p);
+#else /* POLYMORPHIC */
+    return vidupq_x_wb_u16(a, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vidupq_x_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vidup.predicated.v4i32.v4i1(<4 x i32> undef, i32 [[TMP0]], i32 2, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+//
+uint32x4_t test_vidupq_x_wb_u32(uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vidupq_x_u32(a, 2, p);
+#else /* POLYMORPHIC */
+    return vidupq_x_wb_u32(a, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_x_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vddup.predicated.v16i8.v16i1(<16 x i8> undef, i32 [[TMP0]], i32 1, <16 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP5]]
+//
+uint8x16_t test_vddupq_x_wb_u8(uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_x_u8(a, 1, p);
+#else /* POLYMORPHIC */
+    return vddupq_x_wb_u8(a, 1, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_x_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vddup.predicated.v8i16.v8i1(<8 x i16> undef, i32 [[TMP0]], i32 4, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP5]]
+//
+uint16x8_t test_vddupq_x_wb_u16(uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_x_u16(a, 4, p);
+#else /* POLYMORPHIC */
+    return vddupq_x_wb_u16(a, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vddupq_x_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vddup.predicated.v4i32.v4i1(<4 x i32> undef, i32 [[TMP0]], i32 4, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+//
+uint32x4_t test_vddupq_x_wb_u32(uint32_t *a, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vddupq_x_u32(a, 4, p);
+#else /* POLYMORPHIC */
+    return vddupq_x_wb_u32(a, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_x_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.predicated.v16i8.v16i1(<16 x i8> undef, i32 [[TMP0]], i32 [[B:%.*]], i32 1, <16 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP5]]
+//
+uint8x16_t test_viwdupq_x_wb_u8(uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_x_u8(a, b, 1, p);
+#else /* POLYMORPHIC */
+    return viwdupq_x_wb_u8(a, b, 1, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_x_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.predicated.v8i16.v8i1(<8 x i16> undef, i32 [[TMP0]], i32 [[B:%.*]], i32 2, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP5]]
+//
+uint16x8_t test_viwdupq_x_wb_u16(uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_x_u16(a, b, 2, p);
+#else /* POLYMORPHIC */
+    return viwdupq_x_wb_u16(a, b, 2, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_viwdupq_x_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.predicated.v4i32.v4i1(<4 x i32> undef, i32 [[TMP0]], i32 [[B:%.*]], i32 1, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+//
+uint32x4_t test_viwdupq_x_wb_u32(uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return viwdupq_x_u32(a, b, 1, p);
+#else /* POLYMORPHIC */
+    return viwdupq_x_wb_u32(a, b, 1, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_x_wb_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.predicated.v16i8.v16i1(<16 x i8> undef, i32 [[TMP0]], i32 [[B:%.*]], i32 4, <16 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <16 x i8>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <16 x i8> [[TMP5]]
+//
+uint8x16_t test_vdwdupq_x_wb_u8(uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_x_u8(a, b, 4, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_x_wb_u8(a, b, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_x_wb_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.predicated.v8i16.v8i1(<8 x i16> undef, i32 [[TMP0]], i32 [[B:%.*]], i32 4, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <8 x i16>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <8 x i16> [[TMP5]]
+//
+uint16x8_t test_vdwdupq_x_wb_u16(uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_x_u16(a, b, 4, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_x_wb_u16(a, b, 4, p);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vdwdupq_x_wb_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.predicated.v4i32.v4i1(<4 x i32> undef, i32 [[TMP0]], i32 [[B:%.*]], i32 4, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 1
+// CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP5]]
+//
+uint32x4_t test_vdwdupq_x_wb_u32(uint32_t *a, uint32_t b, mve_pred16_t p)
+{
+#ifdef POLYMORPHIC
+    return vdwdupq_x_u32(a, b, 4, p);
+#else /* POLYMORPHIC */
+    return vdwdupq_x_wb_u32(a, b, 4, p);
+#endif /* POLYMORPHIC */
+}

clang/test/Sema/arm-mve-immediates.c

void test_simd_vmvn(void)
     w = vmvnq_n_u32(0x00FF0000);
     w = vmvnq_n_u32(0x01000000);
     w = vmvnq_n_u32(0xFF000000);
     w = vmvnq_n_u32(0x01000001); // expected-error {{argument should be an 8-bit value shifted by a multiple of 8 bits, or in the form 0x??FF}}
     w = vmvnq_n_u32(0x01FFFFFF); // expected-error {{argument should be an 8-bit value shifted by a multiple of 8 bits, or in the form 0x??FF}}
     w = vmvnq_n_u32(0x0001FFFF); // expected-error {{argument should be an 8-bit value shifted by a multiple of 8 bits, or in the form 0x??FF}}
     w = vmvnq_n_u32(0x000001FF);
 }
+
+void test_vidup(void)
+{
+    vidupq_n_u16(0x12345678, 1);
+    vidupq_n_u16(0x12345678, 2);
+    vidupq_n_u16(0x12345678, 4);
+    vidupq_n_u16(0x12345678, 8);
+
+    vidupq_n_u16(0x12345678, 0); // expected-error {{argument value 0 is outside the valid range [1, 8]}}
+    vidupq_n_u16(0x12345678, 16); // expected-error {{argument value 16 is outside the valid range [1, 8]}}
+    vidupq_n_u16(0x12345678, -1); // expected-error {{argument value -1 is outside the valid range [1, 8]}}
+    vidupq_n_u16(0x12345678, -2); // expected-error {{argument value -2 is outside the valid range [1, 8]}}
+    vidupq_n_u16(0x12345678, -4); // expected-error {{argument value -4 is outside the valid range [1, 8]}}
+    vidupq_n_u16(0x12345678, -8); // expected-error {{argument value -8 is outside the valid range [1, 8]}}
+    vidupq_n_u16(0x12345678, 3); // expected-error {{argument should be a power of 2}}
+    vidupq_n_u16(0x12345678, 7); // expected-error {{argument should be a power of 2}}
+}

clang/utils/TableGen/MveEmitter.cpp

std::vector<Ptr> prerequisites() const {
     std::vector<Ptr> ToRet;
     if (Predecessor)
       ToRet.push_back(Predecessor);
     morePrerequisites(ToRet);
     return ToRet;
   }
 
   void setPredecessor(Ptr p) {
-    assert(!Predecessor);
-    Predecessor = p;
+    // If the user has nested one 'seq' node inside another, and this
+    // method is called on the return value of the inner 'seq' (i.e.
+    // the final item inside it), then we can't link _this_ node to p,
+    // because it already has a predecessor. Instead, walk the chain
+    // until we find the first item in the inner seq, and link that to
+    // p, so that nesting seqs has the obvious effect of linking
+    // everything together into one long sequential chain.
+    Result *r = this;
+    while (r->Predecessor)
+      r = r->Predecessor.get();
+    r->Predecessor = p;
   }
 
   // Each Result will be assigned a variable name in the output code, but not
   // all those variable names will actually be used (e.g. the return value of
   // Builder.CreateStore has void type, so nobody will want to refer to it). To
   // prevent annoying compiler warnings, we track whether each Result's
   // variable name was ever actually mentioned in subsequent statements, so
   // that it can be left out of the final generated code.

llvm/include/llvm/IR/IntrinsicsARM.td

defm int_arm_mve_vmlldava: MVEPredicated<
   llvm_anyvector_ty>;
 
 // Version with 72-bit rounded result, vrml{a,s}ldavh[a][x]
 defm int_arm_mve_vrmlldavha: MVEPredicated<
   [llvm_i32_ty, llvm_i32_ty],
   [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty,
    llvm_i32_ty, llvm_i32_ty, llvm_anyvector_ty, LLVMMatchType<0>],
   llvm_anyvector_ty>;
+
+defm int_arm_mve_vidup: MVEMXPredicated<
+   [llvm_anyvector_ty /* output */, llvm_i32_ty /* written-back base */], [],
+   [llvm_i32_ty /* base */, llvm_i32_ty /* step */],
+   LLVMMatchType<0>, llvm_anyvector_ty>;
+defm int_arm_mve_vddup: MVEMXPredicated<
+   [llvm_anyvector_ty /* output */, llvm_i32_ty /* written-back base */], [],
+   [llvm_i32_ty /* base */, llvm_i32_ty /* step */],
+   LLVMMatchType<0>, llvm_anyvector_ty>;
+defm int_arm_mve_viwdup: MVEMXPredicated<
+   [llvm_anyvector_ty /* output */, llvm_i32_ty /* written-back base */], [],
+   [llvm_i32_ty /* base */, llvm_i32_ty /* limit */, llvm_i32_ty /* step */],
+   LLVMMatchType<0>, llvm_anyvector_ty>;
+defm int_arm_mve_vdwdup: MVEMXPredicated<
+   [llvm_anyvector_ty /* output */, llvm_i32_ty /* written-back base */], [],
+   [llvm_i32_ty /* base */, llvm_i32_ty /* limit */, llvm_i32_ty /* step */],
+   LLVMMatchType<0>, llvm_anyvector_ty>;
+
 } // end TargetPrefix

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

private:
   /// SelectMVE_VLD - Select MVE interleaving load intrinsics. NumVecs
   /// should be 2 or 4. The opcode array specifies the instructions
   /// used for 8, 16 and 32-bit lane sizes respectively, and each
   /// pointer points to a set of NumVecs sub-opcodes used for the
   /// different stages (e.g. VLD20 versus VLD21) of each load family.
   void SelectMVE_VLD(SDNode *N, unsigned NumVecs,
                      const uint16_t *const *Opcodes, bool HasWriteback);
 
+  /// SelectMVE_VxDUP - Select MVE incrementing-dup instructions. Opcodes is an
+  /// array of 3 elements for the 8, 16 and 32-bit lane sizes.
+  void SelectMVE_VxDUP(SDNode *N, const uint16_t *Opcodes,
+                       bool Wrapping, bool Predicated);
+
   /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
   /// should be 1, 2, 3 or 4.  The opcode array specifies the instructions used
   /// for loading D registers.
   void SelectVLDDup(SDNode *N, bool IsIntrinsic, bool isUpdating,
                     unsigned NumVecs, const uint16_t *DOpcodes,
                     const uint16_t *QOpcodes0 = nullptr,
                     const uint16_t *QOpcodes1 = nullptr);
 
ReplaceUses(SDValue(N, i),
                 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT,
                                                SDValue(LoadInst, 0)));
   if (HasWriteback)
     ReplaceUses(SDValue(N, i++), SDValue(LoadInst, 1));
   ReplaceUses(SDValue(N, i), SDValue(LoadInst, HasWriteback ? 2 : 1));
   CurDAG->RemoveDeadNode(N);
 }
 
+void ARMDAGToDAGISel::SelectMVE_VxDUP(SDNode *N, const uint16_t *Opcodes,
+                                      bool Wrapping, bool Predicated) {
+  EVT VT = N->getValueType(0);
+  SDLoc Loc(N);
+
+  uint16_t Opcode;
+  switch (VT.getScalarSizeInBits()) {
+  case 8:
+    Opcode = Opcodes[0];
+    break;
+  case 16:
+    Opcode = Opcodes[1];
+    break;
+  case 32:
+    Opcode = Opcodes[2];
+    break;
+  default:
+    llvm_unreachable("bad vector element size in SelectMVE_VxDUP");
+  }
+
+  SmallVector<SDValue, 8> Ops;
+  unsigned OpIdx = 1;
+
+  SDValue Inactive;
+  if (Predicated)
+    Inactive = N->getOperand(OpIdx++);
+
+  Ops.push_back(N->getOperand(OpIdx++));     // base
+  if (Wrapping)
+    Ops.push_back(N->getOperand(OpIdx++));   // limit
+
+  SDValue ImmOp = N->getOperand(OpIdx++);    // step
+  int ImmValue = cast<ConstantSDNode>(ImmOp)->getZExtValue();
+  Ops.push_back(getI32Imm(ImmValue, Loc));
+
+  if (Predicated)
+    AddMVEPredicateToOps(Ops, Loc, N->getOperand(OpIdx), Inactive);
+  else
+    AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0));
+
+  CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
+}
+
 void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic,
                                    bool isUpdating, unsigned NumVecs,
                                    const uint16_t *DOpcodes,
                                    const uint16_t *QOpcodes0,
                                    const uint16_t *QOpcodes1) {
   assert(Subtarget->hasNEON());
   assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
   SDLoc dl(N);
case Intrinsic::arm_mve_vrmlldavha_predicated: {
           ARM::MVE_VRMLALDAVHxs32, ARM::MVE_VRMLALDAVHaxs32,
           ARM::MVE_VRMLSLDAVHs32,  ARM::MVE_VRMLSLDAVHas32,
           ARM::MVE_VRMLSLDAVHxs32, ARM::MVE_VRMLSLDAVHaxs32,
       };
       SelectMVE_VRMLLDAVH(N, IntNo == Intrinsic::arm_mve_vrmlldavha_predicated,
                           OpcodesS, OpcodesU);
       return;
     }
+
+    case Intrinsic::arm_mve_vidup:
+    case Intrinsic::arm_mve_vidup_predicated: {
+      static const uint16_t Opcodes[] = {
+          ARM::MVE_VIDUPu8, ARM::MVE_VIDUPu16, ARM::MVE_VIDUPu32,
+      };
+      SelectMVE_VxDUP(N, Opcodes, false,
+                      IntNo == Intrinsic::arm_mve_vidup_predicated);
+      return;
+    }
+
+    case Intrinsic::arm_mve_vddup:
+    case Intrinsic::arm_mve_vddup_predicated: {
+      static const uint16_t Opcodes[] = {
+          ARM::MVE_VDDUPu8, ARM::MVE_VDDUPu16, ARM::MVE_VDDUPu32,
+      };
+      SelectMVE_VxDUP(N, Opcodes, false,
+                      IntNo == Intrinsic::arm_mve_vddup_predicated);
+      return;
+    }
+
+    case Intrinsic::arm_mve_viwdup:
+    case Intrinsic::arm_mve_viwdup_predicated: {
+      static const uint16_t Opcodes[] = {
+          ARM::MVE_VIWDUPu8, ARM::MVE_VIWDUPu16, ARM::MVE_VIWDUPu32,
+      };
+      SelectMVE_VxDUP(N, Opcodes, true,
+                      IntNo == Intrinsic::arm_mve_viwdup_predicated);
+      return;
+    }
+
+    case Intrinsic::arm_mve_vdwdup:
+    case Intrinsic::arm_mve_vdwdup_predicated: {
+      static const uint16_t Opcodes[] = {
+          ARM::MVE_VDWDUPu8, ARM::MVE_VDWDUPu16, ARM::MVE_VDWDUPu32,
+      };
+      SelectMVE_VxDUP(N, Opcodes, true,
+                      IntNo == Intrinsic::arm_mve_vdwdup_predicated);
+      return;
+    }
     }
     break;
   }
 
   case ISD::ATOMIC_CMP_SWAP:
     SelectCMP_SWAP(N);
     return;
   }

llvm/test/CodeGen/Thumb2/mve-intrinsics/idup.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <16 x i8> @test_vidupq_n_u8(i32 %a) {
+; CHECK-LABEL: test_vidupq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vidup.u8 q0, r0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vidup.v16i8(i32 %a, i32 4)
+  %1 = extractvalue { <16 x i8>, i32 } %0, 0
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vidupq_n_u16(i32 %a) {
+; CHECK-LABEL: test_vidupq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vidup.u16 q0, r0, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vidup.v8i16(i32 %a, i32 1)
+  %1 = extractvalue { <8 x i16>, i32 } %0, 0
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vidupq_n_u32(i32 %a) {
+; CHECK-LABEL: test_vidupq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vidup.u32 q0, r0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 %a, i32 4)
+  %1 = extractvalue { <4 x i32>, i32 } %0, 0
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vddupq_n_u8(i32 %a) {
+; CHECK-LABEL: test_vddupq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vddup.u8 q0, r0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vddup.v16i8(i32 %a, i32 2)
+  %1 = extractvalue { <16 x i8>, i32 } %0, 0
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vddupq_n_u16(i32 %a) {
+; CHECK-LABEL: test_vddupq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vddup.u16 q0, r0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vddup.v8i16(i32 %a, i32 4)
+  %1 = extractvalue { <8 x i16>, i32 } %0, 0
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vddupq_n_u32(i32 %a) {
+; CHECK-LABEL: test_vddupq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vddup.u32 q0, r0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vddup.v4i32(i32 %a, i32 2)
+  %1 = extractvalue { <4 x i32>, i32 } %0, 0
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_viwdupq_n_u8(i32 %a, i32 %b) {
+; CHECK-LABEL: test_viwdupq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    viwdup.u8 q0, r0, r1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.v16i8(i32 %a, i32 %b, i32 4)
+  %1 = extractvalue { <16 x i8>, i32 } %0, 0
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_viwdupq_n_u16(i32 %a, i32 %b) {
+; CHECK-LABEL: test_viwdupq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    viwdup.u16 q0, r0, r1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.v8i16(i32 %a, i32 %b, i32 2)
+  %1 = extractvalue { <8 x i16>, i32 } %0, 0
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_viwdupq_n_u32(i32 %a, i32 %b) {
+; CHECK-LABEL: test_viwdupq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    viwdup.u32 q0, r0, r1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.v4i32(i32 %a, i32 %b, i32 8)
+  %1 = extractvalue { <4 x i32>, i32 } %0, 0
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vdwdupq_n_u8(i32 %a, i32 %b) {
+; CHECK-LABEL: test_vdwdupq_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdwdup.u8 q0, r0, r1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.v16i8(i32 %a, i32 %b, i32 4)
+  %1 = extractvalue { <16 x i8>, i32 } %0, 0
+  ret <16 x i8> %1
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vdwdupq_n_u16(i32 %a, i32 %b) {
+; CHECK-LABEL: test_vdwdupq_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdwdup.u16 q0, r0, r1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.v8i16(i32 %a, i32 %b, i32 8)
+  %1 = extractvalue { <8 x i16>, i32 } %0, 0
+  ret <8 x i16> %1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vdwdupq_n_u32(i32 %a, i32 %b) {
+; CHECK-LABEL: test_vdwdupq_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdwdup.u32 q0, r0, r1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.v4i32(i32 %a, i32 %b, i32 1)
+  %1 = extractvalue { <4 x i32>, i32 } %0, 0
+  ret <4 x i32> %1
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vidupq_wb_u8(i32* nocapture %a) {
+; CHECK-LABEL: test_vidupq_wb_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vidup.u8 q0, r2, #8
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vidup.v16i8(i32 %0, i32 8)
+  %2 = extractvalue { <16 x i8>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <16 x i8>, i32 } %1, 0
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vidupq_wb_u16(i32* nocapture %a) {
+; CHECK-LABEL: test_vidupq_wb_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vidup.u16 q0, r2, #1
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vidup.v8i16(i32 %0, i32 1)
+  %2 = extractvalue { <8 x i16>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <8 x i16>, i32 } %1, 0
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vidupq_wb_u32(i32* nocapture %a) {
+; CHECK-LABEL: test_vidupq_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vidup.u32 q0, r2, #4
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 %0, i32 4)
+  %2 = extractvalue { <4 x i32>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <4 x i32>, i32 } %1, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vddupq_wb_u8(i32* nocapture %a) {
+; CHECK-LABEL: test_vddupq_wb_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vddup.u8 q0, r2, #2
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vddup.v16i8(i32 %0, i32 2)
+  %2 = extractvalue { <16 x i8>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <16 x i8>, i32 } %1, 0
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vddupq_wb_u16(i32* nocapture %a) {
+; CHECK-LABEL: test_vddupq_wb_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vddup.u16 q0, r2, #8
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vddup.v8i16(i32 %0, i32 8)
+  %2 = extractvalue { <8 x i16>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <8 x i16>, i32 } %1, 0
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vddupq_wb_u32(i32* nocapture %a) {
+; CHECK-LABEL: test_vddupq_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vddup.u32 q0, r2, #2
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vddup.v4i32(i32 %0, i32 2)
+  %2 = extractvalue { <4 x i32>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <4 x i32>, i32 } %1, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vdwdupq_wb_u8(i32* nocapture %a, i32 %b) {
+; CHECK-LABEL: test_vdwdupq_wb_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vdwdup.u8 q0, r2, r1, #4
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.v16i8(i32 %0, i32 %b, i32 4)
+  %2 = extractvalue { <16 x i8>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <16 x i8>, i32 } %1, 0
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vdwdupq_wb_u16(i32* nocapture %a, i32 %b) {
+; CHECK-LABEL: test_vdwdupq_wb_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vdwdup.u16 q0, r2, r1, #4
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.v8i16(i32 %0, i32 %b, i32 4)
+  %2 = extractvalue { <8 x i16>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <8 x i16>, i32 } %1, 0
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_viwdupq_wb_u8(i32* nocapture %a, i32 %b) {
+; CHECK-LABEL: test_viwdupq_wb_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    viwdup.u8 q0, r2, r1, #1
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.v16i8(i32 %0, i32 %b, i32 1)
+  %2 = extractvalue { <16 x i8>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <16 x i8>, i32 } %1, 0
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_viwdupq_wb_u16(i32* nocapture %a, i32 %b) {
+; CHECK-LABEL: test_viwdupq_wb_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    viwdup.u16 q0, r2, r1, #1
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.v8i16(i32 %0, i32 %b, i32 1)
+  %2 = extractvalue { <8 x i16>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <8 x i16>, i32 } %1, 0
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_viwdupq_wb_u32(i32* nocapture %a, i32 %b) {
+; CHECK-LABEL: test_viwdupq_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    viwdup.u32 q0, r2, r1, #8
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.v4i32(i32 %0, i32 %b, i32 8)
+  %2 = extractvalue { <4 x i32>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <4 x i32>, i32 } %1, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vdwdupq_wb_u32(i32* nocapture %a, i32 %b) {
+; CHECK-LABEL: test_vdwdupq_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vdwdup.u32 q0, r2, r1, #2
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.v4i32(i32 %0, i32 %b, i32 2)
+  %2 = extractvalue { <4 x i32>, i32 } %1, 1
+  store i32 %2, i32* %a, align 4
+  %3 = extractvalue { <4 x i32>, i32 } %1, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vidupq_m_n_u8(<16 x i8> %inactive, i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vidupq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vidupt.u8 q0, r0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vidup.predicated.v16i8.v16i1(<16 x i8> %inactive, i32 %a, i32 8, <16 x i1> %1)
+  %3 = extractvalue { <16 x i8>, i32 } %2, 0
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vidupq_m_n_u16(<8 x i16> %inactive, i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vidupq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vidupt.u16 q0, r0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vidup.predicated.v8i16.v8i1(<8 x i16> %inactive, i32 %a, i32 8, <8 x i1> %1)
+  %3 = extractvalue { <8 x i16>, i32 } %2, 0
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vidupq_m_n_u32(<4 x i32> %inactive, i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vidupq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vidupt.u32 q0, r0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.predicated.v4i32.v4i1(<4 x i32> %inactive, i32 %a, i32 2, <4 x i1> %1)
+  %3 = extractvalue { <4 x i32>, i32 } %2, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vddupq_m_n_u8(<16 x i8> %inactive, i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vddupq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vddupt.u8 q0, r0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vddup.predicated.v16i8.v16i1(<16 x i8> %inactive, i32 %a, i32 8, <16 x i1> %1)
+  %3 = extractvalue { <16 x i8>, i32 } %2, 0
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vddupq_m_n_u16(<8 x i16> %inactive, i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vddupq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vddupt.u16 q0, r0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vddup.predicated.v8i16.v8i1(<8 x i16> %inactive, i32 %a, i32 2, <8 x i1> %1)
+  %3 = extractvalue { <8 x i16>, i32 } %2, 0
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vddupq_m_n_u32(<4 x i32> %inactive, i32 %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vddupq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vddupt.u32 q0, r0, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vddup.predicated.v4i32.v4i1(<4 x i32> %inactive, i32 %a, i32 8, <4 x i1> %1)
+  %3 = extractvalue { <4 x i32>, i32 } %2, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_viwdupq_m_n_u8(<16 x i8> %inactive, i32 %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_viwdupq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    viwdupt.u8 q0, r0, r1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.predicated.v16i8.v16i1(<16 x i8> %inactive, i32 %a, i32 %b, i32 8, <16 x i1> %1)
+  %3 = extractvalue { <16 x i8>, i32 } %2, 0
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_viwdupq_m_n_u16(<8 x i16> %inactive, i32 %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_viwdupq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    viwdupt.u16 q0, r0, r1, #8
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.predicated.v8i16.v8i1(<8 x i16> %inactive, i32 %a, i32 %b, i32 8, <8 x i1> %1)
+  %3 = extractvalue { <8 x i16>, i32 } %2, 0
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_viwdupq_m_n_u32(<4 x i32> %inactive, i32 %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_viwdupq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    viwdupt.u32 q0, r0, r1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.predicated.v4i32.v4i1(<4 x i32> %inactive, i32 %a, i32 %b, i32 4, <4 x i1> %1)
+  %3 = extractvalue { <4 x i32>, i32 } %2, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vdwdupq_m_n_u8(<16 x i8> %inactive, i32 %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vdwdupq_m_n_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vdwdupt.u8 q0, r0, r1, #1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.predicated.v16i8.v16i1(<16 x i8> %inactive, i32 %a, i32 %b, i32 1, <16 x i1> %1)
+  %3 = extractvalue { <16 x i8>, i32 } %2, 0
+  ret <16 x i8> %3
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vdwdupq_m_n_u16(<8 x i16> %inactive, i32 %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vdwdupq_m_n_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vdwdupt.u16 q0, r0, r1, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.predicated.v8i16.v8i1(<8 x i16> %inactive, i32 %a, i32 %b, i32 2, <8 x i1> %1)
+  %3 = extractvalue { <8 x i16>, i32 } %2, 0
+  ret <8 x i16> %3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vdwdupq_m_n_u32(<4 x i32> %inactive, i32 %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vdwdupq_m_n_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vdwdupt.u32 q0, r0, r1, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
+  %2 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.predicated.v4i32.v4i1(<4 x i32> %inactive, i32 %a, i32 %b, i32 4, <4 x i1> %1)
+  %3 = extractvalue { <4 x i32>, i32 } %2, 0
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vidupq_m_wb_u8(<16 x i8> %inactive, i32* nocapture %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vidupq_m_wb_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vidupt.u8 q0, r2, #8
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  %3 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vidup.predicated.v16i8.v16i1(<16 x i8> %inactive, i32 %0, i32 8, <16 x i1> %2)
+  %4 = extractvalue { <16 x i8>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <16 x i8>, i32 } %3, 0
+  ret <16 x i8> %5
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vidupq_m_wb_u16(<8 x i16> %inactive, i32* nocapture %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vidupq_m_wb_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vidupt.u16 q0, r2, #2
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vidup.predicated.v8i16.v8i1(<8 x i16> %inactive, i32 %0, i32 2, <8 x i1> %2)
+  %4 = extractvalue { <8 x i16>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <8 x i16>, i32 } %3, 0
+  ret <8 x i16> %5
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vidupq_m_wb_u32(<4 x i32> %inactive, i32* nocapture %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vidupq_m_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vidupt.u32 q0, r2, #8
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.predicated.v4i32.v4i1(<4 x i32> %inactive, i32 %0, i32 8, <4 x i1> %2)
+  %4 = extractvalue { <4 x i32>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <4 x i32>, i32 } %3, 0
+  ret <4 x i32> %5
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vddupq_m_wb_u8(<16 x i8> %inactive, i32* nocapture %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vddupq_m_wb_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vddupt.u8 q0, r2, #1
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  %3 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vddup.predicated.v16i8.v16i1(<16 x i8> %inactive, i32 %0, i32 1, <16 x i1> %2)
+  %4 = extractvalue { <16 x i8>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <16 x i8>, i32 } %3, 0
+  ret <16 x i8> %5
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vddupq_m_wb_u16(<8 x i16> %inactive, i32* nocapture %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vddupq_m_wb_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vddupt.u16 q0, r2, #1
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vddup.predicated.v8i16.v8i1(<8 x i16> %inactive, i32 %0, i32 1, <8 x i1> %2)
+  %4 = extractvalue { <8 x i16>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <8 x i16>, i32 } %3, 0
+  ret <8 x i16> %5
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vddupq_m_wb_u32(<4 x i32> %inactive, i32* nocapture %a, i16 zeroext %p) {
+; CHECK-LABEL: test_vddupq_m_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr r2, [r0]
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vddupt.u32 q0, r2, #4
+; CHECK-NEXT:    str r2, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vddup.predicated.v4i32.v4i1(<4 x i32> %inactive, i32 %0, i32 4, <4 x i1> %2)
+  %4 = extractvalue { <4 x i32>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <4 x i32>, i32 } %3, 0
+  ret <4 x i32> %5
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_viwdupq_m_wb_u8(<16 x i8> %inactive, i32* nocapture %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_viwdupq_m_wb_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr.w r12, [r0]
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    viwdupt.u8 q0, r12, r1, #8
+; CHECK-NEXT:    str.w r12, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  %3 = tail call { <16 x i8>, i32 } @llvm.arm.mve.viwdup.predicated.v16i8.v16i1(<16 x i8> %inactive, i32 %0, i32 %b, i32 8, <16 x i1> %2)
+  %4 = extractvalue { <16 x i8>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <16 x i8>, i32 } %3, 0
+  ret <16 x i8> %5
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_viwdupq_m_wb_u16(<8 x i16> %inactive, i32* nocapture %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_viwdupq_m_wb_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr.w r12, [r0]
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    viwdupt.u16 q0, r12, r1, #8
+; CHECK-NEXT:    str.w r12, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = tail call { <8 x i16>, i32 } @llvm.arm.mve.viwdup.predicated.v8i16.v8i1(<8 x i16> %inactive, i32 %0, i32 %b, i32 8, <8 x i1> %2)
+  %4 = extractvalue { <8 x i16>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <8 x i16>, i32 } %3, 0
+  ret <8 x i16> %5
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_viwdupq_m_wb_u32(<4 x i32> %inactive, i32* nocapture %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_viwdupq_m_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr.w r12, [r0]
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    viwdupt.u32 q0, r12, r1, #4
+; CHECK-NEXT:    str.w r12, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.viwdup.predicated.v4i32.v4i1(<4 x i32> %inactive, i32 %0, i32 %b, i32 4, <4 x i1> %2)
+  %4 = extractvalue { <4 x i32>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <4 x i32>, i32 } %3, 0
+  ret <4 x i32> %5
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vdwdupq_m_wb_u8(<16 x i8> %inactive, i32* nocapture %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vdwdupq_m_wb_u8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr.w r12, [r0]
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vdwdupt.u8 q0, r12, r1, #1
+; CHECK-NEXT:    str.w r12, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %1)
+  %3 = tail call { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.predicated.v16i8.v16i1(<16 x i8> %inactive, i32 %0, i32 %b, i32 1, <16 x i1> %2)
+  %4 = extractvalue { <16 x i8>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <16 x i8>, i32 } %3, 0
+  ret <16 x i8> %5
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test_vdwdupq_m_wb_u16(<8 x i16> %inactive, i32* nocapture %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vdwdupq_m_wb_u16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr.w r12, [r0]
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vdwdupt.u16 q0, r12, r1, #4
+; CHECK-NEXT:    str.w r12, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
+  %3 = tail call { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.predicated.v8i16.v8i1(<8 x i16> %inactive, i32 %0, i32 %b, i32 4, <8 x i1> %2)
+  %4 = extractvalue { <8 x i16>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <8 x i16>, i32 } %3, 0
+  ret <8 x i16> %5
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vdwdupq_m_wb_u32(<4 x i32> %inactive, i32* nocapture %a, i32 %b, i16 zeroext %p) {
+; CHECK-LABEL: test_vdwdupq_m_wb_u32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldr.w r12, [r0]
+; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vdwdupt.u32 q0, r12, r1, #4
+; CHECK-NEXT:    str.w r12, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load i32, i32* %a, align 4
+  %1 = zext i16 %p to i32
+  %2 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.predicated.v4i32.v4i1(<4 x i32> %inactive, i32 %0, i32 %b, i32 4, <4 x i1> %2)
+  %4 = extractvalue { <4 x i32>, i32 } %3, 1
+  store i32 %4, i32* %a, align 4
+  %5 = extractvalue { <4 x i32>, i32 } %3, 0
+  ret <4 x i32> %5
+}
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+
+declare { <16 x i8>, i32 } @llvm.arm.mve.vidup.v16i8(i32, i32)
+declare { <8 x i16>, i32 } @llvm.arm.mve.vidup.v8i16(i32, i32)
+declare { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32, i32)
+declare { <16 x i8>, i32 } @llvm.arm.mve.vddup.v16i8(i32, i32)
+declare { <8 x i16>, i32 } @llvm.arm.mve.vddup.v8i16(i32, i32)
+declare { <4 x i32>, i32 } @llvm.arm.mve.vddup.v4i32(i32, i32)
+declare { <16 x i8>, i32 } @llvm.arm.mve.viwdup.v16i8(i32, i32, i32)
+declare { <8 x i16>, i32 } @llvm.arm.mve.viwdup.v8i16(i32, i32, i32)
+declare { <4 x i32>, i32 } @llvm.arm.mve.viwdup.v4i32(i32, i32, i32)
+declare { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.v16i8(i32, i32, i32)
+declare { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.v8i16(i32, i32, i32)
+declare { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.v4i32(i32, i32, i32)
+declare { <16 x i8>, i32 } @llvm.arm.mve.vidup.predicated.v16i8.v16i1(<16 x i8>, i32, i32, <16 x i1>)
+declare { <8 x i16>, i32 } @llvm.arm.mve.vidup.predicated.v8i16.v8i1(<8 x i16>, i32, i32, <8 x i1>)
+declare { <4 x i32>, i32 } @llvm.arm.mve.vidup.predicated.v4i32.v4i1(<4 x i32>, i32, i32, <4 x i1>)
+declare { <16 x i8>, i32 } @llvm.arm.mve.vddup.predicated.v16i8.v16i1(<16 x i8>, i32, i32, <16 x i1>)
+declare { <8 x i16>, i32 } @llvm.arm.mve.vddup.predicated.v8i16.v8i1(<8 x i16>, i32, i32, <8 x i1>)
+declare { <4 x i32>, i32 } @llvm.arm.mve.vddup.predicated.v4i32.v4i1(<4 x i32>, i32, i32, <4 x i1>)
+declare { <16 x i8>, i32 } @llvm.arm.mve.viwdup.predicated.v16i8.v16i1(<16 x i8>, i32, i32, i32, <16 x i1>)
+declare { <8 x i16>, i32 } @llvm.arm.mve.viwdup.predicated.v8i16.v8i1(<8 x i16>, i32, i32, i32, <8 x i1>)
+declare { <4 x i32>, i32 } @llvm.arm.mve.viwdup.predicated.v4i32.v4i1(<4 x i32>, i32, i32, i32, <4 x i1>)
+declare { <16 x i8>, i32 } @llvm.arm.mve.vdwdup.predicated.v16i8.v16i1(<16 x i8>, i32, i32, i32, <16 x i1>)
+declare { <8 x i16>, i32 } @llvm.arm.mve.vdwdup.predicated.v8i16.v8i1(<8 x i16>, i32, i32, i32, <8 x i1>)
+declare { <4 x i32>, i32 } @llvm.arm.mve.vdwdup.predicated.v4i32.v4i1(<4 x i32>, i32, i32, i32, <4 x i1>)