[ARM] Recognize VIDUP from BUILDVECTORs of additions

[dmgreen]

Summary

This adds a pattern to recognize VIDUP from BUILD_VECTOR of incrementing adds. This can come up from either geps or adds, and came up recently in D100550. We are just looking for a BUILD_VECTOR where each lane is an add of the first lane with N*i, where i is the lane and N is one of 1, 2, 4, or 8, supported by the VIDUP instruction.

[simon_tatham]

That's a particularly striking improvement for the vectors of u8 :-)

llvm/lib/Target/ARM/ARMISelLowering.h

enum NodeType : unsigned {
     VQMOVNs, // Vector (V) Saturating (Q) Move and Narrow (N), signed (s)
     VQMOVNu, // Vector (V) Saturating (Q) Move and Narrow (N), unsigned (u)
 
     // MVE float <> half converts
     VCVTN, // MVE vcvt f32 -> f16, truncating into either the bottom or top
            // lanes
     VCVTL, // MVE vcvt f16 -> f32, extending from either the bottom or top lanes
 
+    // MVE VIDUP instruction, taking a start value and increment.
+    VIDUP,
+
     // Vector multiply long:
     VMULLs, // ...signed
     VMULLu, // ...unsigned
 
     VQDMULH, // MVE vqdmulh instruction
 
     // MVE reductions
     VADDVs,  // sign- or zero-extend the elements of a vector to i32,

llvm/lib/Target/ARM/ARMISelLowering.cpp

case ARMISD::FIRST_NUMBER:
     MAKE_CASE(ARMISD::VTRN)
     MAKE_CASE(ARMISD::VTBL1)
     MAKE_CASE(ARMISD::VTBL2)
     MAKE_CASE(ARMISD::VMOVN)
     MAKE_CASE(ARMISD::VQMOVNs)
     MAKE_CASE(ARMISD::VQMOVNu)
     MAKE_CASE(ARMISD::VCVTN)
     MAKE_CASE(ARMISD::VCVTL)
+    MAKE_CASE(ARMISD::VIDUP)
     MAKE_CASE(ARMISD::VMULLs)
     MAKE_CASE(ARMISD::VMULLu)
     MAKE_CASE(ARMISD::VQDMULH)
     MAKE_CASE(ARMISD::VADDVs)
     MAKE_CASE(ARMISD::VADDVu)
     MAKE_CASE(ARMISD::VADDVps)
     MAKE_CASE(ARMISD::VADDVpu)
     MAKE_CASE(ARMISD::VADDLVs)
if (isa<ConstantSDNode>(V) || V.isUndef())
       continue;
     Base = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Base, V,
                        DAG.getConstant(i, dl, MVT::i32));
   }
 
   return Base;
 }
 
+static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
+                                        const ARMSubtarget *ST) {
+  if (!ST->hasMVEIntegerOps())
+    return SDValue();
+
+  // We are looking for a buildvector where each element is Op[0] + i*N
+  EVT VT = Op.getValueType();
+  SDValue Op0 = Op.getOperand(0);
+  unsigned NumElts = VT.getVectorNumElements();
+
+  // Get the increment value from operand 1
+  SDValue Op1 = Op.getOperand(1);
+  if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(0) != Op0 ||
+      !isa<ConstantSDNode>(Op1.getOperand(1)))
+    return SDValue();
+  unsigned N = Op1.getConstantOperandVal(1);
+  if (N != 1 && N != 2 && N != 4 && N != 8)
+    return SDValue();
+
+  // Check that each other operand matches
+  for (int I = 2; I < NumElts; I++) {
+    SDValue OpI = Op.getOperand(I);
+    if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(0) != Op0 ||
+        !isa<ConstantSDNode>(OpI.getOperand(1)) ||
+        OpI.getConstantOperandVal(1) != I * N)
+      return SDValue();
+  }
+
+  SDLoc DL(Op);
+  return DAG.getNode(ARMISD::VIDUP, DL, DAG.getVTList(VT, MVT::i32), Op0,
+                     DAG.getConstant(N, DL, MVT::i32));
+}
+
 // If this is a case we can't handle, return null and let the default
 // expansion code take care of it.
 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
                                              const ARMSubtarget *ST) const {
   BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
   SDLoc dl(Op);
   EVT VT = Op.getValueType();
 
   if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
     return LowerBUILD_VECTOR_i1(Op, DAG, ST);
 
+  if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
+    return R;
+
   APInt SplatBits, SplatUndef;
   unsigned SplatBitSize;
   bool HasAnyUndefs;
   if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
     if (SplatUndef.isAllOnesValue())
       return DAG.getUNDEF(VT);
 
     if ((ST->hasNEON() && SplatBitSize <= 64) ||

llvm/lib/Target/ARM/ARMInstrInfo.td

 def ARMvdup      : SDNode<"ARMISD::VDUP", SDTypeProfile<1, 1, [SDTCisVec<0>]>>;
 
 // VDUPLANE can produce a quad-register result from a double-register source,
 // so the result is not constrained to match the source.
 def ARMvduplane  : SDNode<"ARMISD::VDUPLANE",
                           SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
                                                SDTCisVT<2, i32>]>>;
 
+def SDTARMVIDUP  : SDTypeProfile<2, 2, [SDTCisVec<0>, SDTCisVT<1, i32>,
+                                          SDTCisVT<2, i32>, SDTCisVT<3, i32>]>;
+def ARMvidup    : SDNode<"ARMISD::VIDUP", SDTARMVIDUP>;
+
 def SDTARMVSHUF   : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0, 1>]>;
 def ARMvrev64    : SDNode<"ARMISD::VREV64", SDTARMVSHUF>;
 def ARMvrev32    : SDNode<"ARMISD::VREV32", SDTARMVSHUF>;
 def ARMvrev16    : SDNode<"ARMISD::VREV16", SDTARMVSHUF>;
 
 def SDTARMVGETLN  : SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisVec<1>,
                                          SDTCisVT<2, i32>]>;
 def ARMvgetlaneu : SDNode<"ARMISD::VGETLANEu", SDTARMVGETLN>;

llvm/lib/Target/ARM/ARMInstrMVE.td

 }
 
 defm MVE_VQDMLAH_qr   : MVE_VQDMLAH_qr_types<"vqdmlah",   0b1, 0b0>;
 defm MVE_VQRDMLAH_qr  : MVE_VQDMLAH_qr_types<"vqrdmlah",  0b0, 0b0>;
 defm MVE_VQDMLASH_qr  : MVE_VQDMLAH_qr_types<"vqdmlash",  0b1, 0b1>;
 defm MVE_VQRDMLASH_qr : MVE_VQDMLAH_qr_types<"vqrdmlash", 0b0, 0b1>;
 
 class MVE_VxDUP<string iname, string suffix, bits<2> size, bit bit_12,
-              list<dag> pattern=[]>
+                ValueType VT, SDPatternOperator vxdup>
   : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),
           (ins tGPREven:$Rn_src, MVE_VIDUP_imm:$imm), NoItinerary,
           iname, suffix, "$Qd, $Rn, $imm", vpred_r, "$Rn = $Rn_src",
-          pattern> {
+          [(set (VT MQPR:$Qd), (i32 tGPREven:$Rn),
+              (vxdup (i32 tGPREven:$Rn_src), (i32 imm:$imm)))]> {
   bits<4> Qd;
   bits<4> Rn;
   bits<2> imm;
 
   let Inst{28} = 0b0;
   let Inst{25-23} = 0b100;
   let Inst{22} = Qd{3};
   let Inst{21-20} = size;
   let Inst{19-17} = Rn{3-1};
   let Inst{16} = 0b1;
   let Inst{15-13} = Qd{2-0};
   let Inst{12} = bit_12;
   let Inst{11-8} = 0b1111;
   let Inst{7} = imm{1};
   let Inst{6-1} = 0b110111;
   let Inst{0} = imm{0};
   let validForTailPredication = 1;
   let hasSideEffects = 0;
 }
 
-def MVE_VIDUPu8  : MVE_VxDUP<"vidup", "u8",  0b00, 0b0>;
-def MVE_VIDUPu16 : MVE_VxDUP<"vidup", "u16", 0b01, 0b0>;
-def MVE_VIDUPu32 : MVE_VxDUP<"vidup", "u32", 0b10, 0b0>;
-
-def MVE_VDDUPu8  : MVE_VxDUP<"vddup", "u8",  0b00, 0b1>;
-def MVE_VDDUPu16 : MVE_VxDUP<"vddup", "u16", 0b01, 0b1>;
-def MVE_VDDUPu32 : MVE_VxDUP<"vddup", "u32", 0b10, 0b1>;
+def MVE_VIDUPu8  : MVE_VxDUP<"vidup", "u8",  0b00, 0b0, v16i8, ARMvidup>;
+def MVE_VIDUPu16 : MVE_VxDUP<"vidup", "u16", 0b01, 0b0, v8i16, ARMvidup>;
+def MVE_VIDUPu32 : MVE_VxDUP<"vidup", "u32", 0b10, 0b0, v4i32, ARMvidup>;
+
+def MVE_VDDUPu8  : MVE_VxDUP<"vddup", "u8",  0b00, 0b1, v16i8, null_frag>;
+def MVE_VDDUPu16 : MVE_VxDUP<"vddup", "u16", 0b01, 0b1, v8i16, null_frag>;
+def MVE_VDDUPu32 : MVE_VxDUP<"vddup", "u32", 0b10, 0b1, v4i32, null_frag>;
 
 class MVE_VxWDUP<string iname, string suffix, bits<2> size, bit bit_12,
                  list<dag> pattern=[]>
   : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),
           (ins tGPREven:$Rn_src, tGPROdd:$Rm, MVE_VIDUP_imm:$imm), NoItinerary,
           iname, suffix, "$Qd, $Rn, $Rm, $imm", vpred_r, "$Rn = $Rn_src",
           pattern> {
   bits<4> Qd;

llvm/test/CodeGen/Thumb2/mve-tailpred-loopinvariant.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
 
 ; This test has an instruction that gets sunk into the loop, that is a
 ; active.lane.mask operand. (%exitcount.ptrcnt.to.int = ptrtoint). We
 ; need to make sure it is loop invariant.
 
 define i32 @a(i32* readnone %b, i8* %c) {
 ; CHECK-LABEL: a:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    it ls
 ; CHECK-NEXT:    popls {r4, pc}
 ; CHECK-NEXT:  .LBB0_1: @ %while.body.preheader
-; CHECK-NEXT:    subs r0, r0, r1
-; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    add.w r2, r0, #15
-; CHECK-NEXT:    mov r12, r1
-; CHECK-NEXT:    bic r2, r2, #15
-; CHECK-NEXT:    subs r2, #16
-; CHECK-NEXT:    add.w r3, r3, r2, lsr #4
+; CHECK-NEXT:    subs r4, r0, r1
 ; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    dls lr, r3
+; CHECK-NEXT:    mov r3, r1
+; CHECK-NEXT:    dlstp.8 lr, r4
 ; CHECK-NEXT:  .LBB0_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    adds r3, r1, r2
-; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vmov.8 q0[0], r3
-; CHECK-NEXT:    adds r4, r3, #1
-; CHECK-NEXT:    vmov.8 q0[1], r4
-; CHECK-NEXT:    adds r4, r3, #2
-; CHECK-NEXT:    vmov.8 q0[2], r4
-; CHECK-NEXT:    adds r4, r3, #3
-; CHECK-NEXT:    vmov.8 q0[3], r4
-; CHECK-NEXT:    adds r4, r3, #4
-; CHECK-NEXT:    vmov.8 q0[4], r4
-; CHECK-NEXT:    adds r4, r3, #5
-; CHECK-NEXT:    vmov.8 q0[5], r4
-; CHECK-NEXT:    adds r4, r3, #6
-; CHECK-NEXT:    vmov.8 q0[6], r4
-; CHECK-NEXT:    adds r4, r3, #7
-; CHECK-NEXT:    vmov.8 q0[7], r4
-; CHECK-NEXT:    add.w r4, r3, #8
-; CHECK-NEXT:    vmov.8 q0[8], r4
-; CHECK-NEXT:    add.w r4, r3, #9
-; CHECK-NEXT:    vmov.8 q0[9], r4
-; CHECK-NEXT:    add.w r4, r3, #10
-; CHECK-NEXT:    vmov.8 q0[10], r4
-; CHECK-NEXT:    add.w r4, r3, #11
-; CHECK-NEXT:    vmov.8 q0[11], r4
-; CHECK-NEXT:    add.w r4, r3, #12
-; CHECK-NEXT:    vmov.8 q0[12], r4
-; CHECK-NEXT:    add.w r4, r3, #13
-; CHECK-NEXT:    vmov.8 q0[13], r4
-; CHECK-NEXT:    add.w r4, r3, #14
+; CHECK-NEXT:    adds r0, r1, r2
 ; CHECK-NEXT:    adds r2, #16
-; CHECK-NEXT:    subs r0, #16
-; CHECK-NEXT:    vmov.8 q0[14], r4
-; CHECK-NEXT:    adds r3, #15
-; CHECK-NEXT:    vmov.8 q0[15], r3
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.8 q0, [r12], #16
-; CHECK-NEXT:    le lr, .LBB0_2
+; CHECK-NEXT:    vidup.u8 q0, r0, #1
+; CHECK-NEXT:    vstrb.8 q0, [r3], #16
+; CHECK-NEXT:    letp lr, .LBB0_2
 ; CHECK-NEXT:  @ %bb.3: @ %while.end
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = bitcast i32* %b to i8*
   %cmp3 = icmp ugt i8* %0, %c
   br i1 %cmp3, label %while.body.preheader, label %while.end
 
 while.body.preheader:                             ; preds = %entry

llvm/test/CodeGen/Thumb2/mve-vidup.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc <4 x i32> @vidup_v4i32_1(i32 %index) {
 ; CHECK-LABEL: vidup_v4i32_1:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    adds r1, r0, #2
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r1
-; CHECK-NEXT:    adds r1, r0, #3
-; CHECK-NEXT:    adds r0, #1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r1
+; CHECK-NEXT:    vidup.u32 q0, r0, #1
 ; CHECK-NEXT:    bx lr
   %a1 = add i32 %index, 1
   %a2 = add i32 %index, 2
   %a3 = add i32 %index, 3
   %a4 = add i32 %index, 4
   %v1  = insertelement <4 x i32> poison, i32 %index, i32 0
   %v2  = insertelement <4 x i32> %v1, i32 %a1, i32 1
   %v3  = insertelement <4 x i32> %v2, i32 %a2, i32 2
   %v4  = insertelement <4 x i32> %v3, i32 %a3, i32 3
   ret <4 x i32> %v4
 }
 
 define arm_aapcs_vfpcc <4 x i32> @vidup_v4i32_2(i32 %index) {
 ; CHECK-LABEL: vidup_v4i32_2:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    adds r1, r0, #4
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r1
-; CHECK-NEXT:    adds r1, r0, #6
-; CHECK-NEXT:    adds r0, #2
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r1
+; CHECK-NEXT:    vidup.u32 q0, r0, #2
 ; CHECK-NEXT:    bx lr
   %a1 = add i32 %index, 2
   %a2 = add i32 %index, 4
   %a3 = add i32 %index, 6
   %a4 = add i32 %index, 8
   %v1  = insertelement <4 x i32> poison, i32 %index, i32 0
   %v2  = insertelement <4 x i32> %v1, i32 %a1, i32 1
   %v3  = insertelement <4 x i32> %v2, i32 %a2, i32 2
; CHECK-NEXT:    bx lr
   %v3  = insertelement <4 x i32> %v2, i32 %a2, i32 2
   %v4  = insertelement <4 x i32> %v3, i32 %a3, i32 3
   ret <4 x i32> %v4
 }
 
 define arm_aapcs_vfpcc <4 x i32> @vidup_v4i32_4(i32 %index) {
 ; CHECK-LABEL: vidup_v4i32_4:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    add.w r1, r0, #8
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r1
-; CHECK-NEXT:    add.w r1, r0, #12
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r1
+; CHECK-NEXT:    vidup.u32 q0, r0, #4
 ; CHECK-NEXT:    bx lr
   %a1 = add i32 %index, 4
   %a2 = add i32 %index, 8
   %a3 = add i32 %index, 12
   %a4 = add i32 %index, 16
   %v1  = insertelement <4 x i32> poison, i32 %index, i32 0
   %v2  = insertelement <4 x i32> %v1, i32 %a1, i32 1
   %v3  = insertelement <4 x i32> %v2, i32 %a2, i32 2
   %v4  = insertelement <4 x i32> %v3, i32 %a3, i32 3
   ret <4 x i32> %v4
 }
 
 define arm_aapcs_vfpcc <4 x i32> @vidup_v4i32_8(i32 %index) {
 ; CHECK-LABEL: vidup_v4i32_8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    add.w r1, r0, #16
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r1
-; CHECK-NEXT:    add.w r1, r0, #24
-; CHECK-NEXT:    adds r0, #8
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r1
+; CHECK-NEXT:    vidup.u32 q0, r0, #8
 ; CHECK-NEXT:    bx lr
   %a1 = add i32 %index, 8
   %a2 = add i32 %index, 16
   %a3 = add i32 %index, 24
   %a4 = add i32 %index, 32
   %v1  = insertelement <4 x i32> poison, i32 %index, i32 0
   %v2  = insertelement <4 x i32> %v1, i32 %a1, i32 1
   %v3  = insertelement <4 x i32> %v2, i32 %a2, i32 2
; CHECK-NEXT:    bx lr
   %v3  = insertelement <4 x i32> %v2, i32 %a3, i32 2
   %v4  = insertelement <4 x i32> %v3, i32 %a4, i32 3
   ret <4 x i32> %v4
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vidup_v8i16_1(i16 %index) {
 ; CHECK-LABEL: vidup_v8i16_1:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    adds r1, r0, #1
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    adds r1, r0, #2
-; CHECK-NEXT:    vmov.16 q0[2], r1
-; CHECK-NEXT:    adds r1, r0, #3
-; CHECK-NEXT:    vmov.16 q0[3], r1
-; CHECK-NEXT:    adds r1, r0, #4
-; CHECK-NEXT:    vmov.16 q0[4], r1
-; CHECK-NEXT:    adds r1, r0, #5
-; CHECK-NEXT:    vmov.16 q0[5], r1
-; CHECK-NEXT:    adds r1, r0, #6
-; CHECK-NEXT:    vmov.16 q0[6], r1
-; CHECK-NEXT:    adds r0, #7
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vidup.u16 q0, r0, #1
 ; CHECK-NEXT:    bx lr
   %a1 = add i16 %index, 1
   %a2 = add i16 %index, 2
   %a3 = add i16 %index, 3
   %a4 = add i16 %index, 4
   %a5 = add i16 %index, 5
   %a6 = add i16 %index, 6
   %a7 = add i16 %index, 7
   %a8 = add i16 %index, 8
   %v1  = insertelement <8 x i16> poison, i16 %index, i32 0
   %v2  = insertelement <8 x i16> %v1, i16 %a1, i32 1
   %v3  = insertelement <8 x i16> %v2, i16 %a2, i32 2
   %v4  = insertelement <8 x i16> %v3, i16 %a3, i32 3
   %v5  = insertelement <8 x i16> %v4, i16 %a4, i32 4
   %v6  = insertelement <8 x i16> %v5, i16 %a5, i32 5
   %v7  = insertelement <8 x i16> %v6, i16 %a6, i32 6
   %v8  = insertelement <8 x i16> %v7, i16 %a7, i32 7
   ret <8 x i16> %v8
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vidup_v8i16_2(i16 %index) {
 ; CHECK-LABEL: vidup_v8i16_2:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    adds r1, r0, #2
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    adds r1, r0, #4
-; CHECK-NEXT:    vmov.16 q0[2], r1
-; CHECK-NEXT:    adds r1, r0, #6
-; CHECK-NEXT:    vmov.16 q0[3], r1
-; CHECK-NEXT:    add.w r1, r0, #8
-; CHECK-NEXT:    vmov.16 q0[4], r1
-; CHECK-NEXT:    add.w r1, r0, #10
-; CHECK-NEXT:    vmov.16 q0[5], r1
-; CHECK-NEXT:    add.w r1, r0, #12
-; CHECK-NEXT:    vmov.16 q0[6], r1
-; CHECK-NEXT:    adds r0, #14
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vidup.u16 q0, r0, #2
 ; CHECK-NEXT:    bx lr
   %a1 = add i16 %index, 2
   %a2 = add i16 %index, 4
   %a3 = add i16 %index, 6
   %a4 = add i16 %index, 8
   %a5 = add i16 %index, 10
   %a6 = add i16 %index, 12
   %a7 = add i16 %index, 14
   %a8 = add i16 %index, 16
   %v1  = insertelement <8 x i16> poison, i16 %index, i32 0
   %v2  = insertelement <8 x i16> %v1, i16 %a1, i32 1
   %v3  = insertelement <8 x i16> %v2, i16 %a2, i32 2
   %v4  = insertelement <8 x i16> %v3, i16 %a3, i32 3
   %v5  = insertelement <8 x i16> %v4, i16 %a4, i32 4
   %v6  = insertelement <8 x i16> %v5, i16 %a5, i32 5
   %v7  = insertelement <8 x i16> %v6, i16 %a6, i32 6
   %v8  = insertelement <8 x i16> %v7, i16 %a7, i32 7
   ret <8 x i16> %v8
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vidup_v8i16_4(i16 %index) {
 ; CHECK-LABEL: vidup_v8i16_4:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    adds r1, r0, #4
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    add.w r1, r0, #8
-; CHECK-NEXT:    vmov.16 q0[2], r1
-; CHECK-NEXT:    add.w r1, r0, #12
-; CHECK-NEXT:    vmov.16 q0[3], r1
-; CHECK-NEXT:    add.w r1, r0, #16
-; CHECK-NEXT:    vmov.16 q0[4], r1
-; CHECK-NEXT:    add.w r1, r0, #20
-; CHECK-NEXT:    vmov.16 q0[5], r1
-; CHECK-NEXT:    add.w r1, r0, #24
-; CHECK-NEXT:    vmov.16 q0[6], r1
-; CHECK-NEXT:    adds r0, #28
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vidup.u16 q0, r0, #4
 ; CHECK-NEXT:    bx lr
   %a1 = add i16 %index, 4
   %a2 = add i16 %index, 8
   %a3 = add i16 %index, 12
   %a4 = add i16 %index, 16
   %a5 = add i16 %index, 20
   %a6 = add i16 %index, 24
   %a7 = add i16 %index, 28
   %a8 = add i16 %index, 32
   %v1  = insertelement <8 x i16> poison, i16 %index, i32 0
   %v2  = insertelement <8 x i16> %v1, i16 %a1, i32 1
   %v3  = insertelement <8 x i16> %v2, i16 %a2, i32 2
   %v4  = insertelement <8 x i16> %v3, i16 %a3, i32 3
   %v5  = insertelement <8 x i16> %v4, i16 %a4, i32 4
   %v6  = insertelement <8 x i16> %v5, i16 %a5, i32 5
   %v7  = insertelement <8 x i16> %v6, i16 %a6, i32 6
   %v8  = insertelement <8 x i16> %v7, i16 %a7, i32 7
   ret <8 x i16> %v8
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vidup_v8i16_8(i16 %index) {
 ; CHECK-LABEL: vidup_v8i16_8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    add.w r1, r0, #8
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    add.w r1, r0, #16
-; CHECK-NEXT:    vmov.16 q0[2], r1
-; CHECK-NEXT:    add.w r1, r0, #24
-; CHECK-NEXT:    vmov.16 q0[3], r1
-; CHECK-NEXT:    add.w r1, r0, #32
-; CHECK-NEXT:    vmov.16 q0[4], r1
-; CHECK-NEXT:    add.w r1, r0, #40
-; CHECK-NEXT:    vmov.16 q0[5], r1
-; CHECK-NEXT:    add.w r1, r0, #48
-; CHECK-NEXT:    vmov.16 q0[6], r1
-; CHECK-NEXT:    adds r0, #56
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vidup.u16 q0, r0, #8
 ; CHECK-NEXT:    bx lr
   %a1 = add i16 %index, 8
   %a2 = add i16 %index, 16
   %a3 = add i16 %index, 24
   %a4 = add i16 %index, 32
   %a5 = add i16 %index, 40
   %a6 = add i16 %index, 48
   %a7 = add i16 %index, 56
   %a8 = add i16 %index, 64
   %v1  = insertelement <8 x i16> poison, i16 %index, i32 0
   %v2  = insertelement <8 x i16> %v1, i16 %a1, i32 1
   %v3  = insertelement <8 x i16> %v2, i16 %a2, i32 2
   %v4  = insertelement <8 x i16> %v3, i16 %a3, i32 3
   %v5  = insertelement <8 x i16> %v4, i16 %a4, i32 4
   %v6  = insertelement <8 x i16> %v5, i16 %a5, i32 5
   %v7  = insertelement <8 x i16> %v6, i16 %a6, i32 6
   %v8  = insertelement <8 x i16> %v7, i16 %a7, i32 7
   ret <8 x i16> %v8
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vidup_v16i8_1(i8 %index) {
 ; CHECK-LABEL: vidup_v16i8_1:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.8 q0[0], r0
-; CHECK-NEXT:    adds r1, r0, #1
-; CHECK-NEXT:    vmov.8 q0[1], r1
-; CHECK-NEXT:    adds r1, r0, #2
-; CHECK-NEXT:    vmov.8 q0[2], r1
-; CHECK-NEXT:    adds r1, r0, #3
-; CHECK-NEXT:    vmov.8 q0[3], r1
-; CHECK-NEXT:    adds r1, r0, #4
-; CHECK-NEXT:    vmov.8 q0[4], r1
-; CHECK-NEXT:    adds r1, r0, #5
-; CHECK-NEXT:    vmov.8 q0[5], r1
-; CHECK-NEXT:    adds r1, r0, #6
-; CHECK-NEXT:    vmov.8 q0[6], r1
-; CHECK-NEXT:    adds r1, r0, #7
-; CHECK-NEXT:    vmov.8 q0[7], r1
-; CHECK-NEXT:    add.w r1, r0, #8
-; CHECK-NEXT:    vmov.8 q0[8], r1
-; CHECK-NEXT:    add.w r1, r0, #9
-; CHECK-NEXT:    vmov.8 q0[9], r1
-; CHECK-NEXT:    add.w r1, r0, #10
-; CHECK-NEXT:    vmov.8 q0[10], r1
-; CHECK-NEXT:    add.w r1, r0, #11
-; CHECK-NEXT:    vmov.8 q0[11], r1
-; CHECK-NEXT:    add.w r1, r0, #12
-; CHECK-NEXT:    vmov.8 q0[12], r1
-; CHECK-NEXT:    add.w r1, r0, #13
-; CHECK-NEXT:    vmov.8 q0[13], r1
-; CHECK-NEXT:    add.w r1, r0, #14
-; CHECK-NEXT:    vmov.8 q0[14], r1
-; CHECK-NEXT:    adds r0, #15
-; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vidup.u8 q0, r0, #1
 ; CHECK-NEXT:    bx lr
   %a1 = add i8 %index, 1
   %a2 = add i8 %index, 2
   %a3 = add i8 %index, 3
   %a4 = add i8 %index, 4
   %a5 = add i8 %index, 5
   %a6 = add i8 %index, 6
   %a7 = add i8 %index, 7
; CHECK-NEXT:    bx lr
   %v15 = insertelement <16 x i8> %v14, i8 %a14, i32 14
   %v16 = insertelement <16 x i8> %v15, i8 %a15, i32 15
   ret <16 x i8> %v16
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vidup_v16i8_4(i8 %index) {
 ; CHECK-LABEL: vidup_v16i8_4:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.8 q0[0], r0
-; CHECK-NEXT:    adds r1, r0, #4
-; CHECK-NEXT:    vmov.8 q0[1], r1
-; CHECK-NEXT:    add.w r1, r0, #8
-; CHECK-NEXT:    vmov.8 q0[2], r1
-; CHECK-NEXT:    add.w r1, r0, #12
-; CHECK-NEXT:    vmov.8 q0[3], r1
-; CHECK-NEXT:    add.w r1, r0, #16
-; CHECK-NEXT:    vmov.8 q0[4], r1
-; CHECK-NEXT:    add.w r1, r0, #20
-; CHECK-NEXT:    vmov.8 q0[5], r1
-; CHECK-NEXT:    add.w r1, r0, #24
-; CHECK-NEXT:    vmov.8 q0[6], r1
-; CHECK-NEXT:    add.w r1, r0, #28
-; CHECK-NEXT:    vmov.8 q0[7], r1
-; CHECK-NEXT:    add.w r1, r0, #32
-; CHECK-NEXT:    vmov.8 q0[8], r1
-; CHECK-NEXT:    add.w r1, r0, #36
-; CHECK-NEXT:    vmov.8 q0[9], r1
-; CHECK-NEXT:    add.w r1, r0, #40
-; CHECK-NEXT:    vmov.8 q0[10], r1
-; CHECK-NEXT:    add.w r1, r0, #44
-; CHECK-NEXT:    vmov.8 q0[11], r1
-; CHECK-NEXT:    add.w r1, r0, #48
-; CHECK-NEXT:    vmov.8 q0[12], r1
-; CHECK-NEXT:    add.w r1, r0, #52
-; CHECK-NEXT:    vmov.8 q0[13], r1
-; CHECK-NEXT:    add.w r1, r0, #56
-; CHECK-NEXT:    vmov.8 q0[14], r1
-; CHECK-NEXT:    adds r0, #60
-; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vidup.u8 q0, r0, #4
 ; CHECK-NEXT:    bx lr
   %a1 = add i8 %index, 4
   %a2 = add i8 %index, 8
   %a3 = add i8 %index, 12
   %a4 = add i8 %index, 16
   %a5 = add i8 %index, 20
   %a6 = add i8 %index, 24
   %a7 = add i8 %index, 28