[ARM][NEON] Combine base address updates for vld1Ndup intrinsics

[dmgreen]

I got confused by the _fixed vs _register vs _UPD for a while, but I see that's pre-existing :)

Nice patch. Thanks for doing it. LGTM

llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp

582

Feel free to format this as it suggests. And maybe make a function/lambda for "is VLD2 DUP" too, if it's used below too.

[krisb]

Apply review comments.

[krisb]

In D103836#2807426, @dmgreen wrote:

Nice patch. Thanks for doing it. LGTM

Thank you for reviewing this!

llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp

 //===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains a pass that expands pseudo instructions into target
struct NEONLdStTableEntry {
     friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned PseudoOpc,
                                                 const NEONLdStTableEntry &TE) {
       return PseudoOpc < TE.PseudoOpc;
     }
   };
 }
 
 static const NEONLdStTableEntry NEONLdStTable[] = {
 { ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
 { ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
 { ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
 { ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
 { ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
 { ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
 
 { ARM::VLD1d16QPseudo,      ARM::VLD1d16Q,     true,  false, false, SingleSpc,  4, 4 ,false},
 { ARM::VLD1d16QPseudoWB_fixed,  ARM::VLD1d16Qwb_fixed,   true, true, false, SingleSpc,  4, 4 ,false},

 { ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1d8Qwb_fixed,   true,  true, true, SingleHighQSpc,  4, 8 ,false},
 { ARM::VLD1q8HighTPseudo,   ARM::VLD1d8T,     true,  false, false, SingleHighTSpc,  3, 8 ,false},
 { ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1d8Twb_fixed,   true,  true, true, SingleHighTSpc,  3, 8 ,false},
 { ARM::VLD1q8LowQPseudo_UPD,  ARM::VLD1d8Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 8 ,false},
 { ARM::VLD1q8LowTPseudo_UPD,  ARM::VLD1d8Twb_fixed,   true,  true, true, SingleLowSpc,  3, 8 ,false},
 
 { ARM::VLD2DUPq16EvenPseudo,  ARM::VLD2DUPd16x2,  true, false, false, EvenDblSpc, 2, 4 ,false},
 { ARM::VLD2DUPq16OddPseudo,   ARM::VLD2DUPd16x2,  true, false, false, OddDblSpc,  2, 4 ,false},
+{ ARM::VLD2DUPq16OddPseudoWB_fixed,   ARM::VLD2DUPd16x2wb_fixed, true, true, false, OddDblSpc,  2, 4 ,false},
+{ ARM::VLD2DUPq16OddPseudoWB_register,   ARM::VLD2DUPd16x2wb_register, true, true, true, OddDblSpc,  2, 4 ,false},
 { ARM::VLD2DUPq32EvenPseudo,  ARM::VLD2DUPd32x2,  true, false, false, EvenDblSpc, 2, 2 ,false},
 { ARM::VLD2DUPq32OddPseudo,   ARM::VLD2DUPd32x2,  true, false, false, OddDblSpc,  2, 2 ,false},
+{ ARM::VLD2DUPq32OddPseudoWB_fixed,   ARM::VLD2DUPd32x2wb_fixed, true, true, false, OddDblSpc,  2, 2 ,false},
+{ ARM::VLD2DUPq32OddPseudoWB_register,   ARM::VLD2DUPd32x2wb_register, true, true, true, OddDblSpc,  2, 2 ,false},
 { ARM::VLD2DUPq8EvenPseudo,   ARM::VLD2DUPd8x2,   true, false, false, EvenDblSpc, 2, 8 ,false},
 { ARM::VLD2DUPq8OddPseudo,    ARM::VLD2DUPd8x2,   true, false, false, OddDblSpc,  2, 8 ,false},
+{ ARM::VLD2DUPq8OddPseudoWB_fixed,    ARM::VLD2DUPd8x2wb_fixed, true, true, false, OddDblSpc,  2, 8 ,false},
+{ ARM::VLD2DUPq8OddPseudoWB_register,    ARM::VLD2DUPd8x2wb_register, true, true, true, OddDblSpc,  2, 8 ,false},
 
 { ARM::VLD2LNd16Pseudo,     ARM::VLD2LNd16,     true, false, false, SingleSpc,  2, 4 ,true},
 { ARM::VLD2LNd16Pseudo_UPD, ARM::VLD2LNd16_UPD, true, true, true,  SingleSpc,  2, 4 ,true},
 { ARM::VLD2LNd32Pseudo,     ARM::VLD2LNd32,     true, false, false, SingleSpc,  2, 2 ,true},
 { ARM::VLD2LNd32Pseudo_UPD, ARM::VLD2LNd32_UPD, true, true, true,  SingleSpc,  2, 2 ,true},
 { ARM::VLD2LNd8Pseudo,      ARM::VLD2LNd8,      true, false, false, SingleSpc,  2, 8 ,true},
 { ARM::VLD2LNd8Pseudo_UPD,  ARM::VLD2LNd8_UPD, true, true, true,  SingleSpc,  2, 8 ,true},
 { ARM::VLD2LNq16Pseudo,     ARM::VLD2LNq16,     true, false, false, EvenDblSpc, 2, 4 ,true},

 { ARM::VLD3DUPd16Pseudo,     ARM::VLD3DUPd16,     true, false, false, SingleSpc, 3, 4,true},
 { ARM::VLD3DUPd16Pseudo_UPD, ARM::VLD3DUPd16_UPD, true, true, true,  SingleSpc, 3, 4,true},
 { ARM::VLD3DUPd32Pseudo,     ARM::VLD3DUPd32,     true, false, false, SingleSpc, 3, 2,true},
 { ARM::VLD3DUPd32Pseudo_UPD, ARM::VLD3DUPd32_UPD, true, true, true,  SingleSpc, 3, 2,true},
 { ARM::VLD3DUPd8Pseudo,      ARM::VLD3DUPd8,      true, false, false, SingleSpc, 3, 8,true},
 { ARM::VLD3DUPd8Pseudo_UPD,  ARM::VLD3DUPd8_UPD, true, true, true,  SingleSpc, 3, 8,true},
 { ARM::VLD3DUPq16EvenPseudo, ARM::VLD3DUPq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
 { ARM::VLD3DUPq16OddPseudo,  ARM::VLD3DUPq16,     true, false, false, OddDblSpc,  3, 4 ,true},
+{ ARM::VLD3DUPq16OddPseudo_UPD,  ARM::VLD3DUPq16_UPD, true, true, true, OddDblSpc,  3, 4 ,true},
 { ARM::VLD3DUPq32EvenPseudo, ARM::VLD3DUPq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
 { ARM::VLD3DUPq32OddPseudo,  ARM::VLD3DUPq32,     true, false, false, OddDblSpc,  3, 2 ,true},
+{ ARM::VLD3DUPq32OddPseudo_UPD,  ARM::VLD3DUPq32_UPD, true, true, true, OddDblSpc,  3, 2 ,true},
 { ARM::VLD3DUPq8EvenPseudo,  ARM::VLD3DUPq8,      true, false, false, EvenDblSpc, 3, 8 ,true},
 { ARM::VLD3DUPq8OddPseudo,   ARM::VLD3DUPq8,      true, false, false, OddDblSpc,  3, 8 ,true},
+{ ARM::VLD3DUPq8OddPseudo_UPD,   ARM::VLD3DUPq8_UPD, true, true, true, OddDblSpc,  3, 8 ,true},
 
 { ARM::VLD3LNd16Pseudo,     ARM::VLD3LNd16,     true, false, false, SingleSpc,  3, 4 ,true},
 { ARM::VLD3LNd16Pseudo_UPD, ARM::VLD3LNd16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
 { ARM::VLD3LNd32Pseudo,     ARM::VLD3LNd32,     true, false, false, SingleSpc,  3, 2 ,true},
 { ARM::VLD3LNd32Pseudo_UPD, ARM::VLD3LNd32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
 { ARM::VLD3LNd8Pseudo,      ARM::VLD3LNd8,      true, false, false, SingleSpc,  3, 8 ,true},
 { ARM::VLD3LNd8Pseudo_UPD,  ARM::VLD3LNd8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
 { ARM::VLD3LNq16Pseudo,     ARM::VLD3LNq16,     true, false, false, EvenDblSpc, 3, 4 ,true},

 { ARM::VLD4DUPd16Pseudo,     ARM::VLD4DUPd16,     true, false, false, SingleSpc, 4, 4,true},
 { ARM::VLD4DUPd16Pseudo_UPD, ARM::VLD4DUPd16_UPD, true, true, true,  SingleSpc, 4, 4,true},
 { ARM::VLD4DUPd32Pseudo,     ARM::VLD4DUPd32,     true, false, false, SingleSpc, 4, 2,true},
 { ARM::VLD4DUPd32Pseudo_UPD, ARM::VLD4DUPd32_UPD, true, true, true,  SingleSpc, 4, 2,true},
 { ARM::VLD4DUPd8Pseudo,      ARM::VLD4DUPd8,      true, false, false, SingleSpc, 4, 8,true},
 { ARM::VLD4DUPd8Pseudo_UPD,  ARM::VLD4DUPd8_UPD, true, true, true,  SingleSpc, 4, 8,true},
 { ARM::VLD4DUPq16EvenPseudo, ARM::VLD4DUPq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
 { ARM::VLD4DUPq16OddPseudo,  ARM::VLD4DUPq16,     true, false, false, OddDblSpc,  4, 4 ,true},
+{ ARM::VLD4DUPq16OddPseudo_UPD,  ARM::VLD4DUPq16_UPD, true, true, true, OddDblSpc,  4, 4 ,true},
 { ARM::VLD4DUPq32EvenPseudo, ARM::VLD4DUPq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
 { ARM::VLD4DUPq32OddPseudo,  ARM::VLD4DUPq32,     true, false, false, OddDblSpc,  4, 2 ,true},
+{ ARM::VLD4DUPq32OddPseudo_UPD,  ARM::VLD4DUPq32_UPD, true, true, true, OddDblSpc,  4, 2 ,true},
 { ARM::VLD4DUPq8EvenPseudo,  ARM::VLD4DUPq8,      true, false, false, EvenDblSpc, 4, 8 ,true},
 { ARM::VLD4DUPq8OddPseudo,   ARM::VLD4DUPq8,      true, false, false, OddDblSpc,  4, 8 ,true},
+{ ARM::VLD4DUPq8OddPseudo_UPD,   ARM::VLD4DUPq8_UPD, true, true, true, OddDblSpc,  4, 8 ,true},
 
 { ARM::VLD4LNd16Pseudo,     ARM::VLD4LNd16,     true, false, false, SingleSpc,  4, 4 ,true},
 { ARM::VLD4LNd16Pseudo_UPD, ARM::VLD4LNd16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
 { ARM::VLD4LNd32Pseudo,     ARM::VLD4LNd32,     true, false, false, SingleSpc,  4, 2 ,true},
 { ARM::VLD4LNd32Pseudo_UPD, ARM::VLD4LNd32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
 { ARM::VLD4LNd8Pseudo,      ARM::VLD4LNd8,      true, false, false, SingleSpc,  4, 8 ,true},
 { ARM::VLD4LNd8Pseudo_UPD,  ARM::VLD4LNd8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
 { ARM::VLD4LNq16Pseudo,     ARM::VLD4LNq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
   unsigned NumRegs = TableEntry->NumRegs;
 
   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
                                     TII->get(TableEntry->RealOpc));
   unsigned OpIdx = 0;
 
   bool DstIsDead = MI.getOperand(OpIdx).isDead();
   Register DstReg = MI.getOperand(OpIdx++).getReg();
   if(TableEntry->RealOpc == ARM::VLD2DUPd8x2 ||
      TableEntry->RealOpc == ARM::VLD2DUPd16x2 ||
-     TableEntry->RealOpc == ARM::VLD2DUPd32x2) {
+     TableEntry->RealOpc == ARM::VLD2DUPd32x2 ||
+     TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_fixed ||
+     TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_fixed ||
+     TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_fixed ||
+     TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_register ||
+     TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_register ||
+     TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_register) {
     unsigned SubRegIndex;
     if (RegSpc == EvenDblSpc) {
       SubRegIndex = ARM::dsub_0;
     } else {
       assert(RegSpc == OddDblSpc && "Unexpected spacing!");
       SubRegIndex = ARM::dsub_1;
     }
     Register SubReg = TRI->getSubReg(DstReg, SubRegIndex);
if (TableEntry->hasWritebackOperand) {
     const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
     if (TableEntry->RealOpc == ARM::VLD1d8Qwb_fixed ||
         TableEntry->RealOpc == ARM::VLD1d16Qwb_fixed ||
         TableEntry->RealOpc == ARM::VLD1d32Qwb_fixed ||
         TableEntry->RealOpc == ARM::VLD1d64Qwb_fixed ||
         TableEntry->RealOpc == ARM::VLD1d8Twb_fixed ||
         TableEntry->RealOpc == ARM::VLD1d16Twb_fixed ||
         TableEntry->RealOpc == ARM::VLD1d32Twb_fixed ||
-        TableEntry->RealOpc == ARM::VLD1d64Twb_fixed) {
+        TableEntry->RealOpc == ARM::VLD1d64Twb_fixed ||
+        TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_fixed ||
+        TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_fixed ||
+        TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_fixed) {
       assert(AM6Offset.getReg() == 0 &&
              "A fixed writing-back pseudo instruction provides an offset "
              "register!");
     } else {
       MIB.add(AM6Offset);
     }
   }
 
   // For an instruction writing double-spaced subregs, the pseudo instruction
   // has an extra operand that is a use of the super-register.  Record the
   // operand index and skip over it.
   unsigned SrcOpIdx = 0;
   if(TableEntry->RealOpc != ARM::VLD2DUPd8x2 &&
      TableEntry->RealOpc != ARM::VLD2DUPd16x2 &&
-     TableEntry->RealOpc != ARM::VLD2DUPd32x2) {
+     TableEntry->RealOpc != ARM::VLD2DUPd32x2 &&
+     TableEntry->RealOpc != ARM::VLD2DUPd8x2wb_fixed &&
+     TableEntry->RealOpc != ARM::VLD2DUPd16x2wb_fixed &&
+     TableEntry->RealOpc != ARM::VLD2DUPd32x2wb_fixed &&
+     TableEntry->RealOpc != ARM::VLD2DUPd8x2wb_register &&
+     TableEntry->RealOpc != ARM::VLD2DUPd16x2wb_register &&
+     TableEntry->RealOpc != ARM::VLD2DUPd32x2wb_register) {
     if (RegSpc == EvenDblSpc || RegSpc == OddDblSpc ||
         RegSpc == SingleLowSpc || RegSpc == SingleHighQSpc ||
         RegSpc == SingleHighTSpc)
       SrcOpIdx = OpIdx++;
   }
 
   // Copy the predicate operands.
   MIB.add(MI.getOperand(OpIdx++));
switch (Opcode) {
     case ARM::VLD4DUPd16Pseudo_UPD:
     case ARM::VLD4DUPd32Pseudo_UPD:
     case ARM::VLD2DUPq8EvenPseudo:
     case ARM::VLD2DUPq8OddPseudo:
     case ARM::VLD2DUPq16EvenPseudo:
     case ARM::VLD2DUPq16OddPseudo:
     case ARM::VLD2DUPq32EvenPseudo:
     case ARM::VLD2DUPq32OddPseudo:
+    case ARM::VLD2DUPq8OddPseudoWB_fixed:
+    case ARM::VLD2DUPq8OddPseudoWB_register:
+    case ARM::VLD2DUPq16OddPseudoWB_fixed:
+    case ARM::VLD2DUPq16OddPseudoWB_register:
+    case ARM::VLD2DUPq32OddPseudoWB_fixed:
+    case ARM::VLD2DUPq32OddPseudoWB_register:
     case ARM::VLD3DUPq8EvenPseudo:
     case ARM::VLD3DUPq8OddPseudo:
     case ARM::VLD3DUPq16EvenPseudo:
     case ARM::VLD3DUPq16OddPseudo:
     case ARM::VLD3DUPq32EvenPseudo:
     case ARM::VLD3DUPq32OddPseudo:
+    case ARM::VLD3DUPq8OddPseudo_UPD:
+    case ARM::VLD3DUPq16OddPseudo_UPD:
+    case ARM::VLD3DUPq32OddPseudo_UPD:
     case ARM::VLD4DUPq8EvenPseudo:
     case ARM::VLD4DUPq8OddPseudo:
     case ARM::VLD4DUPq16EvenPseudo:
     case ARM::VLD4DUPq16OddPseudo:
     case ARM::VLD4DUPq32EvenPseudo:
     case ARM::VLD4DUPq32OddPseudo:
+    case ARM::VLD4DUPq8OddPseudo_UPD:
+    case ARM::VLD4DUPq16OddPseudo_UPD:
+    case ARM::VLD4DUPq32OddPseudo_UPD:
       ExpandVLD(MBBI);
       return true;
 
     case ARM::VST2q8Pseudo:
     case ARM::VST2q16Pseudo:
     case ARM::VST2q32Pseudo:
     case ARM::VST2q8PseudoWB_fixed:
     case ARM::VST2q16PseudoWB_fixed:

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines an instruction selector for the ARM target.
static bool isVLDfixed(unsigned Opc)
   case ARM::VLD2d16wb_fixed : return true;
   case ARM::VLD2d32wb_fixed : return true;
   case ARM::VLD2q8PseudoWB_fixed : return true;
   case ARM::VLD2q16PseudoWB_fixed : return true;
   case ARM::VLD2q32PseudoWB_fixed : return true;
   case ARM::VLD2DUPd8wb_fixed : return true;
   case ARM::VLD2DUPd16wb_fixed : return true;
   case ARM::VLD2DUPd32wb_fixed : return true;
+  case ARM::VLD2DUPq8OddPseudoWB_fixed: return true;
+  case ARM::VLD2DUPq16OddPseudoWB_fixed: return true;
+  case ARM::VLD2DUPq32OddPseudoWB_fixed: return true;
   }
 }
 
 static bool isVSTfixed(unsigned Opc)
 {
   switch (Opc) {
   default: return false;
   case ARM::VST1d8wb_fixed : return true;
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
   case ARM::VLD1d32QPseudoWB_fixed: return ARM::VLD1d32QPseudoWB_register;
   case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
   case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
   case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
   case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
   case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
   case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
   case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
+  case ARM::VLD2DUPq8OddPseudoWB_fixed: return ARM::VLD2DUPq8OddPseudoWB_register;
+  case ARM::VLD2DUPq16OddPseudoWB_fixed: return ARM::VLD2DUPq16OddPseudoWB_register;
+  case ARM::VLD2DUPq32OddPseudoWB_fixed: return ARM::VLD2DUPq32OddPseudoWB_register;
 
   case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
   case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
   case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
   case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
   case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
   case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
   case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic,
   ResTys.push_back(ResTy);
   if (isUpdating)
     ResTys.push_back(MVT::i32);
   ResTys.push_back(MVT::Other);
 
   SDValue Pred = getAL(CurDAG, dl);
   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
 
-  SDNode *VLdDup;
-  if (is64BitVector || NumVecs == 1) {
-    SmallVector<SDValue, 6> Ops;
-    Ops.push_back(MemAddr);
-    Ops.push_back(Align);
-    unsigned Opc = is64BitVector ? DOpcodes[OpcodeIndex] :
-                                   QOpcodes0[OpcodeIndex];
-    if (isUpdating) {
-      // fixed-stride update instructions don't have an explicit writeback
-      // operand. It's implicit in the opcode itself.
-      SDValue Inc = N->getOperand(2);
-      bool IsImmUpdate =
-          isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
-      if (NumVecs <= 2 && !IsImmUpdate)
+  unsigned Opc = is64BitVector    ? DOpcodes[OpcodeIndex]
+                 : (NumVecs == 1) ? QOpcodes0[OpcodeIndex]
+                                  : QOpcodes1[OpcodeIndex];
+    if (IsImmUpdate) {
+      if (!isVLDfixed(Opc))
+        Ops.push_back(Reg0);
+    } else {
+      if (isVLDfixed(Opc))
         Opc = getVLDSTRegisterUpdateOpcode(Opc);
-      if (!IsImmUpdate)
-        Ops.push_back(Inc);
-      // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
-      else if (NumVecs > 2)
-        Ops.push_back(Reg0);
     }
-    Ops.push_back(Pred);
-    Ops.push_back(Reg0);
-    Ops.push_back(Chain);
-    VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
+  }
+  if (is64BitVector || NumVecs == 1) {
+    // Double registers and VLD1 quad registers are directly supported.
   } else if (NumVecs == 2) {
-    const SDValue OpsA[] = { MemAddr, Align, Pred, Reg0, Chain };
-    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
-                                          dl, ResTys, OpsA);
-
+    const SDValue OpsA[] = {MemAddr, Align, Pred, Reg0, Chain};
+    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, ResTy,
+                                          MVT::Other, OpsA);
     Chain = SDValue(VLdA, 1);
-    const SDValue OpsB[] = { MemAddr, Align, Pred, Reg0, Chain };
-    VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
   } else {
-    SDValue ImplDef =
-      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
-    const SDValue OpsA[] = { MemAddr, Align, ImplDef, Pred, Reg0, Chain };
-    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
-                                          dl, ResTys, OpsA);
-
-    SDValue SuperReg = SDValue(VLdA, 0);
+    SDValue ImplDef = SDValue(
+        CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
+    const SDValue OpsA[] = {MemAddr, Align, ImplDef, Pred, Reg0, Chain};
+    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, ResTy,
+                                          MVT::Other, OpsA);
+    Ops.push_back(SDValue(VLdA, 0));
     Chain = SDValue(VLdA, 1);
-    const SDValue OpsB[] = { MemAddr, Align, SuperReg, Pred, Reg0, Chain };
-    VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
   }
 
+  Ops.push_back(Pred);
+  Ops.push_back(Reg0);
+  Ops.push_back(Chain);
+
+  SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
+
   // Transfer memoperands.
   MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
   CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {MemOp});
 
   // Extract the subregisters.
   if (NumVecs == 1) {
     ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
   } else {
case ARMISD::VLD1DUP_UPD: {
     static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
                                          ARM::VLD1DUPq16wb_fixed,
                                          ARM::VLD1DUPq32wb_fixed };
     SelectVLDDup(N, /* IsIntrinsic= */ false, true, 1, DOpcodes, QOpcodes);
     return;
   }
 
   case ARMISD::VLD2DUP_UPD: {
-    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
+    static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
                                          ARM::VLD2DUPd16wb_fixed,
-                                        ARM::VLD2DUPd32wb_fixed };
-    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 2, Opcodes);
+                                         ARM::VLD2DUPd32wb_fixed,
+                                         ARM::VLD1q64wb_fixed };
+    static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
+                                          ARM::VLD2DUPq16EvenPseudo,
+                                          ARM::VLD2DUPq32EvenPseudo };
+    static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
+                                          ARM::VLD2DUPq16OddPseudoWB_fixed,
+                                          ARM::VLD2DUPq32OddPseudoWB_fixed };
+    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
     return;
   }
 
   case ARMISD::VLD3DUP_UPD: {
-    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
+    static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
                                          ARM::VLD3DUPd16Pseudo_UPD,
-                                        ARM::VLD3DUPd32Pseudo_UPD };
-    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 3, Opcodes);
+                                         ARM::VLD3DUPd32Pseudo_UPD,
+                                         ARM::VLD1d64TPseudoWB_fixed };
+    static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
+                                          ARM::VLD3DUPq16EvenPseudo,
+                                          ARM::VLD3DUPq32EvenPseudo };
+    static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
+                                          ARM::VLD3DUPq16OddPseudo_UPD,
+                                          ARM::VLD3DUPq32OddPseudo_UPD };
+    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
     return;
   }
 
   case ARMISD::VLD4DUP_UPD: {
-    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
+    static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
                                          ARM::VLD4DUPd16Pseudo_UPD,
-                                        ARM::VLD4DUPd32Pseudo_UPD };
-    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 4, Opcodes);
+                                         ARM::VLD4DUPd32Pseudo_UPD,
+                                         ARM::VLD1d64QPseudoWB_fixed };
+    static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
+                                          ARM::VLD4DUPq16EvenPseudo,
+                                          ARM::VLD4DUPq32EvenPseudo };
+    static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
+                                          ARM::VLD4DUPq16OddPseudo_UPD,
+                                          ARM::VLD4DUPq32OddPseudo_UPD };
+    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
     return;
   }
 
   case ARMISD::VLD1_UPD: {
     static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
                                          ARM::VLD1d16wb_fixed,
                                          ARM::VLD1d32wb_fixed,
                                          ARM::VLD1d64wb_fixed };

llvm/lib/Target/ARM/ARMISelLowering.cpp

 //===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines the interfaces that ARM uses to lower LLVM code into a
if (isIntrinsic) {
       case Intrinsic::arm_neon_vld4:     NewOpc = ARMISD::VLD4_UPD;
         NumVecs = 4; break;
       case Intrinsic::arm_neon_vld1x2:   NewOpc = ARMISD::VLD1x2_UPD;
         NumVecs = 2; hasAlignment = false; break;
       case Intrinsic::arm_neon_vld1x3:   NewOpc = ARMISD::VLD1x3_UPD;
         NumVecs = 3; hasAlignment = false; break;
       case Intrinsic::arm_neon_vld1x4:   NewOpc = ARMISD::VLD1x4_UPD;
         NumVecs = 4; hasAlignment = false; break;
-      case Intrinsic::arm_neon_vld2dup:
-      case Intrinsic::arm_neon_vld3dup:
-      case Intrinsic::arm_neon_vld4dup:
-        // TODO: Support updating VLDxDUP nodes. For now, we just skip
-        // combining base updates for such intrinsics.
-        continue;
+      case Intrinsic::arm_neon_vld2dup:  NewOpc = ARMISD::VLD2DUP_UPD;
+        NumVecs = 2; break;
+      case Intrinsic::arm_neon_vld3dup:  NewOpc = ARMISD::VLD3DUP_UPD;
+        NumVecs = 3; break;
+      case Intrinsic::arm_neon_vld4dup:  NewOpc = ARMISD::VLD4DUP_UPD;
+        NumVecs = 4; break;
       case Intrinsic::arm_neon_vld2lane: NewOpc = ARMISD::VLD2LN_UPD;
         NumVecs = 2; isLaneOp = true; break;
       case Intrinsic::arm_neon_vld3lane: NewOpc = ARMISD::VLD3LN_UPD;
         NumVecs = 3; isLaneOp = true; break;
       case Intrinsic::arm_neon_vld4lane: NewOpc = ARMISD::VLD4LN_UPD;
         NumVecs = 4; isLaneOp = true; break;
       case Intrinsic::arm_neon_vst1:     NewOpc = ARMISD::VST1_UPD;
         NumVecs = 1; isLoadOp = false; break;
if (isLoadOp) {
       VecTy = N->getValueType(0);
     } else if (isIntrinsic) {
       VecTy = N->getOperand(AddrOpIdx+1).getValueType();
     } else {
       assert(isStore && "Node has to be a load, a store, or an intrinsic!");
       VecTy = N->getOperand(1).getValueType();
     }
 
+    bool isVLDDUPOp =
+        NewOpc == ARMISD::VLD1DUP_UPD || NewOpc == ARMISD::VLD2DUP_UPD ||
+        NewOpc == ARMISD::VLD3DUP_UPD || NewOpc == ARMISD::VLD4DUP_UPD;
+
     unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
-    if (isLaneOp)
+    if (isLaneOp || isVLDDUPOp)
       NumBytes /= VecTy.getVectorNumElements();
 
     // If the increment is a constant, it must match the memory ref size.
     SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
     ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode());
     if (NumBytes >= 3 * 16 && (!CInc || CInc->getZExtValue() != NumBytes)) {
       // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
       // separate instructions that make it harder to use a non-constant update.

llvm/lib/Target/ARM/ARMInstrNEON.td

 
 defm VLD2DUPd8x2wb  : VLD2DUPWB<{0,0,1,0}, "8",  VecListDPairSpacedAllLanes,
                                 addrmode6dupalign16>;
 defm VLD2DUPd16x2wb : VLD2DUPWB<{0,1,1,?}, "16", VecListDPairSpacedAllLanes,
                                 addrmode6dupalign32>;
 defm VLD2DUPd32x2wb : VLD2DUPWB<{1,0,1,?}, "32", VecListDPairSpacedAllLanes,
                                 addrmode6dupalign64>;
 
+def VLD2DUPq8OddPseudoWB_fixed     : VLDQQWBfixedPseudo<IIC_VLD2dup>, Sched<[WriteVLD2]>;
+def VLD2DUPq16OddPseudoWB_fixed    : VLDQQWBfixedPseudo<IIC_VLD2dup>, Sched<[WriteVLD2]>;
+def VLD2DUPq32OddPseudoWB_fixed    : VLDQQWBfixedPseudo<IIC_VLD2dup>, Sched<[WriteVLD2]>;
+def VLD2DUPq8OddPseudoWB_register  : VLDQQWBPseudo<IIC_VLD2dup>, Sched<[WriteVLD2]>;
+def VLD2DUPq16OddPseudoWB_register : VLDQQWBPseudo<IIC_VLD2dup>, Sched<[WriteVLD2]>;
+def VLD2DUPq32OddPseudoWB_register : VLDQQWBPseudo<IIC_VLD2dup>, Sched<[WriteVLD2]>;
+
 //   VLD3DUP  : Vector Load (single 3-element structure to all lanes)
 class VLD3DUP<bits<4> op7_4, string Dt>
   : NLdSt<1, 0b10, 0b1110, op7_4, (outs DPR:$Vd, DPR:$dst2, DPR:$dst3),
           (ins addrmode6dup:$Rn), IIC_VLD3dup,
           "vld3", Dt, "\\{$Vd[], $dst2[], $dst3[]\\}, $Rn", "", []>,
     Sched<[WriteVLD2]> {
   let Rm = 0b1111;
   let Inst{4} = 0;

 def VLD3DUPq8_UPD  : VLD3DUPWB<{0,0,1,0}, "8",  addrmode6dupalign64>;
 def VLD3DUPq16_UPD : VLD3DUPWB<{0,1,1,?}, "16", addrmode6dupalign64>;
 def VLD3DUPq32_UPD : VLD3DUPWB<{1,0,1,?}, "32", addrmode6dupalign64>;
 
 def VLD3DUPd8Pseudo_UPD  : VLDQQWBPseudo<IIC_VLD3dupu>, Sched<[WriteVLD2]>;
 def VLD3DUPd16Pseudo_UPD : VLDQQWBPseudo<IIC_VLD3dupu>, Sched<[WriteVLD2]>;
 def VLD3DUPd32Pseudo_UPD : VLDQQWBPseudo<IIC_VLD3dupu>, Sched<[WriteVLD2]>;
 
+def VLD3DUPq8OddPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD3dupu>, Sched<[WriteVLD2]>;
+def VLD3DUPq16OddPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD3dupu>, Sched<[WriteVLD2]>;
+def VLD3DUPq32OddPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD3dupu>, Sched<[WriteVLD2]>;
+
 //   VLD4DUP  : Vector Load (single 4-element structure to all lanes)
 class VLD4DUP<bits<4> op7_4, string Dt>
   : NLdSt<1, 0b10, 0b1111, op7_4,
           (outs DPR:$Vd, DPR:$dst2, DPR:$dst3, DPR:$dst4),
           (ins addrmode6dup:$Rn), IIC_VLD4dup,
           "vld4", Dt, "\\{$Vd[], $dst2[], $dst3[], $dst4[]\\}, $Rn", "", []> {
   let Rm = 0b1111;
   let Inst{4} = Rn{4};

 def VLD4DUPq8_UPD  : VLD4DUPWB<{0,0,1,0}, "8">;
 def VLD4DUPq16_UPD : VLD4DUPWB<{0,1,1,?}, "16">;
 def VLD4DUPq32_UPD : VLD4DUPWB<{1,?,1,?}, "32"> { let Inst{6} = Rn{5}; }
 
 def VLD4DUPd8Pseudo_UPD  : VLDQQWBPseudo<IIC_VLD4dupu>, Sched<[WriteVLD2]>;
 def VLD4DUPd16Pseudo_UPD : VLDQQWBPseudo<IIC_VLD4dupu>, Sched<[WriteVLD2]>;
 def VLD4DUPd32Pseudo_UPD : VLDQQWBPseudo<IIC_VLD4dupu>, Sched<[WriteVLD2]>;
 
+def VLD4DUPq8OddPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD4dupu>, Sched<[WriteVLD2]>;
+def VLD4DUPq16OddPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD4dupu>, Sched<[WriteVLD2]>;
+def VLD4DUPq32OddPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD4dupu>, Sched<[WriteVLD2]>;
+
 } // mayLoad = 1, hasSideEffects = 0, hasExtraDefRegAllocReq = 1
 
 let mayStore = 1, hasSideEffects = 0, hasExtraSrcRegAllocReq = 1 in {
 
 // Classes for VST* pseudo-instructions with multi-register operands.
 // These are expanded to real instructions after register allocation.
 class VSTQPseudo<InstrItinClass itin>
   : PseudoNLdSt<(outs), (ins addrmode6:$addr, QPR:$src), itin, "">;

llvm/test/CodeGen/ARM/arm-vlddup-update.ll

 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -verify-machineinstrs \
 ; RUN:     -asm-verbose=false | FileCheck %s
 
+%struct.uint16x4x2_t = type { <4 x i16>, <4 x i16> }
+%struct.uint16x4x3_t = type { <4 x i16>, <4 x i16>, <4 x i16> }
+%struct.uint16x4x4_t = type { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> }
+
 %struct.uint32x2x2_t = type { <2 x i32>, <2 x i32> }
 %struct.uint32x2x3_t = type { <2 x i32>, <2 x i32>, <2 x i32> }
 %struct.uint32x2x4_t = type { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> }
 
+%struct.uint64x1x2_t = type { <1 x i64>, <1 x i64> }
+%struct.uint64x1x3_t = type { <1 x i64>, <1 x i64>, <1 x i64> }
+%struct.uint64x1x4_t = type { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> }
+
+%struct.uint8x8x2_t = type { <8 x i8>, <8 x i8> }
+%struct.uint8x8x3_t = type { <8 x i8>, <8 x i8>, <8 x i8> }
+%struct.uint8x8x4_t = type { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> }
+
+%struct.uint16x8x2_t = type { <8 x i16>, <8 x i16> }
+%struct.uint16x8x3_t = type { <8 x i16>, <8 x i16>, <8 x i16> }
+%struct.uint16x8x4_t = type { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> }
+
+%struct.uint32x4x2_t = type { <4 x i32>, <4 x i32> }
+%struct.uint32x4x3_t = type { <4 x i32>, <4 x i32>, <4 x i32> }
+%struct.uint32x4x4_t = type { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> }
+
+%struct.uint8x16x2_t = type { <16 x i8>, <16 x i8> }
+%struct.uint8x16x3_t = type { <16 x i8>, <16 x i8>, <16 x i8> }
+%struct.uint8x16x4_t = type { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> }
+
+declare %struct.uint8x8x2_t @llvm.arm.neon.vld2dup.v8i8.p0i8(i8*, i32)
+declare %struct.uint16x4x2_t @llvm.arm.neon.vld2dup.v4i16.p0i8(i8*, i32)
 declare %struct.uint32x2x2_t @llvm.arm.neon.vld2dup.v2i32.p0i8(i8*, i32)
+declare %struct.uint64x1x2_t @llvm.arm.neon.vld2dup.v1i64.p0i8(i8*, i32)
+
+declare %struct.uint8x8x3_t @llvm.arm.neon.vld3dup.v8i8.p0i8(i8*, i32)
+declare %struct.uint16x4x3_t @llvm.arm.neon.vld3dup.v4i16.p0i8(i8*, i32)
 declare %struct.uint32x2x3_t @llvm.arm.neon.vld3dup.v2i32.p0i8(i8*, i32)
+declare %struct.uint64x1x3_t @llvm.arm.neon.vld3dup.v1i64.p0i8(i8*, i32)
+
+declare %struct.uint8x8x4_t @llvm.arm.neon.vld4dup.v8i8.p0i8(i8*, i32)
+declare %struct.uint16x4x4_t @llvm.arm.neon.vld4dup.v4i16.p0i8(i8*, i32)
 declare %struct.uint32x2x4_t @llvm.arm.neon.vld4dup.v2i32.p0i8(i8*, i32)
+declare %struct.uint64x1x4_t @llvm.arm.neon.vld4dup.v1i64.p0i8(i8*, i32)
+
+declare %struct.uint8x16x2_t @llvm.arm.neon.vld2dup.v16i8.p0i8(i8*, i32)
+declare %struct.uint16x8x2_t @llvm.arm.neon.vld2dup.v8i16.p0i8(i8*, i32)
+declare %struct.uint32x4x2_t @llvm.arm.neon.vld2dup.v4i32.p0i8(i8*, i32)
+
+declare %struct.uint8x16x3_t @llvm.arm.neon.vld3dup.v16i8.p0i8(i8*, i32)
+declare %struct.uint16x8x3_t @llvm.arm.neon.vld3dup.v8i16.p0i8(i8*, i32)
+declare %struct.uint32x4x3_t @llvm.arm.neon.vld3dup.v4i32.p0i8(i8*, i32)
+
+declare %struct.uint8x16x4_t @llvm.arm.neon.vld4dup.v16i8.p0i8(i8*, i32)
+declare %struct.uint16x8x4_t @llvm.arm.neon.vld4dup.v8i16.p0i8(i8*, i32)
+declare %struct.uint32x4x4_t @llvm.arm.neon.vld4dup.v4i32.p0i8(i8*, i32)
+
+define i8* @test_vld2_dup_u16_update(%struct.uint16x4x2_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld2_dup_u16_update:
+; CHECK:         vld2.16 {d16[], d17[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint16x4x2_t @llvm.arm.neon.vld2dup.v4i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x4x2_t %tmp, %struct.uint16x4x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 4
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2_dup_u16_update_reg(%struct.uint16x4x2_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld2_dup_u16_update_reg:
+; CHECK:         vld2.16 {d16[], d17[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint16x4x2_t @llvm.arm.neon.vld2dup.v4i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x4x2_t %tmp, %struct.uint16x4x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
 
-; CHECK-LABEL: test_vld2_dup_update
-; CHECK: vld2.32 {d16[], d17[]}, {{\[}}[[SRC_R:r[0-9]+]]]
-; CHECK: add {{r[0-9]+|lr}}, [[SRC_R]], #4
 define i8* @test_vld2_dup_update(%struct.uint32x2x2_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld2_dup_update:
+; CHECK:         vld2.32 {d16[], d17[]}, [r1]!
 entry:
   %tmp = tail call %struct.uint32x2x2_t @llvm.arm.neon.vld2dup.v2i32.p0i8(i8* %src, i32 4)
   store %struct.uint32x2x2_t %tmp, %struct.uint32x2x2_t* %dest, align 8
-  %updated_src = getelementptr inbounds i8, i8* %src, i32 4
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 8
   ret i8* %updated_src
 }
 
-; CHECK-LABEL: test_vld3_dup_update
-; CHECK: vld3.32 {d16[], d17[], d18[]}, {{\[}}[[SRC_R:r[0-9]+]]]
-; CHECK: add {{r[0-9]+|lr}}, [[SRC_R]], #4
-define i8* @test_vld3_dup_update(%struct.uint32x2x3_t* %dest, i8* %src) {
+define i8* @test_vld2_dup_update_reg(%struct.uint32x2x2_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld2_dup_update_reg:
+; CHECK:         vld2.32 {d16[], d17[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint32x2x2_t @llvm.arm.neon.vld2dup.v2i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x2x2_t %tmp, %struct.uint32x2x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2_dup_u64_update(%struct.uint64x1x2_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld2_dup_u64_update:
+; CHECK:         vld1.64 {d16, d17}, [r1:64]!
+entry:
+  %tmp = tail call %struct.uint64x1x2_t @llvm.arm.neon.vld2dup.v1i64.p0i8(i8* %src, i32 8)
+  store %struct.uint64x1x2_t %tmp, %struct.uint64x1x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 16
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2_dup_u64_update_reg(%struct.uint64x1x2_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld2_dup_u64_update_reg:
+; CHECK:         vld1.64 {d16, d17}, [r1:64], r2
+entry:
+  %tmp = tail call %struct.uint64x1x2_t @llvm.arm.neon.vld2dup.v1i64.p0i8(i8* %src, i32 8)
+  store %struct.uint64x1x2_t %tmp, %struct.uint64x1x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2_dup_u8_update(%struct.uint8x8x2_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld2_dup_u8_update:
+; CHECK:         vld2.8 {d16[], d17[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint8x8x2_t @llvm.arm.neon.vld2dup.v8i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x8x2_t %tmp, %struct.uint8x8x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 2
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2_dup_u8_update_reg(%struct.uint8x8x2_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld2_dup_u8_update_reg:
+; CHECK:         vld2.8 {d16[], d17[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint8x8x2_t @llvm.arm.neon.vld2dup.v8i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x8x2_t %tmp, %struct.uint8x8x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3_dup_u16_update(%struct.uint16x4x3_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld3_dup_u16_update:
+; CHECK:         vld3.16 {d16[], d17[], d18[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint16x4x3_t @llvm.arm.neon.vld3dup.v4i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x4x3_t %tmp, %struct.uint16x4x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 6
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3_dup_u16_update_reg(%struct.uint16x4x3_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld3_dup_u16_update_reg:
+; CHECK:         vld3.16 {d16[], d17[], d18[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint16x4x3_t @llvm.arm.neon.vld3dup.v4i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x4x3_t %tmp, %struct.uint16x4x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3_dup_u32_update(%struct.uint32x2x3_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld3_dup_u32_update:
+; CHECK:         vld3.32 {d16[], d17[], d18[]}, [r1]!
 entry:
   %tmp = tail call %struct.uint32x2x3_t @llvm.arm.neon.vld3dup.v2i32.p0i8(i8* %src, i32 4)
   store %struct.uint32x2x3_t %tmp, %struct.uint32x2x3_t* %dest, align 8
-  %updated_src = getelementptr inbounds i8, i8* %src, i32 4
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 12
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3_dup_u32_update_reg(%struct.uint32x2x3_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld3_dup_u32_update_reg:
+; CHECK:         vld3.32 {d16[], d17[], d18[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint32x2x3_t @llvm.arm.neon.vld3dup.v2i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x2x3_t %tmp, %struct.uint32x2x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3_dup_u64_update(%struct.uint64x1x3_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld3_dup_u64_update:
+; CHECK:         vld1.64 {d16, d17, d18}, [r1]!
+entry:
+  %tmp = tail call %struct.uint64x1x3_t @llvm.arm.neon.vld3dup.v1i64.p0i8(i8* %src, i32 8)
+  store %struct.uint64x1x3_t %tmp, %struct.uint64x1x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 24
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3_dup_u64_update_reg(%struct.uint64x1x3_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld3_dup_u64_update_reg:
+; CHECK:         vld1.64 {d16, d17, d18}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint64x1x3_t @llvm.arm.neon.vld3dup.v1i64.p0i8(i8* %src, i32 8)
+  store %struct.uint64x1x3_t %tmp, %struct.uint64x1x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3_dup_u8_update(%struct.uint8x8x3_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld3_dup_u8_update:
+; CHECK:         vld3.8 {d16[], d17[], d18[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint8x8x3_t @llvm.arm.neon.vld3dup.v8i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x8x3_t %tmp, %struct.uint8x8x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 3
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3_dup_u8_update_reg(%struct.uint8x8x3_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld3_dup_u8_update_reg:
+; CHECK:         vld3.8 {d16[], d17[], d18[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint8x8x3_t @llvm.arm.neon.vld3dup.v8i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x8x3_t %tmp, %struct.uint8x8x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4_dup_u16_update(%struct.uint16x4x4_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld4_dup_u16_update:
+; CHECK:         vld4.16 {d16[], d17[], d18[], d19[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint16x4x4_t @llvm.arm.neon.vld4dup.v4i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x4x4_t %tmp, %struct.uint16x4x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 8
   ret i8* %updated_src
 }
 
-; CHECK-LABEL: test_vld4_dup_update
-; CHECK: vld4.32 {d16[], d17[], d18[], d19[]}, {{\[}}[[SRC_R:r[0-9]+]]]
-; CHECK: add {{r[0-9]+|lr}}, [[SRC_R]], #4
-define i8* @test_vld4_dup_update(%struct.uint32x2x4_t* %dest, i8* %src) {
+define i8* @test_vld4_dup_u16_update_reg(%struct.uint16x4x4_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld4_dup_u16_update_reg:
+; CHECK:         vld4.16 {d16[], d17[], d18[], d19[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint16x4x4_t @llvm.arm.neon.vld4dup.v4i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x4x4_t %tmp, %struct.uint16x4x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4_dup_u32_update(%struct.uint32x2x4_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld4_dup_u32_update:
+; CHECK:         vld4.32 {d16[], d17[], d18[], d19[]}, [r1]!
 entry:
   %tmp = tail call %struct.uint32x2x4_t @llvm.arm.neon.vld4dup.v2i32.p0i8(i8* %src, i32 4)
   store %struct.uint32x2x4_t %tmp, %struct.uint32x2x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 16
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4_dup_u32_update_reg(%struct.uint32x2x4_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld4_dup_u32_update_reg:
+; CHECK:         vld4.32 {d16[], d17[], d18[], d19[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint32x2x4_t @llvm.arm.neon.vld4dup.v2i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x2x4_t %tmp, %struct.uint32x2x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4_dup_u64_update(%struct.uint64x1x4_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld4_dup_u64_update:
+; CHECK:         vld1.64 {d16, d17, d18, d19}, [r1:64]!
+entry:
+  %tmp = tail call %struct.uint64x1x4_t @llvm.arm.neon.vld4dup.v1i64.p0i8(i8* %src, i32 8)
+  store %struct.uint64x1x4_t %tmp, %struct.uint64x1x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 32
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4_dup_u64_update_reg(%struct.uint64x1x4_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld4_dup_u64_update_reg:
+; CHECK:         vld1.64 {d16, d17, d18, d19}, [r1:64], r2
+entry:
+  %tmp = tail call %struct.uint64x1x4_t @llvm.arm.neon.vld4dup.v1i64.p0i8(i8* %src, i32 8)
+  store %struct.uint64x1x4_t %tmp, %struct.uint64x1x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4_dup_u8_update(%struct.uint8x8x4_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld4_dup_u8_update:
+; CHECK:         vld4.8 {d16[], d17[], d18[], d19[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint8x8x4_t @llvm.arm.neon.vld4dup.v8i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x8x4_t %tmp, %struct.uint8x8x4_t* %dest, align 8
   %updated_src = getelementptr inbounds i8, i8* %src, i32 4
   ret i8* %updated_src
 }
+
+define i8* @test_vld4_dup_u8_update_reg(%struct.uint8x8x4_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld4_dup_u8_update_reg:
+; CHECK:         vld4.8 {d16[], d17[], d18[], d19[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint8x8x4_t @llvm.arm.neon.vld4dup.v8i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x8x4_t %tmp, %struct.uint8x8x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2q_dup_u16_update(%struct.uint16x8x2_t* %dest, i8* %src, <8 x i16>* %dest0) {
+; CHECK-LABEL: test_vld2q_dup_u16_update:
+; CHECK:         vld2.16 {d16[], d18[]}, [r1]
+; CHECK-NEXT:    vld2.16 {d17[], d19[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint16x8x2_t @llvm.arm.neon.vld2dup.v8i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x8x2_t %tmp, %struct.uint16x8x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 4
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2q_dup_u16_update_reg(%struct.uint16x8x2_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld2q_dup_u16_update_reg:
+; CHECK:         vld2.16 {d16[], d18[]}, [r1]
+; CHECK-NEXT:    vld2.16 {d17[], d19[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint16x8x2_t @llvm.arm.neon.vld2dup.v8i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x8x2_t %tmp, %struct.uint16x8x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2q_dup_u32_update(%struct.uint32x4x2_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld2q_dup_u32_update:
+; CHECK:         vld2.32 {d16[], d18[]}, [r1]
+; CHECK-NEXT:    vld2.32 {d17[], d19[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint32x4x2_t @llvm.arm.neon.vld2dup.v4i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x4x2_t %tmp, %struct.uint32x4x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 8
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2q_dup_u32_update_reg(%struct.uint32x4x2_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld2q_dup_u32_update_reg:
+; CHECK:         vld2.32 {d16[], d18[]}, [r1]
+; CHECK-NEXT:    vld2.32 {d17[], d19[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint32x4x2_t @llvm.arm.neon.vld2dup.v4i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x4x2_t %tmp, %struct.uint32x4x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2q_dup_u8_update(%struct.uint8x16x2_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld2q_dup_u8_update:
+; CHECK:         vld2.8 {d16[], d18[]}, [r1]
+; CHECK-NEXT:    vld2.8 {d17[], d19[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint8x16x2_t @llvm.arm.neon.vld2dup.v16i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x16x2_t %tmp, %struct.uint8x16x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 2
+  ret i8* %updated_src
+}
+
+define i8* @test_vld2q_dup_u8_update_reg(%struct.uint8x16x2_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld2q_dup_u8_update_reg:
+; CHECK:         vld2.8 {d16[], d18[]}, [r1]
+; CHECK-NEXT:    vld2.8 {d17[], d19[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint8x16x2_t @llvm.arm.neon.vld2dup.v16i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x16x2_t %tmp, %struct.uint8x16x2_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3q_dup_u16_update(%struct.uint16x8x3_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld3q_dup_u16_update:
+; CHECK:         vld3.16 {d16[], d18[], d20[]}, [r1]
+; CHECK:         vld3.16 {d17[], d19[], d21[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint16x8x3_t @llvm.arm.neon.vld3dup.v8i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x8x3_t %tmp, %struct.uint16x8x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 6
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3q_dup_u16_update_reg(%struct.uint16x8x3_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld3q_dup_u16_update_reg:
+; CHECK:         vld3.16 {d16[], d18[], d20[]}, [r1]
+; CHECK-NEXT:    vld3.16 {d17[], d19[], d21[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint16x8x3_t @llvm.arm.neon.vld3dup.v8i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x8x3_t %tmp, %struct.uint16x8x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3q_dup_u32_update(%struct.uint32x4x3_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld3q_dup_u32_update:
+; CHECK:         vld3.32 {d16[], d18[], d20[]}, [r1]
+; CHECK:         vld3.32 {d17[], d19[], d21[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint32x4x3_t @llvm.arm.neon.vld3dup.v4i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x4x3_t %tmp, %struct.uint32x4x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 12
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3q_dup_u32_update_reg(%struct.uint32x4x3_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld3q_dup_u32_update_reg:
+; CHECK:         vld3.32 {d16[], d18[], d20[]}, [r1]
+; CHECK-NEXT:    vld3.32 {d17[], d19[], d21[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint32x4x3_t @llvm.arm.neon.vld3dup.v4i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x4x3_t %tmp, %struct.uint32x4x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3q_dup_u8_update(%struct.uint8x16x3_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld3q_dup_u8_update:
+; CHECK:         vld3.8 {d16[], d18[], d20[]}, [r1]
+; CHECK:         vld3.8 {d17[], d19[], d21[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint8x16x3_t @llvm.arm.neon.vld3dup.v16i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x16x3_t %tmp, %struct.uint8x16x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 3
+  ret i8* %updated_src
+}
+
+define i8* @test_vld3q_dup_u8_update_reg(%struct.uint8x16x3_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld3q_dup_u8_update_reg:
+; CHECK:         vld3.8 {d16[], d18[], d20[]}, [r1]
+; CHECK-NEXT:    vld3.8 {d17[], d19[], d21[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint8x16x3_t @llvm.arm.neon.vld3dup.v16i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x16x3_t %tmp, %struct.uint8x16x3_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4q_dup_u16_update(%struct.uint16x8x4_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld4q_dup_u16_update:
+; CHECK:         vld4.16 {d16[], d18[], d20[], d22[]}, [r1]
+; CHECK:         vld4.16 {d17[], d19[], d21[], d23[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint16x8x4_t @llvm.arm.neon.vld4dup.v8i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x8x4_t %tmp, %struct.uint16x8x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 8
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4q_dup_u16_update_reg(%struct.uint16x8x4_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld4q_dup_u16_update_reg:
+; CHECK:         vld4.16 {d16[], d18[], d20[], d22[]}, [r1]
+; CHECK-NEXT:    vld4.16 {d17[], d19[], d21[], d23[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint16x8x4_t @llvm.arm.neon.vld4dup.v8i16.p0i8(i8* %src, i32 2)
+  store %struct.uint16x8x4_t %tmp, %struct.uint16x8x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4q_dup_u32_update(%struct.uint32x4x4_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld4q_dup_u32_update:
+; CHECK:         vld4.32 {d16[], d18[], d20[], d22[]}, [r1]
+; CHECK:         vld4.32 {d17[], d19[], d21[], d23[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint32x4x4_t @llvm.arm.neon.vld4dup.v4i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x4x4_t %tmp, %struct.uint32x4x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 16
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4q_dup_u32_update_reg(%struct.uint32x4x4_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld4q_dup_u32_update_reg:
+; CHECK:         vld4.32 {d16[], d18[], d20[], d22[]}, [r1]
+; CHECK-NEXT:    vld4.32 {d17[], d19[], d21[], d23[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint32x4x4_t @llvm.arm.neon.vld4dup.v4i32.p0i8(i8* %src, i32 4)
+  store %struct.uint32x4x4_t %tmp, %struct.uint32x4x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4q_dup_u8_update(%struct.uint8x16x4_t* %dest, i8* %src) {
+; CHECK-LABEL: test_vld4q_dup_u8_update:
+; CHECK:         vld4.8 {d16[], d18[], d20[], d22[]}, [r1]
+; CHECK:         vld4.8 {d17[], d19[], d21[], d23[]}, [r1]!
+entry:
+  %tmp = tail call %struct.uint8x16x4_t @llvm.arm.neon.vld4dup.v16i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x16x4_t %tmp, %struct.uint8x16x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 4
+  ret i8* %updated_src
+}
+
+define i8* @test_vld4q_dup_u8_update_reg(%struct.uint8x16x4_t* %dest, i8* %src, i32 %inc) {
+; CHECK-LABEL: test_vld4q_dup_u8_update_reg:
+; CHECK:         vld4.8 {d16[], d18[], d20[], d22[]}, [r1]
+; CHECK-NEXT:    vld4.8 {d17[], d19[], d21[], d23[]}, [r1], r2
+entry:
+  %tmp = tail call %struct.uint8x16x4_t @llvm.arm.neon.vld4dup.v16i8.p0i8(i8* %src, i32 1)
+  store %struct.uint8x16x4_t %tmp, %struct.uint8x16x4_t* %dest, align 8
+  %updated_src = getelementptr inbounds i8, i8* %src, i32 %inc
+  ret i8* %updated_src
+}