[ARM][NEON] Combine base address updates for vld1x intrinsics

[krisb]

Remove llvm/test/CodeGen/ARM/pr45824.ll as it is no longer needed.

[dmgreen]

Looks great. I was wondering on D102256 if you were going to do loads too, but didn't want to presume.

Thanks for the patch. LGTM.

[krisb]

@dmgreen thank you for reviewing this!
I think I can also take a look at vldxdup intrinsics by the end of the week. They seem to be the last group of intrinsics that CombineBaseUpdate() hasn't been supporting yet.

[krisb]

Applied a few clang-format suggestions. NFC.

llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp

 //===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains a pass that expands pseudo instructions into target

 { ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
 { ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
 { ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
 { ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
 { ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
 { ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
 
 { ARM::VLD1d16QPseudo,      ARM::VLD1d16Q,     true,  false, false, SingleSpc,  4, 4 ,false},
+{ ARM::VLD1d16QPseudoWB_fixed,  ARM::VLD1d16Qwb_fixed,   true, true, false, SingleSpc,  4, 4 ,false},
+{ ARM::VLD1d16QPseudoWB_register,  ARM::VLD1d16Qwb_register, true, true, true, SingleSpc,  4, 4 ,false},
 { ARM::VLD1d16TPseudo,      ARM::VLD1d16T,     true,  false, false, SingleSpc,  3, 4 ,false},
+{ ARM::VLD1d16TPseudoWB_fixed,  ARM::VLD1d16Twb_fixed,   true, true, false, SingleSpc,  3, 4 ,false},
+{ ARM::VLD1d16TPseudoWB_register,  ARM::VLD1d16Twb_register, true, true, true, SingleSpc,  3, 4 ,false},
+
 { ARM::VLD1d32QPseudo,      ARM::VLD1d32Q,     true,  false, false, SingleSpc,  4, 2 ,false},
+{ ARM::VLD1d32QPseudoWB_fixed,  ARM::VLD1d32Qwb_fixed,   true, true, false, SingleSpc,  4, 2 ,false},
+{ ARM::VLD1d32QPseudoWB_register,  ARM::VLD1d32Qwb_register, true, true, true, SingleSpc,  4, 2 ,false},
 { ARM::VLD1d32TPseudo,      ARM::VLD1d32T,     true,  false, false, SingleSpc,  3, 2 ,false},
+{ ARM::VLD1d32TPseudoWB_fixed,  ARM::VLD1d32Twb_fixed,   true, true, false, SingleSpc,  3, 2 ,false},
+{ ARM::VLD1d32TPseudoWB_register,  ARM::VLD1d32Twb_register, true, true, true, SingleSpc,  3, 2 ,false},
+
 { ARM::VLD1d64QPseudo,      ARM::VLD1d64Q,     true,  false, false, SingleSpc,  4, 1 ,false},
 { ARM::VLD1d64QPseudoWB_fixed,  ARM::VLD1d64Qwb_fixed,   true,  true, false, SingleSpc,  4, 1 ,false},
 { ARM::VLD1d64QPseudoWB_register,  ARM::VLD1d64Qwb_register,   true,  true, true, SingleSpc,  4, 1 ,false},
 { ARM::VLD1d64TPseudo,      ARM::VLD1d64T,     true,  false, false, SingleSpc,  3, 1 ,false},
 { ARM::VLD1d64TPseudoWB_fixed,  ARM::VLD1d64Twb_fixed,   true,  true, false, SingleSpc,  3, 1 ,false},
 { ARM::VLD1d64TPseudoWB_register,  ARM::VLD1d64Twb_register, true, true, true,  SingleSpc,  3, 1 ,false},
+
 { ARM::VLD1d8QPseudo,       ARM::VLD1d8Q,      true,  false, false, SingleSpc,  4, 8 ,false},
+{ ARM::VLD1d8QPseudoWB_fixed,   ARM::VLD1d8Qwb_fixed,    true,  true, false, SingleSpc,  4, 8 ,false},
+{ ARM::VLD1d8QPseudoWB_register,   ARM::VLD1d8Qwb_register,  true, true, true, SingleSpc,  4, 8 ,false},
 { ARM::VLD1d8TPseudo,       ARM::VLD1d8T,      true,  false, false, SingleSpc,  3, 8 ,false},
+{ ARM::VLD1d8TPseudoWB_fixed,   ARM::VLD1d8Twb_fixed,    true,  true, false, SingleSpc,  3, 8 ,false},
+{ ARM::VLD1d8TPseudoWB_register,   ARM::VLD1d8Twb_register,  true,  true, true, SingleSpc,  3, 8 ,false},
+
 { ARM::VLD1q16HighQPseudo,  ARM::VLD1d16Q,     true,  false, false, SingleHighQSpc,  4, 4 ,false},
+{ ARM::VLD1q16HighQPseudo_UPD,  ARM::VLD1d16Qwb_fixed, true,  true, true, SingleHighQSpc,  4, 4 ,false},
 { ARM::VLD1q16HighTPseudo,  ARM::VLD1d16T,     true,  false, false, SingleHighTSpc,  3, 4 ,false},
+{ ARM::VLD1q16HighTPseudo_UPD,  ARM::VLD1d16Twb_fixed, true,  true, true, SingleHighTSpc,  3, 4 ,false},
 { ARM::VLD1q16LowQPseudo_UPD,  ARM::VLD1d16Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 4 ,false},
 { ARM::VLD1q16LowTPseudo_UPD,  ARM::VLD1d16Twb_fixed,   true,  true, true, SingleLowSpc,  3, 4 ,false},
+
 { ARM::VLD1q32HighQPseudo,  ARM::VLD1d32Q,     true,  false, false, SingleHighQSpc,  4, 2 ,false},
+{ ARM::VLD1q32HighQPseudo_UPD,  ARM::VLD1d32Qwb_fixed, true,  true, true, SingleHighQSpc,  4, 2 ,false},
 { ARM::VLD1q32HighTPseudo,  ARM::VLD1d32T,     true,  false, false, SingleHighTSpc,  3, 2 ,false},
+{ ARM::VLD1q32HighTPseudo_UPD,  ARM::VLD1d32Twb_fixed, true,  true, true, SingleHighTSpc,  3, 2 ,false},
 { ARM::VLD1q32LowQPseudo_UPD,  ARM::VLD1d32Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 2 ,false},
 { ARM::VLD1q32LowTPseudo_UPD,  ARM::VLD1d32Twb_fixed,   true,  true, true, SingleLowSpc,  3, 2 ,false},
+
 { ARM::VLD1q64HighQPseudo,  ARM::VLD1d64Q,     true,  false, false, SingleHighQSpc,  4, 1 ,false},
+{ ARM::VLD1q64HighQPseudo_UPD,  ARM::VLD1d64Qwb_fixed, true,  true, true, SingleHighQSpc,  4, 1 ,false},
 { ARM::VLD1q64HighTPseudo,  ARM::VLD1d64T,     true,  false, false, SingleHighTSpc,  3, 1 ,false},
+{ ARM::VLD1q64HighTPseudo_UPD,  ARM::VLD1d64Twb_fixed, true,  true, true, SingleHighTSpc,  3, 1 ,false},
 { ARM::VLD1q64LowQPseudo_UPD,  ARM::VLD1d64Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 1 ,false},
 { ARM::VLD1q64LowTPseudo_UPD,  ARM::VLD1d64Twb_fixed,   true,  true, true, SingleLowSpc,  3, 1 ,false},
+
 { ARM::VLD1q8HighQPseudo,   ARM::VLD1d8Q,     true,  false, false, SingleHighQSpc,  4, 8 ,false},
+{ ARM::VLD1q8HighQPseudo_UPD,   ARM::VLD1d8Qwb_fixed, true,  true, true, SingleHighQSpc,  4, 8 ,false},
 { ARM::VLD1q8HighTPseudo,   ARM::VLD1d8T,     true,  false, false, SingleHighTSpc,  3, 8 ,false},
+{ ARM::VLD1q8HighTPseudo_UPD,   ARM::VLD1d8Twb_fixed, true,  true, true, SingleHighTSpc,  3, 8 ,false},
 { ARM::VLD1q8LowQPseudo_UPD,  ARM::VLD1d8Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 8 ,false},
 { ARM::VLD1q8LowTPseudo_UPD,  ARM::VLD1d8Twb_fixed,   true,  true, true, SingleLowSpc,  3, 8 ,false},
 
 { ARM::VLD2DUPq16EvenPseudo,  ARM::VLD2DUPd16x2,  true, false, false, EvenDblSpc, 2, 4 ,false},
 { ARM::VLD2DUPq16OddPseudo,   ARM::VLD2DUPd16x2,  true, false, false, OddDblSpc,  2, 4 ,false},
 { ARM::VLD2DUPq32EvenPseudo,  ARM::VLD2DUPd32x2,  true, false, false, EvenDblSpc, 2, 2 ,false},
 { ARM::VLD2DUPq32OddPseudo,   ARM::VLD2DUPd32x2,  true, false, false, OddDblSpc,  2, 2 ,false},
 { ARM::VLD2DUPq8EvenPseudo,   ARM::VLD2DUPd8x2,   true, false, false, EvenDblSpc, 2, 8 ,false},
switch (Opcode) {
     case ARM::VLD2q32PseudoWB_fixed:
     case ARM::VLD2q8PseudoWB_register:
     case ARM::VLD2q16PseudoWB_register:
     case ARM::VLD2q32PseudoWB_register:
     case ARM::VLD3d8Pseudo:
     case ARM::VLD3d16Pseudo:
     case ARM::VLD3d32Pseudo:
     case ARM::VLD1d8TPseudo:
+    case ARM::VLD1d8TPseudoWB_fixed:
+    case ARM::VLD1d8TPseudoWB_register:
     case ARM::VLD1d16TPseudo:
+    case ARM::VLD1d16TPseudoWB_fixed:
+    case ARM::VLD1d16TPseudoWB_register:
     case ARM::VLD1d32TPseudo:
+    case ARM::VLD1d32TPseudoWB_fixed:
+    case ARM::VLD1d32TPseudoWB_register:
     case ARM::VLD1d64TPseudo:
     case ARM::VLD1d64TPseudoWB_fixed:
     case ARM::VLD1d64TPseudoWB_register:
     case ARM::VLD3d8Pseudo_UPD:
     case ARM::VLD3d16Pseudo_UPD:
     case ARM::VLD3d32Pseudo_UPD:
     case ARM::VLD3q8Pseudo_UPD:
     case ARM::VLD3q16Pseudo_UPD:
     case ARM::VLD3q32Pseudo_UPD:
     case ARM::VLD3q8oddPseudo:
     case ARM::VLD3q16oddPseudo:
     case ARM::VLD3q32oddPseudo:
     case ARM::VLD3q8oddPseudo_UPD:
     case ARM::VLD3q16oddPseudo_UPD:
     case ARM::VLD3q32oddPseudo_UPD:
     case ARM::VLD4d8Pseudo:
     case ARM::VLD4d16Pseudo:
     case ARM::VLD4d32Pseudo:
     case ARM::VLD1d8QPseudo:
+    case ARM::VLD1d8QPseudoWB_fixed:
+    case ARM::VLD1d8QPseudoWB_register:
     case ARM::VLD1d16QPseudo:
+    case ARM::VLD1d16QPseudoWB_fixed:
+    case ARM::VLD1d16QPseudoWB_register:
     case ARM::VLD1d32QPseudo:
+    case ARM::VLD1d32QPseudoWB_fixed:
+    case ARM::VLD1d32QPseudoWB_register:
     case ARM::VLD1d64QPseudo:
     case ARM::VLD1d64QPseudoWB_fixed:
     case ARM::VLD1d64QPseudoWB_register:
     case ARM::VLD1q8HighQPseudo:
+    case ARM::VLD1q8HighQPseudo_UPD:
     case ARM::VLD1q8LowQPseudo_UPD:
     case ARM::VLD1q8HighTPseudo:
+    case ARM::VLD1q8HighTPseudo_UPD:
     case ARM::VLD1q8LowTPseudo_UPD:
     case ARM::VLD1q16HighQPseudo:
+    case ARM::VLD1q16HighQPseudo_UPD:
     case ARM::VLD1q16LowQPseudo_UPD:
     case ARM::VLD1q16HighTPseudo:
+    case ARM::VLD1q16HighTPseudo_UPD:
     case ARM::VLD1q16LowTPseudo_UPD:
     case ARM::VLD1q32HighQPseudo:
+    case ARM::VLD1q32HighQPseudo_UPD:
     case ARM::VLD1q32LowQPseudo_UPD:
     case ARM::VLD1q32HighTPseudo:
+    case ARM::VLD1q32HighTPseudo_UPD:
     case ARM::VLD1q32LowTPseudo_UPD:
     case ARM::VLD1q64HighQPseudo:
+    case ARM::VLD1q64HighQPseudo_UPD:
     case ARM::VLD1q64LowQPseudo_UPD:
     case ARM::VLD1q64HighTPseudo:
+    case ARM::VLD1q64HighTPseudo_UPD:
     case ARM::VLD1q64LowTPseudo_UPD:
     case ARM::VLD4d8Pseudo_UPD:
     case ARM::VLD4d16Pseudo_UPD:
     case ARM::VLD4d32Pseudo_UPD:
     case ARM::VLD4q8Pseudo_UPD:
     case ARM::VLD4q16Pseudo_UPD:
     case ARM::VLD4q32Pseudo_UPD:
     case ARM::VLD4q8oddPseudo:

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines an instruction selector for the ARM target.

 {
   switch (Opc) {
   default: return false;
   case ARM::VLD1d8wb_fixed : return true;
   case ARM::VLD1d16wb_fixed : return true;
   case ARM::VLD1d64Qwb_fixed : return true;
   case ARM::VLD1d32wb_fixed : return true;
   case ARM::VLD1d64wb_fixed : return true;
+  case ARM::VLD1d8TPseudoWB_fixed : return true;
+  case ARM::VLD1d16TPseudoWB_fixed : return true;
+  case ARM::VLD1d32TPseudoWB_fixed : return true;
   case ARM::VLD1d64TPseudoWB_fixed : return true;
+  case ARM::VLD1d8QPseudoWB_fixed : return true;
+  case ARM::VLD1d16QPseudoWB_fixed : return true;
+  case ARM::VLD1d32QPseudoWB_fixed : return true;
   case ARM::VLD1d64QPseudoWB_fixed : return true;
   case ARM::VLD1q8wb_fixed : return true;
   case ARM::VLD1q16wb_fixed : return true;
   case ARM::VLD1q32wb_fixed : return true;
   case ARM::VLD1q64wb_fixed : return true;
   case ARM::VLD1DUPd8wb_fixed : return true;
   case ARM::VLD1DUPd16wb_fixed : return true;
   case ARM::VLD1DUPd32wb_fixed : return true;
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
   case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
   case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
   case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
   case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
   case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
   case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
   case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
   case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
+  case ARM::VLD1d8TPseudoWB_fixed: return ARM::VLD1d8TPseudoWB_register;
+  case ARM::VLD1d16TPseudoWB_fixed: return ARM::VLD1d16TPseudoWB_register;
+  case ARM::VLD1d32TPseudoWB_fixed: return ARM::VLD1d32TPseudoWB_register;
   case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
+  case ARM::VLD1d8QPseudoWB_fixed: return ARM::VLD1d8QPseudoWB_register;
+  case ARM::VLD1d16QPseudoWB_fixed: return ARM::VLD1d16QPseudoWB_register;
+  case ARM::VLD1d32QPseudoWB_fixed: return ARM::VLD1d32QPseudoWB_register;
   case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
   case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
   case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
   case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
   case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
   case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
   case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
 
if (Subtarget->hasNEON()) {
                                            ARM::MVE_VLD42_32,
                                            ARM::MVE_VLD43_32_wb};
       static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
       SelectMVE_VLD(N, 4, Opcodes, true);
     }
     return;
   }
 
+    case ARMISD::VLD1x2_UPD: {
+      if (Subtarget->hasNEON()) {
+        static const uint16_t DOpcodes[] = { ARM::VLD1q8wb_fixed,
+                                             ARM::VLD1q16wb_fixed,
+                                             ARM::VLD1q32wb_fixed,
+                                             ARM::VLD1q64wb_fixed };
+        static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudoWB_fixed,
+                                             ARM::VLD1d16QPseudoWB_fixed,
+                                             ARM::VLD1d32QPseudoWB_fixed,
+                                             ARM::VLD1d64QPseudoWB_fixed };
+        SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
+        return;
+      }
+      break;
+    }
+
+    case ARMISD::VLD1x3_UPD: {
+      if (Subtarget->hasNEON()) {
+        static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudoWB_fixed,
+                                             ARM::VLD1d16TPseudoWB_fixed,
+                                             ARM::VLD1d32TPseudoWB_fixed,
+                                             ARM::VLD1d64TPseudoWB_fixed };
+        static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
+                                              ARM::VLD1q16LowTPseudo_UPD,
+                                              ARM::VLD1q32LowTPseudo_UPD,
+                                              ARM::VLD1q64LowTPseudo_UPD };
+        static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo_UPD,
+                                              ARM::VLD1q16HighTPseudo_UPD,
+                                              ARM::VLD1q32HighTPseudo_UPD,
+                                              ARM::VLD1q64HighTPseudo_UPD };
+        SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
+        return;
+      }
+      break;
+    }
+
+    case ARMISD::VLD1x4_UPD: {
+      if (Subtarget->hasNEON()) {
+        static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudoWB_fixed,
+                                             ARM::VLD1d16QPseudoWB_fixed,
+                                             ARM::VLD1d32QPseudoWB_fixed,
+                                             ARM::VLD1d64QPseudoWB_fixed };
+        static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
+                                              ARM::VLD1q16LowQPseudo_UPD,
+                                              ARM::VLD1q32LowQPseudo_UPD,
+                                              ARM::VLD1q64LowQPseudo_UPD };
+        static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo_UPD,
+                                              ARM::VLD1q16HighQPseudo_UPD,
+                                              ARM::VLD1q32HighQPseudo_UPD,
+                                              ARM::VLD1q64HighQPseudo_UPD };
+        SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
+        return;
+      }
+      break;
+    }
+
+
   case ARMISD::VLD2LN_UPD: {
     static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
                                          ARM::VLD2LNd16Pseudo_UPD,
                                          ARM::VLD2LNd32Pseudo_UPD };
     static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
                                          ARM::VLD2LNq32Pseudo_UPD };
     SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
     return;

llvm/lib/Target/ARM/ARMISelLowering.h

enum NodeType : unsigned {
     VLD4_UPD,
     VLD2LN_UPD,
     VLD3LN_UPD,
     VLD4LN_UPD,
     VLD1DUP_UPD,
     VLD2DUP_UPD,
     VLD3DUP_UPD,
     VLD4DUP_UPD,
+    VLD1x2_UPD,
+    VLD1x3_UPD,
+    VLD1x4_UPD,
 
     // NEON stores with post-increment base updates:
     VST1_UPD,
     VST2_UPD,
     VST3_UPD,
     VST4_UPD,
     VST2LN_UPD,
     VST3LN_UPD,

llvm/lib/Target/ARM/ARMISelLowering.cpp

 //===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines the interfaces that ARM uses to lower LLVM code into a
case ARMISD::FIRST_NUMBER:
     MAKE_CASE(ARMISD::VLD1DUP)
     MAKE_CASE(ARMISD::VLD2DUP)
     MAKE_CASE(ARMISD::VLD3DUP)
     MAKE_CASE(ARMISD::VLD4DUP)
     MAKE_CASE(ARMISD::VLD1_UPD)
     MAKE_CASE(ARMISD::VLD2_UPD)
     MAKE_CASE(ARMISD::VLD3_UPD)
     MAKE_CASE(ARMISD::VLD4_UPD)
+    MAKE_CASE(ARMISD::VLD1x2_UPD)
+    MAKE_CASE(ARMISD::VLD1x3_UPD)
+    MAKE_CASE(ARMISD::VLD1x4_UPD)
     MAKE_CASE(ARMISD::VLD2LN_UPD)
     MAKE_CASE(ARMISD::VLD3LN_UPD)
     MAKE_CASE(ARMISD::VLD4LN_UPD)
     MAKE_CASE(ARMISD::VLD1DUP_UPD)
     MAKE_CASE(ARMISD::VLD2DUP_UPD)
     MAKE_CASE(ARMISD::VLD3DUP_UPD)
     MAKE_CASE(ARMISD::VLD4DUP_UPD)
     MAKE_CASE(ARMISD::VST1_UPD)
for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
     Worklist.push_back(User);
     if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
         SDNode::hasPredecessorHelper(User, Visited, Worklist))
       continue;
 
     // Find the new opcode for the updating load/store.
     bool isLoadOp = true;
     bool isLaneOp = false;
-    // Workaround for vst1x and vld1x which do not have alignment operand.
+    // Workaround for vst1x and vld1x intrinsics which do not have alignment
+    // as an operand.
     bool hasAlignment = true;
     unsigned NewOpc = 0;
     unsigned NumVecs = 0;
     if (isIntrinsic) {
       unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
       switch (IntNo) {
       default: llvm_unreachable("unexpected intrinsic for Neon base update");
       case Intrinsic::arm_neon_vld1:     NewOpc = ARMISD::VLD1_UPD;
         NumVecs = 1; break;
       case Intrinsic::arm_neon_vld2:     NewOpc = ARMISD::VLD2_UPD;
         NumVecs = 2; break;
       case Intrinsic::arm_neon_vld3:     NewOpc = ARMISD::VLD3_UPD;
         NumVecs = 3; break;
       case Intrinsic::arm_neon_vld4:     NewOpc = ARMISD::VLD4_UPD;
         NumVecs = 4; break;
-      case Intrinsic::arm_neon_vld1x2:
-      case Intrinsic::arm_neon_vld1x3:
-      case Intrinsic::arm_neon_vld1x4:
+      case Intrinsic::arm_neon_vld1x2:   NewOpc = ARMISD::VLD1x2_UPD;
+        NumVecs = 2; hasAlignment = false; break;
+      case Intrinsic::arm_neon_vld1x3:   NewOpc = ARMISD::VLD1x3_UPD;
+        NumVecs = 3; hasAlignment = false; break;
+      case Intrinsic::arm_neon_vld1x4:   NewOpc = ARMISD::VLD1x4_UPD;
+        NumVecs = 4; hasAlignment = false; break;
       case Intrinsic::arm_neon_vld2dup:
       case Intrinsic::arm_neon_vld3dup:
       case Intrinsic::arm_neon_vld4dup:
-        // TODO: Support updating VLD1x and VLDxDUP nodes. For now, we just skip
+        // TODO: Support updating VLDxDUP nodes. For now, we just skip
         // combining base updates for such intrinsics.
         continue;
       case Intrinsic::arm_neon_vld2lane: NewOpc = ARMISD::VLD2LN_UPD;
         NumVecs = 2; isLaneOp = true; break;
       case Intrinsic::arm_neon_vld3lane: NewOpc = ARMISD::VLD3LN_UPD;
         NumVecs = 3; isLaneOp = true; break;
       case Intrinsic::arm_neon_vld4lane: NewOpc = ARMISD::VLD4LN_UPD;
         NumVecs = 4; isLaneOp = true; break;

llvm/lib/Target/ARM/ARMInstrNEON.td

 def VLD1d32T     : VLD1D3<{1,0,0,?}, "32", addrmode6align64>;
 def VLD1d64T     : VLD1D3<{1,1,0,?}, "64", addrmode6align64>;
 
 defm VLD1d8Twb  : VLD1D3WB<{0,0,0,?}, "8",  addrmode6align64>;
 defm VLD1d16Twb : VLD1D3WB<{0,1,0,?}, "16", addrmode6align64>;
 defm VLD1d32Twb : VLD1D3WB<{1,0,0,?}, "32", addrmode6align64>;
 defm VLD1d64Twb : VLD1D3WB<{1,1,0,?}, "64", addrmode6align64>;
 
-def VLD1d8TPseudo  : VLDQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1d16TPseudo : VLDQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1d32TPseudo : VLDQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1d64TPseudo : VLDQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1d64TPseudoWB_fixed : VLDQQWBfixedPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d8TPseudo             : VLDQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d8TPseudoWB_fixed     : VLDQQWBfixedPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d8TPseudoWB_register  : VLDQQWBregisterPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d16TPseudo            : VLDQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d16TPseudoWB_fixed    : VLDQQWBfixedPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d16TPseudoWB_register : VLDQQWBregisterPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d32TPseudo            : VLDQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d32TPseudoWB_fixed    : VLDQQWBfixedPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d32TPseudoWB_register : VLDQQWBregisterPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d64TPseudo            : VLDQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1d64TPseudoWB_fixed    : VLDQQWBfixedPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
 def VLD1d64TPseudoWB_register : VLDQQWBregisterPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
 
-def VLD1q8HighTPseudo     : VLDQQQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1q8LowTPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1q16HighTPseudo    : VLDQQQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1q16LowTPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1q32HighTPseudo    : VLDQQQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1q32LowTPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1q64HighTPseudo    : VLDQQQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
-def VLD1q64LowTPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q8HighTPseudo      : VLDQQQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q8HighTPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q8LowTPseudo_UPD   : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q16HighTPseudo     : VLDQQQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q16HighTPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q16LowTPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q32HighTPseudo     : VLDQQQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q32HighTPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q32LowTPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q64HighTPseudo     : VLDQQQQPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q64HighTPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
+def VLD1q64LowTPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x3>, Sched<[WriteVLD3]>;
 
 // ...with 4 registers
 class VLD1D4<bits<4> op7_4, string Dt, Operand AddrMode>
   : NLdSt<0, 0b10, 0b0010, op7_4, (outs VecListFourD:$Vd),
           (ins AddrMode:$Rn), IIC_VLD1x4, "vld1", Dt,
           "$Vd, $Rn", "", []>, Sched<[WriteVLD4]> {
   let Rm = 0b1111;
   let Inst{5-4} = Rn{5-4};

 def VLD1d32Q     : VLD1D4<{1,0,?,?}, "32", addrmode6align64or128or256>;
 def VLD1d64Q     : VLD1D4<{1,1,?,?}, "64", addrmode6align64or128or256>;
 
 defm VLD1d8Qwb   : VLD1D4WB<{0,0,?,?}, "8",  addrmode6align64or128or256>;
 defm VLD1d16Qwb  : VLD1D4WB<{0,1,?,?}, "16", addrmode6align64or128or256>;
 defm VLD1d32Qwb  : VLD1D4WB<{1,0,?,?}, "32", addrmode6align64or128or256>;
 defm VLD1d64Qwb  : VLD1D4WB<{1,1,?,?}, "64", addrmode6align64or128or256>;
 
-def VLD1d8QPseudo  : VLDQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1d16QPseudo : VLDQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1d32QPseudo : VLDQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1d64QPseudo : VLDQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1d64QPseudoWB_fixed : VLDQQWBfixedPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d8QPseudo             : VLDQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d8QPseudoWB_fixed     : VLDQQWBfixedPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d8QPseudoWB_register  : VLDQQWBregisterPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d16QPseudo            : VLDQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d16QPseudoWB_fixed    : VLDQQWBfixedPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d16QPseudoWB_register : VLDQQWBregisterPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d32QPseudo            : VLDQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d32QPseudoWB_fixed    : VLDQQWBfixedPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d32QPseudoWB_register : VLDQQWBregisterPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d64QPseudo            : VLDQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1d64QPseudoWB_fixed    : VLDQQWBfixedPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
 def VLD1d64QPseudoWB_register : VLDQQWBregisterPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
 
-def VLD1q8LowQPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1q8HighQPseudo     : VLDQQQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1q16LowQPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1q16HighQPseudo    : VLDQQQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1q32LowQPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1q32HighQPseudo    : VLDQQQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1q64LowQPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
-def VLD1q64HighQPseudo    : VLDQQQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q8LowQPseudo_UPD   : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q8HighQPseudo      : VLDQQQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q8HighQPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q16LowQPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q16HighQPseudo     : VLDQQQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q16HighQPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q32LowQPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q32HighQPseudo     : VLDQQQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q32HighQPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q64LowQPseudo_UPD  : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q64HighQPseudo     : VLDQQQQPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
+def VLD1q64HighQPseudo_UPD : VLDQQQQWBPseudo<IIC_VLD1x4>, Sched<[WriteVLD4]>;
 
 //   VLD2     : Vector Load (multiple 2-element structures)
 class VLD2<bits<4> op11_8, bits<4> op7_4, string Dt, RegisterOperand VdTy,
            InstrItinClass itin, Operand AddrMode>
   : NLdSt<0, 0b10, op11_8, op7_4, (outs VdTy:$Vd),
           (ins AddrMode:$Rn), itin,
           "vld2", Dt, "$Vd, $Rn", "", []> {
   let Rm = 0b1111;

llvm/test/CodeGen/ARM/arm-vld1.ll

 
 ; CHECK-LABEL: test_vld1q_u8_x4
 ; CHECK: vld1.8 {d16, d17, d18, d19}, [r1:256]!
 ; CHECK: vld1.8 {d20, d21, d22, d23}, [r1:256]
 define %struct.uint8x16x4_t @test_vld1q_u8_x4(i8* %a) nounwind {
   %tmp = tail call %struct.uint8x16x4_t @llvm.arm.neon.vld1x4.v16i8.p0i8(i8* %a)
   ret %struct.uint8x16x4_t %tmp
 }
+
+; Post-increment.
+
+define %struct.uint16x4x2_t @test_vld1_u16_x2_post_imm(i16* %a, i16** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u16_x2_post_imm:
+; CHECK:         vld1.16 {d16, d17}, [r0:64]!
+  %ld = tail call %struct.uint16x4x2_t @llvm.arm.neon.vld1x2.v4i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 8
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x4x2_t %ld
+}
+
+define %struct.uint16x4x2_t @test_vld1_u16_x2_post_reg(i16* %a, i16** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u16_x2_post_reg:
+; CHECK:         lsl r2, r2, #1
+; CHECK-NEXT:    vld1.16 {d16, d17}, [r0:64], r2
+  %ld = tail call %struct.uint16x4x2_t @llvm.arm.neon.vld1x2.v4i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 %inc
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x4x2_t %ld
+}
+
+define %struct.uint16x4x3_t @test_vld1_u16_x3_post_imm(i16* %a, i16** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u16_x3_post_imm:
+; CHECK:         vld1.16 {d16, d17, d18}, [r1:64]!
+  %ld = tail call %struct.uint16x4x3_t @llvm.arm.neon.vld1x3.v4i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 12
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x4x3_t %ld
+}
+
+define %struct.uint16x4x3_t @test_vld1_u16_x3_post_reg(i16* %a, i16** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u16_x3_post_reg:
+; CHECK:         lsl r3, r3, #1
+; CHECK-NEXT:    vld1.16 {d16, d17, d18}, [r1:64], r3
+  %ld = tail call %struct.uint16x4x3_t @llvm.arm.neon.vld1x3.v4i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 %inc
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x4x3_t %ld
+}
+
+define %struct.uint16x4x4_t @test_vld1_u16_x4_post_imm(i16* %a, i16** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u16_x4_post_imm:
+; CHECK:         vld1.16 {d16, d17, d18, d19}, [r1:256]!
+  %ld = tail call %struct.uint16x4x4_t @llvm.arm.neon.vld1x4.v4i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 16
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x4x4_t %ld
+}
+
+define %struct.uint16x4x4_t @test_vld1_u16_x4_post_reg(i16* %a, i16** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u16_x4_post_reg:
+; CHECK:         lsl r3, r3, #1
+; CHECK-NEXT:    vld1.16 {d16, d17, d18, d19}, [r1:256], r3
+  %ld = tail call %struct.uint16x4x4_t @llvm.arm.neon.vld1x4.v4i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 %inc
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x4x4_t %ld
+}
+
+define %struct.uint32x2x2_t @test_vld1_u32_x2_post_imm(i32* %a, i32** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u32_x2_post_imm:
+; CHECK:         vld1.32 {d16, d17}, [r0:64]!
+  %ld = tail call %struct.uint32x2x2_t @llvm.arm.neon.vld1x2.v2i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 4
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x2x2_t %ld
+}
+
+define %struct.uint32x2x2_t @test_vld1_u32_x2_post_reg(i32* %a, i32** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u32_x2_post_reg:
+; CHECK:         lsl r2, r2, #2
+; CHECK-NEXT:    vld1.32 {d16, d17}, [r0:64], r2
+  %ld = tail call %struct.uint32x2x2_t @llvm.arm.neon.vld1x2.v2i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 %inc
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x2x2_t %ld
+}
+
+define %struct.uint32x2x3_t @test_vld1_u32_x3_post_imm(i32* %a, i32** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u32_x3_post_imm:
+; CHECK:         vld1.32 {d16, d17, d18}, [r1:64]!
+  %ld = tail call %struct.uint32x2x3_t @llvm.arm.neon.vld1x3.v2i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 6
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x2x3_t %ld
+}
+
+define %struct.uint32x2x3_t @test_vld1_u32_x3_post_reg(i32* %a, i32** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u32_x3_post_reg:
+; CHECK:         lsl r3, r3, #2
+; CHECK-NEXT:    vld1.32 {d16, d17, d18}, [r1:64], r3
+  %ld = tail call %struct.uint32x2x3_t @llvm.arm.neon.vld1x3.v2i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 %inc
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x2x3_t %ld
+}
+
+define %struct.uint32x2x4_t @test_vld1_u32_x4_post_imm(i32* %a, i32** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u32_x4_post_imm:
+; CHECK:         vld1.32 {d16, d17, d18, d19}, [r1:256]!
+  %ld = tail call %struct.uint32x2x4_t @llvm.arm.neon.vld1x4.v2i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 8
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x2x4_t %ld
+}
+
+define %struct.uint32x2x4_t @test_vld1_u32_x4_post_reg(i32* %a, i32** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u32_x4_post_reg:
+; CHECK:         lsl r3, r3, #2
+; CHECK-NEXT:    vld1.32 {d16, d17, d18, d19}, [r1:256], r3
+  %ld = tail call %struct.uint32x2x4_t @llvm.arm.neon.vld1x4.v2i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 %inc
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x2x4_t %ld
+}
+
+define %struct.uint64x1x2_t @test_vld1_u64_x2_post_imm(i64* %a, i64** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u64_x2_post_imm:
+; CHECK:         vld1.64 {d16, d17}, [r0:64]!
+  %ld = tail call %struct.uint64x1x2_t @llvm.arm.neon.vld1x2.v1i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 2
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x1x2_t %ld
+}
+
+define %struct.uint64x1x2_t @test_vld1_u64_x2_post_reg(i64* %a, i64** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u64_x2_post_reg:
+; CHECK:         lsl r2, r2, #3
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r0:64], r2
+  %ld = tail call %struct.uint64x1x2_t @llvm.arm.neon.vld1x2.v1i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 %inc
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x1x2_t %ld
+}
+
+define %struct.uint64x1x3_t @test_vld1_u64_x3_post_imm(i64* %a, i64** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u64_x3_post_imm:
+; CHECK:         vld1.64 {d16, d17, d18}, [r1:64]!
+  %ld = tail call %struct.uint64x1x3_t @llvm.arm.neon.vld1x3.v1i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 3
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x1x3_t %ld
+}
+
+define %struct.uint64x1x3_t @test_vld1_u64_x3_post_reg(i64* %a, i64** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u64_x3_post_reg:
+; CHECK:         lsl r3, r3, #3
+; CHECK-NEXT:    vld1.64 {d16, d17, d18}, [r1:64], r3
+  %ld = tail call %struct.uint64x1x3_t @llvm.arm.neon.vld1x3.v1i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 %inc
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x1x3_t %ld
+}
+
+define %struct.uint64x1x4_t @test_vld1_u64_x4_post_imm(i64* %a, i64** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u64_x4_post_imm:
+; CHECK:         vld1.64 {d16, d17, d18, d19}, [r1:256]!
+  %ld = tail call %struct.uint64x1x4_t @llvm.arm.neon.vld1x4.v1i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 4
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x1x4_t %ld
+}
+
+define %struct.uint64x1x4_t @test_vld1_u64_x4_post_reg(i64* %a, i64** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u64_x4_post_reg:
+; CHECK:         lsl r3, r3, #3
+; CHECK-NEXT:    vld1.64 {d16, d17, d18, d19}, [r1:256], r3
+  %ld = tail call %struct.uint64x1x4_t @llvm.arm.neon.vld1x4.v1i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 %inc
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x1x4_t %ld
+}
+
+define %struct.uint8x8x2_t @test_vld1_u8_x2_post_imm(i8* %a, i8** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u8_x2_post_imm:
+; CHECK:         vld1.8 {d16, d17}, [r0:64]!
+  %ld = tail call %struct.uint8x8x2_t @llvm.arm.neon.vld1x2.v8i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 16
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x8x2_t %ld
+}
+
+define %struct.uint8x8x2_t @test_vld1_u8_x2_post_reg(i8* %a, i8** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u8_x2_post_reg:
+; CHECK:         vld1.8 {d16, d17}, [r0:64], r2
+  %ld = tail call %struct.uint8x8x2_t @llvm.arm.neon.vld1x2.v8i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 %inc
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x8x2_t %ld
+}
+
+define %struct.uint8x8x3_t @test_vld1_u8_x3_post_imm(i8* %a, i8** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u8_x3_post_imm:
+; CHECK:         vld1.8 {d16, d17, d18}, [r1:64]!
+  %ld = tail call %struct.uint8x8x3_t @llvm.arm.neon.vld1x3.v8i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 24
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x8x3_t %ld
+}
+
+define %struct.uint8x8x3_t @test_vld1_u8_x3_post_reg(i8* %a, i8** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u8_x3_post_reg:
+; CHECK:         vld1.8 {d16, d17, d18}, [r1:64], r3
+  %ld = tail call %struct.uint8x8x3_t @llvm.arm.neon.vld1x3.v8i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 %inc
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x8x3_t %ld
+}
+
+define %struct.uint8x8x4_t @test_vld1_u8_x4_post_imm(i8* %a, i8** %ptr) nounwind {
+; CHECK-LABEL: test_vld1_u8_x4_post_imm:
+; CHECK:         vld1.8 {d16, d17, d18, d19}, [r1:256]!
+  %ld = tail call %struct.uint8x8x4_t @llvm.arm.neon.vld1x4.v8i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 32
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x8x4_t %ld
+}
+
+define %struct.uint8x8x4_t @test_vld1_u8_x4_post_reg(i8* %a, i8** %ptr, i32 %inc) nounwind {
+; CHECK-LABEL: test_vld1_u8_x4_post_reg:
+; CHECK:         vld1.8 {d16, d17, d18, d19}, [r1:256], r3
+  %ld = tail call %struct.uint8x8x4_t @llvm.arm.neon.vld1x4.v8i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 %inc
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x8x4_t %ld
+}
+
+define %struct.uint16x8x2_t @test_vld1q_u16_x2_post_imm(i16* %a, i16** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u16_x2_post_imm:
+; CHECK:         vld1.16 {d16, d17, d18, d19}, [r1:256]!
+  %ld = tail call %struct.uint16x8x2_t @llvm.arm.neon.vld1x2.v8i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 16
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x8x2_t %ld
+}
+
+define %struct.uint16x8x3_t @test_vld1q_u16_x3_post_imm(i16* %a, i16** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u16_x3_post_imm:
+; CHECK:         vld1.16 {d16, d17, d18}, [r1:64]!
+; CHECK-NEXT:    vld1.16 {d19, d20, d21}, [r1:64]!
+  %ld = tail call %struct.uint16x8x3_t @llvm.arm.neon.vld1x3.v8i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 24
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x8x3_t %ld
+}
+
+define %struct.uint16x8x4_t @test_vld1q_u16_x4_post_imm(i16* %a, i16** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u16_x4_post_imm:
+; CHECK:         vld1.16 {d16, d17, d18, d19}, [r1:256]!
+; CHECK-NEXT:    vld1.16 {d20, d21, d22, d23}, [r1:256]!
+  %ld = tail call %struct.uint16x8x4_t @llvm.arm.neon.vld1x4.v8i16.p0i16(i16* %a)
+  %tmp = getelementptr i16, i16* %a, i32 32
+  store i16* %tmp, i16** %ptr
+  ret %struct.uint16x8x4_t %ld
+}
+
+define %struct.uint32x4x2_t @test_vld1q_u32_x2_post_imm(i32* %a, i32** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u32_x2_post_imm:
+; CHECK:         vld1.32 {d16, d17, d18, d19}, [r1:256]!
+  %ld = tail call %struct.uint32x4x2_t @llvm.arm.neon.vld1x2.v4i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 8
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x4x2_t %ld
+}
+
+define %struct.uint32x4x3_t @test_vld1q_u32_x3_post_imm(i32* %a, i32** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u32_x3_post_imm:
+; CHECK:         vld1.32 {d16, d17, d18}, [r1:64]!
+; CHECK-NEXT:    vld1.32 {d19, d20, d21}, [r1:64]!
+  %ld = tail call %struct.uint32x4x3_t @llvm.arm.neon.vld1x3.v4i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 12
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x4x3_t %ld
+}
+
+define %struct.uint32x4x4_t @test_vld1q_u32_x4_post_imm(i32* %a, i32** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u32_x4_post_imm:
+; CHECK:         vld1.32 {d16, d17, d18, d19}, [r1:256]!
+; CHECK-NEXT:    vld1.32 {d20, d21, d22, d23}, [r1:256]!
+  %ld = tail call %struct.uint32x4x4_t @llvm.arm.neon.vld1x4.v4i32.p0i32(i32* %a)
+  %tmp = getelementptr i32, i32* %a, i32 16
+  store i32* %tmp, i32** %ptr
+  ret %struct.uint32x4x4_t %ld
+}
+
+define %struct.uint64x2x2_t @test_vld1q_u64_x2_post_imm(i64* %a, i64** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u64_x2_post_imm:
+; CHECK:         vld1.64 {d16, d17, d18, d19}, [r1:256]!
+  %ld = tail call %struct.uint64x2x2_t @llvm.arm.neon.vld1x2.v2i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 4
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x2x2_t %ld
+}
+
+define %struct.uint64x2x3_t @test_vld1q_u64_x3_post_imm(i64* %a, i64** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u64_x3_post_imm:
+; CHECK:         vld1.64 {d16, d17, d18}, [r1:64]!
+; CHECK-NEXT:    vld1.64 {d19, d20, d21}, [r1:64]!
+  %ld = tail call %struct.uint64x2x3_t @llvm.arm.neon.vld1x3.v2i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 6
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x2x3_t %ld
+}
+
+define %struct.uint64x2x4_t @test_vld1q_u64_x4_post_imm(i64* %a, i64** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u64_x4_post_imm:
+; CHECK:         vld1.64 {d16, d17, d18, d19}, [r1:256]!
+; CHECK-NEXT:    vld1.64 {d20, d21, d22, d23}, [r1:256]!
+  %ld = tail call %struct.uint64x2x4_t @llvm.arm.neon.vld1x4.v2i64.p0i64(i64* %a)
+  %tmp = getelementptr i64, i64* %a, i32 8
+  store i64* %tmp, i64** %ptr
+  ret %struct.uint64x2x4_t %ld
+}
+
+define %struct.uint8x16x2_t @test_vld1q_u8_x2_post_imm(i8* %a, i8** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u8_x2_post_imm:
+; CHECK:         vld1.8 {d16, d17, d18, d19}, [r1:256]!
+  %ld = tail call %struct.uint8x16x2_t @llvm.arm.neon.vld1x2.v16i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 32
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x16x2_t %ld
+}
+
+define %struct.uint8x16x3_t @test_vld1q_u8_x3_post_imm(i8* %a, i8** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u8_x3_post_imm:
+; CHECK:         vld1.8 {d16, d17, d18}, [r1:64]!
+; CHECK-NEXT:    vld1.8 {d19, d20, d21}, [r1:64]!
+  %ld = tail call %struct.uint8x16x3_t @llvm.arm.neon.vld1x3.v16i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 48
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x16x3_t %ld
+}
+
+define %struct.uint8x16x4_t @test_vld1q_u8_x4_post_imm(i8* %a, i8** %ptr) nounwind {
+; CHECK-LABEL: test_vld1q_u8_x4_post_imm:
+; CHECK:         vld1.8 {d16, d17, d18, d19}, [r1:256]!
+; CHECK-NEXT:    vld1.8 {d20, d21, d22, d23}, [r1:256]!
+  %ld = tail call %struct.uint8x16x4_t @llvm.arm.neon.vld1x4.v16i8.p0i8(i8* %a)
+  %tmp = getelementptr i8, i8* %a, i32 64
+  store i8* %tmp, i8** %ptr
+  ret %struct.uint8x16x4_t %ld
+}