[ARM][NEON] Combine base address updates for vst1x intrinsics

[krisb]

Oh, I didn't notice the recent patch https://reviews.llvm.org/rG76786037c68163c48d7d829bb654de6c8298bbb0.
@dmgreen, could I ask you to take a look at this? I'll rebase the patch on the top of 'main' soon.

[dmgreen]

Oh yeah. Sounds great. It will certainly do better that just disabling the postinc!

[krisb]

Rebased on the top of main.

[dmgreen]

This LGTM, from what I can tell. Thanks for the patch.

llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp

struct NEONLdStTableEntry {
     friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned PseudoOpc,
                                                 const NEONLdStTableEntry &TE) {
       return PseudoOpc < TE.PseudoOpc;
     }
   };
 }
 
 static const NEONLdStTableEntry NEONLdStTable[] = {
 { ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
 { ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
 { ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
 { ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
 { ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
 { ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
 
 { ARM::VLD1d16QPseudo,      ARM::VLD1d16Q,     true,  false, false, SingleSpc,  4, 4 ,false},
 { ARM::VLD1d16TPseudo,      ARM::VLD1d16T,     true,  false, false, SingleSpc,  3, 4 ,false},

 { ARM::VST1LNq16Pseudo,     ARM::VST1LNd16,    false, false, false, EvenDblSpc, 1, 4 ,true},
 { ARM::VST1LNq16Pseudo_UPD, ARM::VST1LNd16_UPD, false, true, true,  EvenDblSpc, 1, 4 ,true},
 { ARM::VST1LNq32Pseudo,     ARM::VST1LNd32,    false, false, false, EvenDblSpc, 1, 2 ,true},
 { ARM::VST1LNq32Pseudo_UPD, ARM::VST1LNd32_UPD, false, true, true,  EvenDblSpc, 1, 2 ,true},
 { ARM::VST1LNq8Pseudo,      ARM::VST1LNd8,     false, false, false, EvenDblSpc, 1, 8 ,true},
 { ARM::VST1LNq8Pseudo_UPD,  ARM::VST1LNd8_UPD, false, true, true,  EvenDblSpc, 1, 8 ,true},
 
 { ARM::VST1d16QPseudo,      ARM::VST1d16Q,     false, false, false, SingleSpc,  4, 4 ,false},
+{ ARM::VST1d16QPseudoWB_fixed,  ARM::VST1d16Qwb_fixed, false, true, false, SingleSpc,  4, 4 ,false},
+{ ARM::VST1d16QPseudoWB_register, ARM::VST1d16Qwb_register, false, true, true, SingleSpc,  4, 4 ,false},
 { ARM::VST1d16TPseudo,      ARM::VST1d16T,     false, false, false, SingleSpc,  3, 4 ,false},
+{ ARM::VST1d16TPseudoWB_fixed,  ARM::VST1d16Twb_fixed, false, true, false, SingleSpc,  3, 4 ,false},
+{ ARM::VST1d16TPseudoWB_register, ARM::VST1d16Twb_register, false, true, true, SingleSpc,  3, 4 ,false},
+
 { ARM::VST1d32QPseudo,      ARM::VST1d32Q,     false, false, false, SingleSpc,  4, 2 ,false},
+{ ARM::VST1d32QPseudoWB_fixed,  ARM::VST1d32Qwb_fixed, false, true, false, SingleSpc,  4, 2 ,false},
+{ ARM::VST1d32QPseudoWB_register, ARM::VST1d32Qwb_register, false, true, true, SingleSpc,  4, 2 ,false},
 { ARM::VST1d32TPseudo,      ARM::VST1d32T,     false, false, false, SingleSpc,  3, 2 ,false},
+{ ARM::VST1d32TPseudoWB_fixed,  ARM::VST1d32Twb_fixed, false, true, false, SingleSpc,  3, 2 ,false},
+{ ARM::VST1d32TPseudoWB_register, ARM::VST1d32Twb_register, false, true, true, SingleSpc,  3, 2 ,false},
+
 { ARM::VST1d64QPseudo,      ARM::VST1d64Q,     false, false, false, SingleSpc,  4, 1 ,false},
 { ARM::VST1d64QPseudoWB_fixed,  ARM::VST1d64Qwb_fixed, false, true, false,  SingleSpc,  4, 1 ,false},
 { ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true,  SingleSpc,  4, 1 ,false},
 { ARM::VST1d64TPseudo,      ARM::VST1d64T,     false, false, false, SingleSpc,  3, 1 ,false},
 { ARM::VST1d64TPseudoWB_fixed,  ARM::VST1d64Twb_fixed, false, true, false,  SingleSpc,  3, 1 ,false},
-{ ARM::VST1d64TPseudoWB_register,  ARM::VST1d64Twb_register, false, true, true,  SingleSpc,  3, 1 ,false},
+{ ARM::VST1d64TPseudoWB_register, ARM::VST1d64Twb_register, false, true, true,  SingleSpc,  3, 1 ,false},
+
 { ARM::VST1d8QPseudo,       ARM::VST1d8Q,      false, false, false, SingleSpc,  4, 8 ,false},
+{ ARM::VST1d8QPseudoWB_fixed,   ARM::VST1d8Qwb_fixed, false, true, false, SingleSpc,  4, 8 ,false},
+{ ARM::VST1d8QPseudoWB_register,  ARM::VST1d8Qwb_register, false, true, true, SingleSpc,  4, 8 ,false},
 { ARM::VST1d8TPseudo,       ARM::VST1d8T,      false, false, false, SingleSpc,  3, 8 ,false},
-{ ARM::VST1q16HighQPseudo,  ARM::VST1d16Q,      false, false, false, SingleHighQSpc,   4, 4 ,false},
-{ ARM::VST1q16HighTPseudo,  ARM::VST1d16T,      false, false, false, SingleHighTSpc,   3, 4 ,false},
+{ ARM::VST1d8TPseudoWB_fixed,   ARM::VST1d8Twb_fixed, false, true, false, SingleSpc,  3, 8 ,false},
+{ ARM::VST1d8TPseudoWB_register,  ARM::VST1d8Twb_register, false, true, true, SingleSpc,  3, 8 ,false},
+
+{ ARM::VST1q16HighQPseudo,  ARM::VST1d16Q,     false, false, false, SingleHighQSpc,   4, 4 ,false},
+{ ARM::VST1q16HighQPseudo_UPD,  ARM::VST1d16Qwb_fixed,  false, true, true, SingleHighQSpc,   4, 8 ,false},
+{ ARM::VST1q16HighTPseudo,  ARM::VST1d16T,     false, false, false, SingleHighTSpc,   3, 4 ,false},
+{ ARM::VST1q16HighTPseudo_UPD,  ARM::VST1d16Twb_fixed,  false, true, true, SingleHighTSpc,   3, 4 ,false},
 { ARM::VST1q16LowQPseudo_UPD,   ARM::VST1d16Qwb_fixed,  false, true, true, SingleLowSpc,   4, 4 ,false},
 { ARM::VST1q16LowTPseudo_UPD,   ARM::VST1d16Twb_fixed,  false, true, true, SingleLowSpc,   3, 4 ,false},
-{ ARM::VST1q32HighQPseudo,  ARM::VST1d32Q,      false, false, false, SingleHighQSpc,   4, 2 ,false},
-{ ARM::VST1q32HighTPseudo,  ARM::VST1d32T,      false, false, false, SingleHighTSpc,   3, 2 ,false},
+
+{ ARM::VST1q32HighQPseudo,  ARM::VST1d32Q,     false, false, false, SingleHighQSpc,   4, 2 ,false},
+{ ARM::VST1q32HighQPseudo_UPD,  ARM::VST1d32Qwb_fixed,  false, true, true, SingleHighQSpc,   4, 8 ,false},
+{ ARM::VST1q32HighTPseudo,  ARM::VST1d32T,     false, false, false, SingleHighTSpc,   3, 2 ,false},
+{ ARM::VST1q32HighTPseudo_UPD,  ARM::VST1d32Twb_fixed,  false, true, true, SingleHighTSpc,   3, 2 ,false},
 { ARM::VST1q32LowQPseudo_UPD,   ARM::VST1d32Qwb_fixed,  false, true, true, SingleLowSpc,   4, 2 ,false},
 { ARM::VST1q32LowTPseudo_UPD,   ARM::VST1d32Twb_fixed,  false, true, true, SingleLowSpc,   3, 2 ,false},
-{ ARM::VST1q64HighQPseudo,  ARM::VST1d64Q,      false, false, false, SingleHighQSpc,   4, 1 ,false},
-{ ARM::VST1q64HighTPseudo,  ARM::VST1d64T,      false, false, false, SingleHighTSpc,   3, 1 ,false},
+
+{ ARM::VST1q64HighQPseudo,  ARM::VST1d64Q,     false, false, false, SingleHighQSpc,   4, 1 ,false},
+{ ARM::VST1q64HighQPseudo_UPD,  ARM::VST1d64Qwb_fixed,  false, true, true, SingleHighQSpc,   4, 8 ,false},
+{ ARM::VST1q64HighTPseudo,  ARM::VST1d64T,     false, false, false, SingleHighTSpc,   3, 1 ,false},
+{ ARM::VST1q64HighTPseudo_UPD,  ARM::VST1d64Twb_fixed,  false, true, true, SingleHighTSpc,   3, 1 ,false},
 { ARM::VST1q64LowQPseudo_UPD,   ARM::VST1d64Qwb_fixed,  false, true, true, SingleLowSpc,   4, 1 ,false},
 { ARM::VST1q64LowTPseudo_UPD,   ARM::VST1d64Twb_fixed,  false, true, true, SingleLowSpc,   3, 1 ,false},
+
 { ARM::VST1q8HighQPseudo,   ARM::VST1d8Q,      false, false, false, SingleHighQSpc,   4, 8 ,false},
+{ ARM::VST1q8HighQPseudo_UPD,  ARM::VST1d8Qwb_fixed,  false, true, true, SingleHighQSpc,   4, 8 ,false},
 { ARM::VST1q8HighTPseudo,   ARM::VST1d8T,      false, false, false, SingleHighTSpc,   3, 8 ,false},
+{ ARM::VST1q8HighTPseudo_UPD,  ARM::VST1d8Twb_fixed,  false, true, true, SingleHighTSpc,   3, 8 ,false},
 { ARM::VST1q8LowQPseudo_UPD,   ARM::VST1d8Qwb_fixed,  false, true, true, SingleLowSpc,   4, 8 ,false},
 { ARM::VST1q8LowTPseudo_UPD,   ARM::VST1d8Twb_fixed,  false, true, true, SingleLowSpc,   3, 8 ,false},
 
 { ARM::VST2LNd16Pseudo,     ARM::VST2LNd16,     false, false, false, SingleSpc, 2, 4 ,true},
 { ARM::VST2LNd16Pseudo_UPD, ARM::VST2LNd16_UPD, false, true, true,  SingleSpc, 2, 4 ,true},
 { ARM::VST2LNd32Pseudo,     ARM::VST2LNd32,     false, false, false, SingleSpc, 2, 2 ,true},
 { ARM::VST2LNd32Pseudo_UPD, ARM::VST2LNd32_UPD, false, true, true,  SingleSpc, 2, 2 ,true},
 { ARM::VST2LNd8Pseudo,      ARM::VST2LNd8,      false, false, false, SingleSpc, 2, 8 ,true},
switch (Opcode) {
     case ARM::VST2q32PseudoWB_fixed:
     case ARM::VST2q8PseudoWB_register:
     case ARM::VST2q16PseudoWB_register:
     case ARM::VST2q32PseudoWB_register:
     case ARM::VST3d8Pseudo:
     case ARM::VST3d16Pseudo:
     case ARM::VST3d32Pseudo:
     case ARM::VST1d8TPseudo:
+    case ARM::VST1d8TPseudoWB_fixed:
+    case ARM::VST1d8TPseudoWB_register:
     case ARM::VST1d16TPseudo:
+    case ARM::VST1d16TPseudoWB_fixed:
+    case ARM::VST1d16TPseudoWB_register:
     case ARM::VST1d32TPseudo:
+    case ARM::VST1d32TPseudoWB_fixed:
+    case ARM::VST1d32TPseudoWB_register:
     case ARM::VST1d64TPseudo:
+    case ARM::VST1d64TPseudoWB_fixed:
+    case ARM::VST1d64TPseudoWB_register:
     case ARM::VST3d8Pseudo_UPD:
     case ARM::VST3d16Pseudo_UPD:
     case ARM::VST3d32Pseudo_UPD:
-    case ARM::VST1d64TPseudoWB_fixed:
-    case ARM::VST1d64TPseudoWB_register:
     case ARM::VST3q8Pseudo_UPD:
     case ARM::VST3q16Pseudo_UPD:
     case ARM::VST3q32Pseudo_UPD:
     case ARM::VST3q8oddPseudo:
     case ARM::VST3q16oddPseudo:
     case ARM::VST3q32oddPseudo:
     case ARM::VST3q8oddPseudo_UPD:
     case ARM::VST3q16oddPseudo_UPD:
     case ARM::VST3q32oddPseudo_UPD:
     case ARM::VST4d8Pseudo:
     case ARM::VST4d16Pseudo:
     case ARM::VST4d32Pseudo:
     case ARM::VST1d8QPseudo:
+    case ARM::VST1d8QPseudoWB_fixed:
+    case ARM::VST1d8QPseudoWB_register:
     case ARM::VST1d16QPseudo:
+    case ARM::VST1d16QPseudoWB_fixed:
+    case ARM::VST1d16QPseudoWB_register:
     case ARM::VST1d32QPseudo:
+    case ARM::VST1d32QPseudoWB_fixed:
+    case ARM::VST1d32QPseudoWB_register:
     case ARM::VST1d64QPseudo:
+    case ARM::VST1d64QPseudoWB_fixed:
+    case ARM::VST1d64QPseudoWB_register:
     case ARM::VST4d8Pseudo_UPD:
     case ARM::VST4d16Pseudo_UPD:
     case ARM::VST4d32Pseudo_UPD:
-    case ARM::VST1d64QPseudoWB_fixed:
-    case ARM::VST1d64QPseudoWB_register:
     case ARM::VST1q8HighQPseudo:
     case ARM::VST1q8LowQPseudo_UPD:
     case ARM::VST1q8HighTPseudo:
     case ARM::VST1q8LowTPseudo_UPD:
     case ARM::VST1q16HighQPseudo:
     case ARM::VST1q16LowQPseudo_UPD:
     case ARM::VST1q16HighTPseudo:
     case ARM::VST1q16LowTPseudo_UPD:
     case ARM::VST1q32HighQPseudo:
     case ARM::VST1q32LowQPseudo_UPD:
     case ARM::VST1q32HighTPseudo:
     case ARM::VST1q32LowTPseudo_UPD:
     case ARM::VST1q64HighQPseudo:
     case ARM::VST1q64LowQPseudo_UPD:
     case ARM::VST1q64HighTPseudo:
     case ARM::VST1q64LowTPseudo_UPD:
+    case ARM::VST1q8HighTPseudo_UPD:
+    case ARM::VST1q16HighTPseudo_UPD:
+    case ARM::VST1q32HighTPseudo_UPD:
+    case ARM::VST1q64HighTPseudo_UPD:
+    case ARM::VST1q8HighQPseudo_UPD:
+    case ARM::VST1q16HighQPseudo_UPD:
+    case ARM::VST1q32HighQPseudo_UPD:
+    case ARM::VST1q64HighQPseudo_UPD:
     case ARM::VST4q8Pseudo_UPD:
     case ARM::VST4q16Pseudo_UPD:
     case ARM::VST4q32Pseudo_UPD:
     case ARM::VST4q8oddPseudo:
     case ARM::VST4q16oddPseudo:
     case ARM::VST4q32oddPseudo:
     case ARM::VST4q8oddPseudo_UPD:
     case ARM::VST4q16oddPseudo_UPD:

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

static bool isVSTfixed(unsigned Opc)
   case ARM::VST1d8wb_fixed : return true;
   case ARM::VST1d16wb_fixed : return true;
   case ARM::VST1d32wb_fixed : return true;
   case ARM::VST1d64wb_fixed : return true;
   case ARM::VST1q8wb_fixed : return true;
   case ARM::VST1q16wb_fixed : return true;
   case ARM::VST1q32wb_fixed : return true;
   case ARM::VST1q64wb_fixed : return true;
+  case ARM::VST1d8TPseudoWB_fixed : return true;
+  case ARM::VST1d16TPseudoWB_fixed : return true;
+  case ARM::VST1d32TPseudoWB_fixed : return true;
   case ARM::VST1d64TPseudoWB_fixed : return true;
+  case ARM::VST1d8QPseudoWB_fixed : return true;
+  case ARM::VST1d16QPseudoWB_fixed : return true;
+  case ARM::VST1d32QPseudoWB_fixed : return true;
   case ARM::VST1d64QPseudoWB_fixed : return true;
   case ARM::VST2d8wb_fixed : return true;
   case ARM::VST2d16wb_fixed : return true;
   case ARM::VST2d32wb_fixed : return true;
   case ARM::VST2q8PseudoWB_fixed : return true;
   case ARM::VST2q16PseudoWB_fixed : return true;
   case ARM::VST2q32PseudoWB_fixed : return true;
   }
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
   case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
   case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
   case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
   case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
   case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
   case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
   case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
   case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
+  case ARM::VST1d8TPseudoWB_fixed: return ARM::VST1d8TPseudoWB_register;
+  case ARM::VST1d16TPseudoWB_fixed: return ARM::VST1d16TPseudoWB_register;
+  case ARM::VST1d32TPseudoWB_fixed: return ARM::VST1d32TPseudoWB_register;
   case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
+  case ARM::VST1d8QPseudoWB_fixed: return ARM::VST1d8QPseudoWB_register;
+  case ARM::VST1d16QPseudoWB_fixed: return ARM::VST1d16QPseudoWB_register;
+  case ARM::VST1d32QPseudoWB_fixed: return ARM::VST1d32QPseudoWB_register;
   case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
 
   case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
   case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
   case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
   case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
   case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
   case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
if (Subtarget->hasNEON()) {
                                            ARM::VST4q16oddPseudo_UPD,
                                            ARM::VST4q32oddPseudo_UPD};
       SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
       return;
     }
     break;
   }
 
+  case ARMISD::VST1x2_UPD: {
+    if (Subtarget->hasNEON()) {
+      static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed,
+                                           ARM::VST1q16wb_fixed,
+                                           ARM::VST1q32wb_fixed,
+                                           ARM::VST1q64wb_fixed};
+      static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
+                                           ARM::VST1d16QPseudoWB_fixed,
+                                           ARM::VST1d32QPseudoWB_fixed,
+                                           ARM::VST1d64QPseudoWB_fixed };
+      SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
+      return;
+    }
+    break;
+  }
+
+  case ARMISD::VST1x3_UPD: {
+    if (Subtarget->hasNEON()) {
+      static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed,
+                                           ARM::VST1d16TPseudoWB_fixed,
+                                           ARM::VST1d32TPseudoWB_fixed,
+                                           ARM::VST1d64TPseudoWB_fixed };
+      static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
+                                            ARM::VST1q16LowTPseudo_UPD,
+                                            ARM::VST1q32LowTPseudo_UPD,
+                                            ARM::VST1q64LowTPseudo_UPD };
+      static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD,
+                                            ARM::VST1q16HighTPseudo_UPD,
+                                            ARM::VST1q32HighTPseudo_UPD,
+                                            ARM::VST1q64HighTPseudo_UPD };
+      SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
+      return;
+    }
+    break;
+  }
+
+  case ARMISD::VST1x4_UPD: {
+    if (Subtarget->hasNEON()) {
+      static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
+                                           ARM::VST1d16QPseudoWB_fixed,
+                                           ARM::VST1d32QPseudoWB_fixed,
+                                           ARM::VST1d64QPseudoWB_fixed };
+      static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
+                                            ARM::VST1q16LowQPseudo_UPD,
+                                            ARM::VST1q32LowQPseudo_UPD,
+                                            ARM::VST1q64LowQPseudo_UPD };
+      static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD,
+                                            ARM::VST1q16HighQPseudo_UPD,
+                                            ARM::VST1q32HighQPseudo_UPD,
+                                            ARM::VST1q64HighQPseudo_UPD };
+      SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
+      return;
+    }
+    break;
+  }
   case ARMISD::VST2LN_UPD: {
     static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
                                          ARM::VST2LNd16Pseudo_UPD,
                                          ARM::VST2LNd32Pseudo_UPD };
     static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
                                          ARM::VST2LNq32Pseudo_UPD };
     SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
     return;

llvm/lib/Target/ARM/ARMISelLowering.h

enum NodeType : unsigned {
     // NEON stores with post-increment base updates:
     VST1_UPD,
     VST2_UPD,
     VST3_UPD,
     VST4_UPD,
     VST2LN_UPD,
     VST3LN_UPD,
     VST4LN_UPD,
+    VST1x2_UPD,
+    VST1x3_UPD,
+    VST1x4_UPD,
 
     // Load/Store of dual registers
     LDRD,
     STRD
   };
 
   } // end namespace ARMISD
 

llvm/lib/Target/ARM/ARMISelLowering.cpp

case ARMISD::FIRST_NUMBER:
     MAKE_CASE(ARMISD::VLD1DUP_UPD)
     MAKE_CASE(ARMISD::VLD2DUP_UPD)
     MAKE_CASE(ARMISD::VLD3DUP_UPD)
     MAKE_CASE(ARMISD::VLD4DUP_UPD)
     MAKE_CASE(ARMISD::VST1_UPD)
     MAKE_CASE(ARMISD::VST2_UPD)
     MAKE_CASE(ARMISD::VST3_UPD)
     MAKE_CASE(ARMISD::VST4_UPD)
+    MAKE_CASE(ARMISD::VST1x2_UPD)
+    MAKE_CASE(ARMISD::VST1x3_UPD)
+    MAKE_CASE(ARMISD::VST1x4_UPD)
     MAKE_CASE(ARMISD::VST2LN_UPD)
     MAKE_CASE(ARMISD::VST3LN_UPD)
     MAKE_CASE(ARMISD::VST4LN_UPD)
     MAKE_CASE(ARMISD::WLS)
     MAKE_CASE(ARMISD::WLSSETUP)
     MAKE_CASE(ARMISD::LE)
     MAKE_CASE(ARMISD::LOOP_DEC)
     MAKE_CASE(ARMISD::CSINV)
for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
     Worklist.push_back(User);
     if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
         SDNode::hasPredecessorHelper(User, Visited, Worklist))
       continue;
 
     // Find the new opcode for the updating load/store.
     bool isLoadOp = true;
     bool isLaneOp = false;
+    // Workaround for vst1x and vld1x which do not have alignment operand.
+    bool hasAlignment = true;
     unsigned NewOpc = 0;
     unsigned NumVecs = 0;
     if (isIntrinsic) {
       unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
       switch (IntNo) {
       default: llvm_unreachable("unexpected intrinsic for Neon base update");
       case Intrinsic::arm_neon_vld1:     NewOpc = ARMISD::VLD1_UPD;
         NumVecs = 1; break;
if (isIntrinsic) {
       case Intrinsic::arm_neon_vst4:     NewOpc = ARMISD::VST4_UPD;
         NumVecs = 4; isLoadOp = false; break;
       case Intrinsic::arm_neon_vst2lane: NewOpc = ARMISD::VST2LN_UPD;
         NumVecs = 2; isLoadOp = false; isLaneOp = true; break;
       case Intrinsic::arm_neon_vst3lane: NewOpc = ARMISD::VST3LN_UPD;
         NumVecs = 3; isLoadOp = false; isLaneOp = true; break;
       case Intrinsic::arm_neon_vst4lane: NewOpc = ARMISD::VST4LN_UPD;
         NumVecs = 4; isLoadOp = false; isLaneOp = true; break;
+      case Intrinsic::arm_neon_vst1x2:   NewOpc = ARMISD::VST1x2_UPD;
+        NumVecs = 2; isLoadOp = false; hasAlignment = false; break;
+      case Intrinsic::arm_neon_vst1x3:   NewOpc = ARMISD::VST1x3_UPD;
+        NumVecs = 3; isLoadOp = false; hasAlignment = false; break;
+      case Intrinsic::arm_neon_vst1x4:   NewOpc = ARMISD::VST1x4_UPD;
+        NumVecs = 4; isLoadOp = false; hasAlignment = false; break;
       }
     } else {
       isLaneOp = true;
       switch (N->getOpcode()) {
       default: llvm_unreachable("unexpected opcode for Neon base update");
       case ARMISD::VLD1DUP: NewOpc = ARMISD::VLD1DUP_UPD; NumVecs = 1; break;
       case ARMISD::VLD2DUP: NewOpc = ARMISD::VLD2DUP_UPD; NumVecs = 2; break;
       case ARMISD::VLD3DUP: NewOpc = ARMISD::VLD3DUP_UPD; NumVecs = 3; break;
for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
     Ops.push_back(Inc);
 
     if (StoreSDNode *StN = dyn_cast<StoreSDNode>(N)) {
       // Try to match the intrinsic's signature
       Ops.push_back(StN->getValue());
     } else {
       // Loads (and of course intrinsics) match the intrinsics' signature,
       // so just add all but the alignment operand.
-      for (unsigned i = AddrOpIdx + 1; i < N->getNumOperands() - 1; ++i)
+      unsigned LastOperand =
+          hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands();
+      for (unsigned i = AddrOpIdx + 1; i < LastOperand; ++i)
         Ops.push_back(N->getOperand(i));
     }
 
     // For all node types, the alignment operand is always the last one.
     Ops.push_back(DAG.getConstant(Alignment, dl, MVT::i32));
 
     // If this is a non-standard-aligned STORE, the penultimate operand is the
     // stored value.  Bitcast it to the aligned type.

llvm/lib/Target/ARM/ARMInstrNEON.td

 def VST1d64T    : VST1D3<{1,1,0,?}, "64", addrmode6align64>;
 
 defm VST1d8Twb  : VST1D3WB<{0,0,0,?}, "8",  addrmode6align64>;
 defm VST1d16Twb : VST1D3WB<{0,1,0,?}, "16", addrmode6align64>;
 defm VST1d32Twb : VST1D3WB<{1,0,0,?}, "32", addrmode6align64>;
 defm VST1d64Twb : VST1D3WB<{1,1,0,?}, "64", addrmode6align64>;
 
 def VST1d8TPseudo             : VSTQQPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+def VST1d8TPseudoWB_fixed     : VSTQQWBfixedPseudo<IIC_VST1x3u>, Sched<[WriteVST3]>;
+def VST1d8TPseudoWB_register  : VSTQQWBPseudo<IIC_VST1x3u>, Sched<[WriteVST3]>;
 def VST1d16TPseudo            : VSTQQPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+def VST1d16TPseudoWB_fixed    : VSTQQWBfixedPseudo<IIC_VST1x3u>, Sched<[WriteVST3]>;
+def VST1d16TPseudoWB_register : VSTQQWBPseudo<IIC_VST1x3u>, Sched<[WriteVST3]>;
 def VST1d32TPseudo            : VSTQQPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+def VST1d32TPseudoWB_fixed    : VSTQQWBfixedPseudo<IIC_VST1x3u>, Sched<[WriteVST3]>;
+def VST1d32TPseudoWB_register : VSTQQWBPseudo<IIC_VST1x3u>, Sched<[WriteVST3]>;
 def VST1d64TPseudo            : VSTQQPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
 def VST1d64TPseudoWB_fixed    : VSTQQWBfixedPseudo<IIC_VST1x3u>, Sched<[WriteVST3]>;
 def VST1d64TPseudoWB_register : VSTQQWBPseudo<IIC_VST1x3u>, Sched<[WriteVST3]>;
 
 def VST1q8HighTPseudo     : VSTQQQQPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
-def VST1q8LowTPseudo_UPD  : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
 def VST1q16HighTPseudo    : VSTQQQQPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
-def VST1q16LowTPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
 def VST1q32HighTPseudo    : VSTQQQQPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
-def VST1q32LowTPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
 def VST1q64HighTPseudo    : VSTQQQQPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+
+def VST1q8HighTPseudo_UPD  : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+def VST1q16HighTPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+def VST1q32HighTPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+def VST1q64HighTPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+
+def VST1q8LowTPseudo_UPD  : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+def VST1q16LowTPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
+def VST1q32LowTPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
 def VST1q64LowTPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x3>, Sched<[WriteVST3]>;
 
 // ...with 4 registers
 class VST1D4<bits<4> op7_4, string Dt, Operand AddrMode>
   : NLdSt<0, 0b00, 0b0010, op7_4, (outs),
           (ins AddrMode:$Rn, VecListFourD:$Vd),
           IIC_VST1x4, "vst1", Dt, "$Vd, $Rn", "",
           []>, Sched<[WriteVST4]> {

 def VST1d64Q    : VST1D4<{1,1,?,?}, "64", addrmode6align64or128or256>;
 
 defm VST1d8Qwb  : VST1D4WB<{0,0,?,?}, "8",  addrmode6align64or128or256>;
 defm VST1d16Qwb : VST1D4WB<{0,1,?,?}, "16", addrmode6align64or128or256>;
 defm VST1d32Qwb : VST1D4WB<{1,0,?,?}, "32", addrmode6align64or128or256>;
 defm VST1d64Qwb : VST1D4WB<{1,1,?,?}, "64", addrmode6align64or128or256>;
 
 def VST1d8QPseudo             : VSTQQPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+def VST1d8QPseudoWB_fixed     : VSTQQWBfixedPseudo<IIC_VST1x4u>, Sched<[WriteVST4]>;
+def VST1d8QPseudoWB_register  : VSTQQWBPseudo<IIC_VST1x4u>, Sched<[WriteVST4]>;
 def VST1d16QPseudo            : VSTQQPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+def VST1d16QPseudoWB_fixed    : VSTQQWBfixedPseudo<IIC_VST1x4u>, Sched<[WriteVST4]>;
+def VST1d16QPseudoWB_register : VSTQQWBPseudo<IIC_VST1x4u>, Sched<[WriteVST4]>;
 def VST1d32QPseudo            : VSTQQPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+def VST1d32QPseudoWB_fixed    : VSTQQWBfixedPseudo<IIC_VST1x4u>, Sched<[WriteVST4]>;
+def VST1d32QPseudoWB_register : VSTQQWBPseudo<IIC_VST1x4u>, Sched<[WriteVST4]>;
 def VST1d64QPseudo            : VSTQQPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
 def VST1d64QPseudoWB_fixed    : VSTQQWBfixedPseudo<IIC_VST1x4u>, Sched<[WriteVST4]>;
 def VST1d64QPseudoWB_register : VSTQQWBPseudo<IIC_VST1x4u>, Sched<[WriteVST4]>;
 
 def VST1q8HighQPseudo     : VSTQQQQPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
-def VST1q8LowQPseudo_UPD  : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
 def VST1q16HighQPseudo    : VSTQQQQPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
-def VST1q16LowQPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
 def VST1q32HighQPseudo    : VSTQQQQPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
-def VST1q32LowQPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
 def VST1q64HighQPseudo    : VSTQQQQPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+
+def VST1q8HighQPseudo_UPD  : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+def VST1q16HighQPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+def VST1q32HighQPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+def VST1q64HighQPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+
+def VST1q8LowQPseudo_UPD  : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+def VST1q16LowQPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
+def VST1q32LowQPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
 def VST1q64LowQPseudo_UPD : VSTQQQQWBPseudo<IIC_VST1x4>, Sched<[WriteVST4]>;
 
 //   VST2     : Vector Store (multiple 2-element structures)
 class VST2<bits<4> op11_8, bits<4> op7_4, string Dt, RegisterOperand VdTy,
             InstrItinClass itin, Operand AddrMode>
   : NLdSt<0, 0b00, op11_8, op7_4, (outs), (ins AddrMode:$Rn, VdTy:$Vd),
           itin, "vst2", Dt, "$Vd, $Rn", "", []> {
   let Rm = 0b1111;

llvm/test/CodeGen/ARM/arm-vst1.ll

 entry:
   %b0 = extractvalue %struct.uint8x16x4_t %b, 0, 0
   %b1 = extractvalue %struct.uint8x16x4_t %b, 0, 1
   %b2 = extractvalue %struct.uint8x16x4_t %b, 0, 2
   %b3 = extractvalue %struct.uint8x16x4_t %b, 0, 3
   tail call void @llvm.arm.neon.vst1x4.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1, <16 x i8> %b2, <16 x i8> %b3)
   ret void
 }
+
+; Updating
+
+; CHECK-LABEL: test_vst1_u8_x2_post_imm
+; CHECK: vst1.8 {d16, d17}, [r0:64]!
+define i8* @test_vst1_u8_x2_post_imm(i8* %a, %struct.uint8x8x2_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x8x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x8x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1)
+  %tmp = getelementptr i8, i8* %a, i32 16
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u8_x2_post_reg
+; CHECK: vst1.8 {d16, d17}, [r0:64], r1
+define i8* @test_vst1_u8_x2_post_reg(i8* %a, %struct.uint8x8x2_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x8x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x8x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1)
+  %tmp = getelementptr i8, i8* %a, i32 %inc
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u16_x2_post_imm
+; CHECK: vst1.16 {d16, d17}, [r0:64]!
+define i16* @test_vst1_u16_x2_post_imm(i16* %a, %struct.uint16x4x2_t %b) nounwind {
+  %b0 = extractvalue %struct.uint16x4x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x4x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1)
+  %tmp = getelementptr i16, i16* %a, i32 8
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u16_x2_post_reg
+; CHECK: vst1.16 {d16, d17}, [r0:64], r1
+define i16* @test_vst1_u16_x2_post_reg(i16* %a, %struct.uint16x4x2_t %b, i32 %inc) nounwind {
+  %b0 = extractvalue %struct.uint16x4x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x4x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1)
+  %tmp = getelementptr i16, i16* %a, i32 %inc
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u32_x2_post_imm
+; CHECK: vst1.32 {d16, d17}, [r0:64]!
+define i32* @test_vst1_u32_x2_post_imm(i32* %a, %struct.uint32x2x2_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x2x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x2x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1)
+  %tmp = getelementptr i32, i32* %a, i32 4
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u32_x2_post_reg
+; CHECK: vst1.32 {d16, d17}, [r0:64], r1
+define i32* @test_vst1_u32_x2_post_reg(i32* %a, %struct.uint32x2x2_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x2x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x2x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1)
+  %tmp = getelementptr i32, i32* %a, i32 %inc
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u64_x2_post_imm
+; CHECK: vst1.64 {d16, d17}, [r0:64]!
+define i64* @test_vst1_u64_x2_post_imm(i64* %a, %struct.uint64x1x2_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x1x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x1x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1)
+  %tmp = getelementptr i64, i64* %a, i32 2
+  ret i64* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u64_x2_post_reg
+; CHECK: vst1.64 {d16, d17}, [r0:64], r1
+define i64* @test_vst1_u64_x2_post_reg(i64* %a, %struct.uint64x1x2_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x1x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x1x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1)
+  %tmp = getelementptr i64, i64* %a, i32 %inc
+  ret i64* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u8_x2_post_imm
+; CHECK: vst1.8  {d16, d17, d18, d19}, [r0:256]!
+define i8* @test_vst1q_u8_x2_post_imm(i8* %a, %struct.uint8x16x2_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x16x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x16x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1)
+  %tmp = getelementptr i8, i8* %a, i32 32
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u8_x2_post_reg
+; CHECK: vst1.8  {d16, d17, d18, d19}, [r0:256], r12
+define i8* @test_vst1q_u8_x2_post_reg(i8* %a, %struct.uint8x16x2_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x16x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x16x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1)
+  %tmp = getelementptr i8, i8* %a, i32 %inc
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u16_x2_post_imm
+; CHECK: vst1.16 {d16, d17, d18, d19}, [r0:256]!
+define i16* @test_vst1q_u16_x2_post_imm(i16* %a, %struct.uint16x8x2_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint16x8x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x8x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i16.v8i16(i16* %a, <8 x i16> %b0, <8 x i16> %b1)
+  %tmp = getelementptr i16, i16* %a, i32 16
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u16_x2_post_reg
+; CHECK: vst1.16 {d16, d17, d18, d19}, [r0:256], r1
+define i16* @test_vst1q_u16_x2_post_reg(i16* %a, %struct.uint16x8x2_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint16x8x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x8x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i16.v8i16(i16* %a, <8 x i16> %b0, <8 x i16> %b1)
+  %tmp = getelementptr i16, i16* %a, i32 %inc
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u32_x2_post_imm
+; CHECK: vst1.32 {d16, d17, d18, d19}, [r0:256]!
+define i32* @test_vst1q_u32_x2_post_imm(i32* %a, %struct.uint32x4x2_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x4x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x4x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i32.v4i32(i32* %a, <4 x i32> %b0, <4 x i32> %b1)
+  %tmp = getelementptr i32, i32* %a, i32 8
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u32_x2_post_reg
+; CHECK: vst1.32 {d16, d17, d18, d19}, [r0:256], r1
+define i32* @test_vst1q_u32_x2_post_reg(i32* %a, %struct.uint32x4x2_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x4x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x4x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i32.v4i32(i32* %a, <4 x i32> %b0, <4 x i32> %b1)
+  %tmp = getelementptr i32, i32* %a, i32 %inc
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u64_x2_post_imm
+; CHECK: vst1.64 {d16, d17, d18, d19}, [r0:256]!
+define i64* @test_vst1q_u64_x2_post_imm(i64* %a, %struct.uint64x2x2_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x2x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x2x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i64.v2i64(i64* %a, <2 x i64> %b0, <2 x i64> %b1)
+  %tmp = getelementptr i64, i64* %a, i32 4
+  ret i64* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u64_x2_post_reg
+; CHECK: vst1.64 {d16, d17, d18, d19}, [r0:256], r1
+define i64* @test_vst1q_u64_x2_post_reg(i64* %a, %struct.uint64x2x2_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x2x2_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x2x2_t %b, 0, 1
+  tail call void @llvm.arm.neon.vst1x2.p0i64.v2i64(i64* %a, <2 x i64> %b0, <2 x i64> %b1)
+  %tmp = getelementptr i64, i64* %a, i32 %inc
+  ret i64* %tmp
+}
+
+
+; CHECK-LABEL: test_vst1_u8_x3_post_imm
+; CHECK: vst1.8 {d16, d17, d18}, [r0:64]!
+define i8* @test_vst1_u8_x3_post_imm(i8* %a, %struct.uint8x8x3_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x8x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x8x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint8x8x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1, <8 x i8> %b2)
+  %tmp = getelementptr i8, i8* %a, i32 24
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u8_x3_post_reg
+; CHECK: vst1.8 {d16, d17, d18}, [r0:64], r1
+define i8* @test_vst1_u8_x3_post_reg(i8* %a, %struct.uint8x8x3_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x8x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x8x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint8x8x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1, <8 x i8> %b2)
+  %tmp = getelementptr i8, i8* %a, i32 %inc
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u16_x3_post_imm
+; CHECK: vst1.16 {d16, d17, d18}, [r0:64]!
+define i16* @test_vst1_u16_x3_post_imm(i16* %a, %struct.uint16x4x3_t %b) nounwind {
+  %b0 = extractvalue %struct.uint16x4x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x4x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint16x4x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1, <4 x i16> %b2)
+  %tmp = getelementptr i16, i16* %a, i32 12
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u16_x3_post_reg
+; CHECK: vst1.16 {d16, d17, d18}, [r0:64], r1
+define i16* @test_vst1_u16_x3_post_reg(i16* %a, %struct.uint16x4x3_t %b, i32 %inc) nounwind {
+  %b0 = extractvalue %struct.uint16x4x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x4x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint16x4x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1, <4 x i16> %b2)
+  %tmp = getelementptr i16, i16* %a, i32 %inc
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u32_x3_post_imm
+; CHECK: vst1.32 {d16, d17, d18}, [r0:64]!
+define i32* @test_vst1_u32_x3_post_imm(i32* %a, %struct.uint32x2x3_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x2x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x2x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint32x2x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1, <2 x i32> %b2)
+  %tmp = getelementptr i32, i32* %a, i32 6
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u32_x3_post_reg
+; CHECK: vst1.32 {d16, d17, d18}, [r0:64], r1
+define i32* @test_vst1_u32_x3_post_reg(i32* %a, %struct.uint32x2x3_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x2x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x2x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint32x2x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1, <2 x i32> %b2)
+  %tmp = getelementptr i32, i32* %a, i32 %inc
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u64_x3_post_imm
+; CHECK: vst1.64 {d16, d17, d18}, [r0:64]!
+define i64* @test_vst1_u64_x3_post_imm(i64* %a, %struct.uint64x1x3_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x1x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x1x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint64x1x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1, <1 x i64> %b2)
+  %tmp = getelementptr i64, i64* %a, i32 3
+  ret i64* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u64_x3_post_reg
+; CHECK: vst1.64 {d16, d17, d18}, [r0:64], r1
+define i64* @test_vst1_u64_x3_post_reg(i64* %a, %struct.uint64x1x3_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x1x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x1x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint64x1x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1, <1 x i64> %b2)
+  %tmp = getelementptr i64, i64* %a, i32 %inc
+  ret i64* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u8_x3_post_imm
+; CHECK: vst1.8 {d16, d17, d18}, [r0:64]!
+; CHECK: vst1.8 {d19, d20, d21}, [r0:64]!
+define i8* @test_vst1q_u8_x3_post_imm(i8* %a, %struct.uint8x16x3_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x16x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x16x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint8x16x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1, <16 x i8> %b2)
+  %tmp = getelementptr i8, i8* %a, i32 48
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u16_x3_post_imm
+; CHECK: vst1.16 {d16, d17, d18}, [r0:64]!
+; CHECK: vst1.16 {d19, d20, d21}, [r0:64]!
+define i16* @test_vst1q_u16_x3_post_imm(i16* %a, %struct.uint16x8x3_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint16x8x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x8x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint16x8x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i16.v8i16(i16* %a, <8 x i16> %b0, <8 x i16> %b1, <8 x i16> %b2)
+  %tmp = getelementptr i16, i16* %a, i32 24
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u32_x3_post_imm
+; CHECK: vst1.32 {d16, d17, d18}, [r0:64]!
+; CHECK: vst1.32 {d19, d20, d21}, [r0:64]!
+define i32* @test_vst1q_u32_x3_post_imm(i32* %a, %struct.uint32x4x3_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x4x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x4x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint32x4x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i32.v4i32(i32* %a, <4 x i32> %b0, <4 x i32> %b1, <4 x i32> %b2)
+  %tmp = getelementptr i32, i32* %a, i32 12
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u64_x3_post_imm
+; CHECK: vst1.64 {d16, d17, d18}, [r0:64]!
+; CHECK: vst1.64 {d19, d20, d21}, [r0:64]!
+define i64* @test_vst1q_u64_x3_post_imm(i64* %a, %struct.uint64x2x3_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x2x3_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x2x3_t %b, 0, 1
+  %b2 = extractvalue %struct.uint64x2x3_t %b, 0, 2
+  tail call void @llvm.arm.neon.vst1x3.p0i64.v2i64(i64* %a, <2 x i64> %b0, <2 x i64> %b1, <2 x i64> %b2)
+  %tmp = getelementptr i64, i64* %a, i32 6
+  ret i64* %tmp
+}
+
+
+; CHECK-LABEL: test_vst1_u8_x4_post_imm
+; CHECK: vst1.8 {d16, d17, d18, d19}, [r0:256]!
+define i8* @test_vst1_u8_x4_post_imm(i8* %a, %struct.uint8x8x4_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x8x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x8x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint8x8x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint8x8x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1, <8 x i8> %b2, <8 x i8> %b3)
+  %tmp = getelementptr i8, i8* %a, i32 32
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u8_x4_post_reg
+; CHECK: vst1.8 {d16, d17, d18, d19}, [r0:256], r1
+define i8* @test_vst1_u8_x4_post_reg(i8* %a, %struct.uint8x8x4_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x8x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x8x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint8x8x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint8x8x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1, <8 x i8> %b2, <8 x i8> %b3)
+  %tmp = getelementptr i8, i8* %a, i32 %inc
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u16_x4_post_imm
+; CHECK: vst1.16 {d16, d17, d18, d19}, [r0:256]!
+define i16* @test_vst1_u16_x4_post_imm(i16* %a, %struct.uint16x4x4_t %b) nounwind {
+  %b0 = extractvalue %struct.uint16x4x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x4x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint16x4x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint16x4x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1, <4 x i16> %b2, <4 x i16> %b3)
+  %tmp = getelementptr i16, i16* %a, i32 16
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u16_x4_post_reg
+; CHECK: vst1.16 {d16, d17, d18, d19}, [r0:256], r1
+define i16* @test_vst1_u16_x4_post_reg(i16* %a, %struct.uint16x4x4_t %b, i32 %inc) nounwind {
+  %b0 = extractvalue %struct.uint16x4x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x4x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint16x4x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint16x4x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1, <4 x i16> %b2, <4 x i16> %b3)
+  %tmp = getelementptr i16, i16* %a, i32 %inc
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u32_x4_post_imm
+; CHECK: vst1.32 {d16, d17, d18, d19}, [r0:256]!
+define i32* @test_vst1_u32_x4_post_imm(i32* %a, %struct.uint32x2x4_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x2x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x2x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint32x2x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint32x2x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1, <2 x i32> %b2, <2 x i32> %b3)
+  %tmp = getelementptr i32, i32* %a, i32 8
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u32_x4_post_reg
+; CHECK: vst1.32 {d16, d17, d18, d19}, [r0:256], r1
+define i32* @test_vst1_u32_x4_post_reg(i32* %a, %struct.uint32x2x4_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x2x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x2x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint32x2x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint32x2x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1, <2 x i32> %b2, <2 x i32> %b3)
+  %tmp = getelementptr i32, i32* %a, i32 %inc
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u64_x4_post_imm
+; CHECK: vst1.64 {d16, d17, d18, d19}, [r0:256]!
+define i64* @test_vst1_u64_x4_post_imm(i64* %a, %struct.uint64x1x4_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x1x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x1x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint64x1x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint64x1x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1, <1 x i64> %b2, <1 x i64> %b3)
+  %tmp = getelementptr i64, i64* %a, i32 4
+  ret i64* %tmp
+}
+
+; CHECK-LABEL: test_vst1_u64_x4_post_reg
+; CHECK: vst1.64 {d16, d17, d18, d19}, [r0:256], r1
+define i64* @test_vst1_u64_x4_post_reg(i64* %a, %struct.uint64x1x4_t %b, i32 %inc) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x1x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x1x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint64x1x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint64x1x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1, <1 x i64> %b2, <1 x i64> %b3)
+  %tmp = getelementptr i64, i64* %a, i32 %inc
+  ret i64* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u8_x4_post_imm
+; CHECK: vst1.8 {d16, d17, d18, d19}, [r0:256]!
+; CHECK: vst1.8 {d20, d21, d22, d23}, [r0:256]!
+define i8* @test_vst1q_u8_x4_post_imm(i8* %a, %struct.uint8x16x4_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint8x16x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint8x16x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint8x16x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint8x16x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1, <16 x i8> %b2, <16 x i8> %b3)
+  %tmp = getelementptr i8, i8* %a, i32 64
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u16_x4_post_imm
+; CHECK: vst1.16 {d16, d17, d18, d19}, [r0:256]!
+; CHECK: vst1.16 {d20, d21, d22, d23}, [r0:256]!
+define i16* @test_vst1q_u16_x4_post_imm(i16* %a, %struct.uint16x8x4_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint16x8x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint16x8x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint16x8x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint16x8x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i16.v8i16(i16* %a, <8 x i16> %b0, <8 x i16> %b1, <8 x i16> %b2, <8 x i16> %b3)
+  %tmp = getelementptr i16, i16* %a, i32 32
+  ret i16* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u32_x4_post_imm
+; CHECK: vst1.32 {d16, d17, d18, d19}, [r0:256]!
+; CHECK: vst1.32 {d20, d21, d22, d23}, [r0:256]!
+define i32* @test_vst1q_u32_x4_post_imm(i32* %a, %struct.uint32x4x4_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint32x4x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint32x4x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint32x4x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint32x4x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i32.v4i32(i32* %a, <4 x i32> %b0, <4 x i32> %b1, <4 x i32> %b2, <4 x i32> %b3)
+  %tmp = getelementptr i32, i32* %a, i32 16
+  ret i32* %tmp
+}
+
+; CHECK-LABEL: test_vst1q_u64_x4_post_imm
+; CHECK: vst1.64 {d16, d17, d18, d19}, [r0:256]!
+; CHECK: vst1.64 {d20, d21, d22, d23}, [r0:256]!
+define i64* @test_vst1q_u64_x4_post_imm(i64* %a, %struct.uint64x2x4_t %b) nounwind {
+entry:
+  %b0 = extractvalue %struct.uint64x2x4_t %b, 0, 0
+  %b1 = extractvalue %struct.uint64x2x4_t %b, 0, 1
+  %b2 = extractvalue %struct.uint64x2x4_t %b, 0, 2
+  %b3 = extractvalue %struct.uint64x2x4_t %b, 0, 3
+  tail call void @llvm.arm.neon.vst1x4.p0i64.v2i64(i64* %a, <2 x i64> %b0, <2 x i64> %b1, <2 x i64> %b2, <2 x i64> %b3)
+  %tmp = getelementptr i64, i64* %a, i32 8
+  ret i64* %tmp
+}
+