[AArch64] FEAT_LRCPC3 load/store optimisations

[tmatheson]

Summary

Extend the load/store optimizer to identify cases where the offset variants of
+rcpc3 instructions can be used.

The non-indexed LDIAPP/STILP are output by instruction selection and are
optimised to their +/- 8/16 byte variants by the load/store optimiser.

[lenary]

Review Notes:

• Post/pre-indexing looks good.
• There's no merging.
• We talked about codegen for the neon instructions in RCPC3. Maybe that should be a different patch when you do implement it?

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

2535–2537

Can you rewrite this with llvm::any?

2540–2542

I think this fixme is unnecessary, or is development-only.

2552–2556

Please split this into three different ifs with comments to make these conditions readable/explanable.

llvm/test/CodeGen/AArch64/aarch64-rcpc3-ldst.ll

10–15

Can you check this TODO?

24–46

The codegen is not different here and is not showing merging. LDIAPP has constraints on when you can merge, which depend on the order of the two accesses, which will need testing.

380–389

TODO?

395–396

TODO?

[tmatheson]

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

2552–2556

There's not really much I can do with it.

llvm/include/llvm/CodeGen/MachineInstr.h

   bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const;
 
   /// Return true if this instruction may have an ordered
   /// or volatile memory reference, or if the information describing the memory
   /// reference is not available. Return false if it is known to have no
   /// ordered or volatile memory references.
   bool hasOrderedMemoryRef() const;
 
+  /// Return true if this instruction has a volatile memory reference.
+  bool hasVolatileMemoryRef() const;
+
   /// Return true if this load instruction never traps and points to a memory
   /// location whose value doesn't change during the execution of this function.
   ///
   /// Examples include loading a value from the constant pool or from the
   /// argument area of a function (if it does not change).  If the instruction
   /// does multiple loads, this returns true only if all of the loads are
   /// dereferenceable and invariant.
   bool isDereferenceableInvariantLoad() const;

llvm/lib/CodeGen/MachineInstr.cpp

if (memoperands_empty())
     return true;
 
   // Check if any of our memory operands are ordered.
   return llvm::any_of(memoperands(), [](const MachineMemOperand *MMO) {
     return !MMO->isUnordered();
   });
 }
 
+/// Return true if this instruction has a volatile memory reference.
+bool MachineInstr::hasVolatileMemoryRef() const {
+  // Check if any of our memory operands are ordered.
+  return llvm::any_of(memoperands(), [](const MachineMemOperand *MMO) {
+    return MMO->isVolatile();
+  });
+}
+
 /// isDereferenceableInvariantLoad - Return true if this instruction will never
 /// trap and is loading from a location whose value is invariant across a run of
 /// this function.
 bool MachineInstr::isDereferenceableInvariantLoad() const {
   // If the instruction doesn't load at all, it isn't an invariant load.
   if (!mayLoad())
     return false;
 

llvm/lib/Target/AArch64/AArch64InstrInfo.h

public:
   static bool isPreLdSt(const MachineInstr &MI);
 
   /// Returns whether the instruction is a paired load/store.
   static bool isPairedLdSt(const MachineInstr &MI);
 
   /// Returns the base register operator of a load/store.
   static const MachineOperand &getLdStBaseOp(const MachineInstr &MI);
 
-  /// Returns the the immediate offset operator of a load/store.
+  /// Returns the immediate offset operator of a load/store.
   static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI);
 
+  /// Returns the offset directly when the instruction has no offset operand.
+  static int64_t getLdStOffset(const MachineInstr &MI);
+
+  /// Return whether this load/store has an offset operand.
+  static bool hasLdStOffsetOp(const MachineInstr &MI);
+
+  /// Return whether a pre/post indexed variant of this instruction exists.
+  /// Used by isCandidateToMergeOrPair, which otherwise uses the
+  /// existence/position of immediate operands to decide whether an op is
+  /// suitable for merging/pairing.
+  static bool hasRCPC3PrePostIndexVariant(const MachineInstr &MI);
+
   /// Returns whether the instruction is FP or NEON.
   static bool isFpOrNEON(const MachineInstr &MI);
 
   /// Returns whether the instruction is in Q form (128 bit operands)
   static bool isQForm(const MachineInstr &MI);
 
   /// Returns the index for the immediate for a given instruction.
   static unsigned getLoadStoreImmIdx(unsigned Opc);

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
   case AArch64::LDRWpre:
   case AArch64::LDURXi:
   case AArch64::LDRXpre:
   case AArch64::LDURSWi:
   case AArch64::LDURHHi:
   case AArch64::LDURBBi:
   case AArch64::LDURSBWi:
   case AArch64::LDURSHWi:
+  case AArch64::LDIAPPWpre:
+  case AArch64::LDIAPPXpre:
+  case AArch64::STILPWpre:
+  case AArch64::STILPXpre:
+  case AArch64::LDAPRWpre:
+  case AArch64::LDAPRXpre:
+  case AArch64::STLRWpre:
+  case AArch64::STLRXpre:
     return true;
   }
 }
 
 std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
   switch (Opc) {
   default: return {};
   case AArch64::PRFMui: return AArch64::PRFUMi;
bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
   case AArch64::STRXui:
   case AArch64::STRWui:
   case AArch64::LDRSui:
   case AArch64::LDRDui:
   case AArch64::LDRQui:
   case AArch64::LDRXui:
   case AArch64::LDRWui:
   case AArch64::LDRSWui:
+  case AArch64::LDAPRW:
+  case AArch64::LDAPRX:
+  case AArch64::STLRW:
+  case AArch64::STLRX:
   // Unscaled instructions.
   case AArch64::STURSi:
   case AArch64::STRSpre:
   case AArch64::STURDi:
   case AArch64::STRDpre:
   case AArch64::STURQi:
   case AArch64::STRQpre:
   case AArch64::STURWi:

 
 // Is this a candidate for ld/st merging or pairing?  For example, we don't
 // touch volatiles or load/stores that have a hint to avoid pair formation.
 bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
 
   bool IsPreLdSt = isPreLdSt(MI);
 
   // If this is a volatile load/store, don't mess with it.
-  if (MI.hasOrderedMemoryRef())
+  if (MI.hasVolatileMemoryRef() || MI.memoperands_empty())
     return false;
 
+  // The only ordered loads/stores that we consider here are those in RCPC3.
+  if (!hasRCPC3PrePostIndexVariant(MI)) {
+    for (const MachineMemOperand *MMO : MI.memoperands())
+      if (!MMO->isUnordered())
+        return false;
+  }
+
+  // FIXME temporary to see what this hits besides the instructions we're adding
+  if (MI.hasOrderedMemoryRef())
+    assert(hasRCPC3PrePostIndexVariant(MI));
+
   // Make sure this is a reg/fi+imm (as opposed to an address reloc).
   // For Pre-inc LD/ST, the operand is shifted by one.
   assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
           MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
          "Expected a reg or frame index operand.");
 
   // For Pre-indexed addressing quadword instructions, the third operand is the
   // immediate value.
-  bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(3).isImm();
+  bool IsImmPreLdSt =
+      IsPreLdSt && MI.getNumOperands() >= 4 && MI.getOperand(3).isImm();
+  bool IsTypicalCase = MI.getNumOperands() >= 3 && MI.getOperand(2).isImm();
 
-  if (!MI.getOperand(2).isImm() && !IsImmPreLdSt)
+  if (!IsTypicalCase && !IsImmPreLdSt && !hasRCPC3PrePostIndexVariant(MI))
     return false;
 
   // Can't merge/pair if the instruction modifies the base register.
   // e.g., ldr x0, [x0]
   // This case will never occur with an FI base.
   // However, if the instruction is an LDR/STR<S,D,Q,W,X>pre, it can be merged.
   // For example:
   //   ldr q0, [x11, #32]!
int AArch64InstrInfo::getMemScale(unsigned Opc) {
   default:
     llvm_unreachable("Opcode has unknown scale!");
   case AArch64::LDRBBui:
   case AArch64::LDURBBi:
   case AArch64::LDRSBWui:
   case AArch64::LDURSBWi:
   case AArch64::STRBBui:
   case AArch64::STURBBi:
+  case AArch64::LDIAPPW:
+  case AArch64::LDIAPPX:
+  case AArch64::STILPW:
+  case AArch64::STILPX:
+  case AArch64::LDAPRW:
+  case AArch64::LDAPRX:
+  case AArch64::STLRW:
+  case AArch64::STLRX:
     return 1;
   case AArch64::LDRHHui:
   case AArch64::LDURHHi:
   case AArch64::LDRSHWui:
   case AArch64::LDURSHWi:
   case AArch64::STRHHui:
   case AArch64::STURHHi:
     return 2;

 bool AArch64InstrInfo::isPreLdSt(const MachineInstr &MI) {
   return isPreLd(MI) || isPreSt(MI);
 }
 
 bool AArch64InstrInfo::isPairedLdSt(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
     return false;
+  case AArch64::LDIAPPW:
+  case AArch64::LDIAPPX:
   case AArch64::LDPSi:
   case AArch64::LDPSWi:
   case AArch64::LDPDi:
   case AArch64::LDPQi:
   case AArch64::LDPWi:
   case AArch64::LDPXi:
+  case AArch64::STILPW:
+  case AArch64::STILPX:
   case AArch64::STPSi:
   case AArch64::STPDi:
   case AArch64::STPQi:
   case AArch64::STPWi:
   case AArch64::STPXi:
   case AArch64::STGPi:
     return true;
   }
 }
 
 const MachineOperand &AArch64InstrInfo::getLdStBaseOp(const MachineInstr &MI) {
   unsigned Idx =
       AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 2
                                                                             : 1;
   return MI.getOperand(Idx);
 }
 
+bool AArch64InstrInfo::hasRCPC3PrePostIndexVariant(const MachineInstr &MI) {
+  switch (MI.getOpcode()) {
+  default:
+    return false;
+  // Added by FEAT_RCPC3
+  case AArch64::LDIAPPW:
+  case AArch64::STILPW:
+  case AArch64::LDIAPPX:
+  case AArch64::STILPX:
+  // Pre-existing, but FEAT_LRCPC3 added pre/post indexed versions
+  case AArch64::LDAPRW:
+  case AArch64::LDAPRX:
+  case AArch64::STLRW:
+  case AArch64::STLRX:
+    return true;
+  }
+}
+
+bool AArch64InstrInfo::hasLdStOffsetOp(const MachineInstr &MI) {
+  // Currently only the FEAT_LRCPC3 instructions don't have an index operand.
+  return !hasRCPC3PrePostIndexVariant(MI);
+}
+
 const MachineOperand &
 AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
+  assert(hasLdStOffsetOp(MI));
   unsigned Idx =
       AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 3
                                                                             : 2;
   return MI.getOperand(Idx);
 }
 
+int64_t AArch64InstrInfo::getLdStOffset(const MachineInstr &MI) {
+  if (!hasLdStOffsetOp(MI))
+    return 0;
+  return AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
+}
+
 static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
                                               Register Reg) {
   if (MI.getParent() == nullptr)
     return nullptr;
   const MachineFunction *MF = MI.getParent()->getParent();
   return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
 }
 

llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp

 // to unwind information.
 //
 //===----------------------------------------------------------------------===//
 
 #include "AArch64InstrInfo.h"
 #include "AArch64MachineFunctionInfo.h"
 #include "AArch64Subtarget.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
+#include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
static unsigned getPreIndexedOpcode(unsigned Opc) {
   case AArch64::STZGOffset:
     return AArch64::STZGPreIndex;
   case AArch64::ST2GOffset:
     return AArch64::ST2GPreIndex;
   case AArch64::STZ2GOffset:
     return AArch64::STZ2GPreIndex;
   case AArch64::STGPi:
     return AArch64::STGPpre;
+  case AArch64::STILPW:
+    return AArch64::STILPWpre;
+  case AArch64::STILPX:
+    return AArch64::STILPXpre;
+  case AArch64::STLRW:
+    return AArch64::STLRWpre;
+  case AArch64::STLRX:
+    return AArch64::STLRXpre;
   }
 }
 
 static unsigned getPostIndexedOpcode(unsigned Opc) {
   switch (Opc) {
   default:
     llvm_unreachable("Opcode has no post-indexed wise equivalent!");
   case AArch64::STRSui:
static unsigned getPostIndexedOpcode(unsigned Opc) {
   case AArch64::LDPDi:
     return AArch64::LDPDpost;
   case AArch64::LDPQi:
     return AArch64::LDPQpost;
   case AArch64::LDPWi:
     return AArch64::LDPWpost;
   case AArch64::LDPXi:
     return AArch64::LDPXpost;
+  case AArch64::LDIAPPW:
+    return AArch64::LDIAPPWpre;
+  case AArch64::LDIAPPX:
+    return AArch64::LDIAPPXpre;
+  case AArch64::LDAPRW:
+    return AArch64::LDAPRWpre;
+  case AArch64::LDAPRX:
+    return AArch64::LDAPRXpre;
+  case AArch64::STLRW:
+    return AArch64::STLRWpre;
+  case AArch64::STLRX:
+    return AArch64::STLRXpre;
   case AArch64::STPSi:
     return AArch64::STPSpost;
   case AArch64::STPDi:
     return AArch64::STPDpost;
   case AArch64::STPQi:
     return AArch64::STPQpost;
   case AArch64::STPWi:
     return AArch64::STPWpost;
case AArch64::LDRQpre:
     return (OpcB == AArch64::LDRQui) || (OpcB == AArch64::LDURQi);
   case AArch64::LDRWpre:
     return (OpcB == AArch64::LDRWui) || (OpcB == AArch64::LDURWi);
   case AArch64::LDRXpre:
     return (OpcB == AArch64::LDRXui) || (OpcB == AArch64::LDURXi);
   }
 }
 
+static bool getRCPC3MemOpInfo(const MachineInstr &MI, int &Scale,
+                              int &MinOffset, int &MaxOffset) {
+  switch (MI.getOpcode()) {
+  default:
+    return false;
+  case AArch64::LDIAPPW:
+    Scale = 1;
+    MinOffset = MaxOffset = 8;
+    return true;
+  case AArch64::STILPW:
+    Scale = 1;
+    MinOffset = MaxOffset = -8;
+    return true;
+  case AArch64::LDIAPPX:
+    Scale = 1;
+    MinOffset = MaxOffset = 16;
+    return true;
+  case AArch64::STILPX:
+    Scale = 1;
+    MinOffset = MaxOffset = -16;
+    return true;
+  case AArch64::LDAPRW:
+    Scale = 1;
+    MinOffset = MaxOffset = 4;
+    return true;
+  case AArch64::STLRWpre:
+    Scale = 1;
+    MinOffset = MaxOffset = -4;
+    return true;
+  case AArch64::LDAPRX:
+    Scale = 1;
+    MinOffset = MaxOffset = 8;
+    return true;
+  case AArch64::STLRXpre:
+    Scale = 1;
+    MinOffset = MaxOffset = -8;
+    return true;
+  }
+}
+
 // Returns the scale and offset range of pre/post indexed variants of MI.
 static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale,
                                        int &MinOffset, int &MaxOffset) {
+  // Special case for LDIAPP/STILP which accept only particular offsets
+  if (getRCPC3MemOpInfo(MI, Scale, MinOffset, MaxOffset))
+    return;
+
   bool IsPaired = AArch64InstrInfo::isPairedLdSt(MI);
   bool IsTagStore = isTagStore(MI);
   // ST*G and all paired ldst have the same scale in pre/post-indexed variants
   // as in the "unsigned offset" variant.
   // All other pre/post indexed ldst instructions are unscaled.
   Scale = (IsTagStore || IsPaired) ? AArch64InstrInfo::getMemScale(MI) : 1;
 
   if (IsPaired) {
static bool isPromotableLoadFromStore(MachineInstr &MI) {
   case AArch64::LDURWi:
   case AArch64::LDURXi:
     return true;
   }
 }
 
 static bool isMergeableLdStUpdate(MachineInstr &MI) {
   unsigned Opc = MI.getOpcode();
+  if (AArch64InstrInfo::hasRCPC3PrePostIndexVariant(MI))
+    return true;
   switch (Opc) {
   default:
     return false;
   // Scaled instructions.
   case AArch64::STRSui:
   case AArch64::STRDui:
   case AArch64::STRQui:
   case AArch64::STRXui:
static bool isMergeableLdStUpdate(MachineInstr &MI) {
   case AArch64::STPSi:
   case AArch64::STPDi:
   case AArch64::STPQi:
   case AArch64::STPWi:
   case AArch64::STPXi:
     // Make sure this is a reg+imm (as opposed to an address reloc).
     if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
       return false;
-
     return true;
   }
 }
 
 MachineBasicBlock::iterator
 AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
                                            MachineBasicBlock::iterator MergeMI,
                                            const LdStPairFlags &Flags) {

 }
 
 // Returns true if FirstMI and MI are candidates for merging or pairing.
 // Otherwise, returns false.
 static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI,
                                        LdStPairFlags &Flags,
                                        const AArch64InstrInfo *TII) {
   // If this is volatile or if pairing is suppressed, not a candidate.
-  if (MI.hasOrderedMemoryRef() || TII->isLdStPairSuppressed(MI))
+  if (MI.hasVolatileMemoryRef() || MI.memoperands_empty())
+    return false;
+  if (TII->isLdStPairSuppressed(MI))
     return false;
 
   // We should have already checked FirstMI for pair suppression and volatility.
-  assert(!FirstMI.hasOrderedMemoryRef() &&
+  assert(!FirstMI.hasVolatileMemoryRef() &&
          !TII->isLdStPairSuppressed(FirstMI) &&
          "FirstMI shouldn't get here if either of these checks are true.");
 
   if (needsWinCFI(MI.getMF()) && (MI.getFlag(MachineInstr::FrameSetup) ||
                                   MI.getFlag(MachineInstr::FrameDestroy)))
     return false;
 
   unsigned OpcA = FirstMI.getOpcode();
   unsigned OpcB = MI.getOpcode();
 
   // Opcodes match: If the opcodes are pre ld/st there is nothing more to check.
   if (OpcA == OpcB)
     return !AArch64InstrInfo::isPreLdSt(FirstMI);
 
+  // For RCPC3, we only merge pairs with matching opcodes.
+  if (AArch64InstrInfo::hasRCPC3PrePostIndexVariant(FirstMI)) {
+    assert(OpcA != OpcB);
+    return false;
+  }
+
   // Try to match a sign-extended load/store with a zero-extended load/store.
   bool IsValidLdStrOpc, PairIsValidLdStrOpc;
   unsigned NonSExtOpc = getMatchingNonSExtOpcode(OpcA, &IsValidLdStrOpc);
   assert(IsValidLdStrOpc &&
          "Given Opc should be a Load or Store with an immediate");
   // OpcA will be the first instruction in the pair.
   if (NonSExtOpc == getMatchingNonSExtOpcode(OpcB, &PairIsValidLdStrOpc)) {
     Flags.setSExtIdx(NonSExtOpc == (unsigned)OpcA ? 1 : 0);
AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
   MachineBasicBlock::iterator MBBIWithRenameReg;
   MachineInstr &FirstMI = *I;
   MBBI = next_nodbg(MBBI, E);
 
   bool MayLoad = FirstMI.mayLoad();
   bool IsUnscaled = TII->hasUnscaledLdStOffset(FirstMI);
   Register Reg = getLdStRegOp(FirstMI).getReg();
   Register BaseReg = AArch64InstrInfo::getLdStBaseOp(FirstMI).getReg();
-  int Offset = AArch64InstrInfo::getLdStOffsetOp(FirstMI).getImm();
+  int Offset = AArch64InstrInfo::getLdStOffset(FirstMI);
   int OffsetStride = IsUnscaled ? TII->getMemScale(FirstMI) : 1;
   bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI);
 
   std::optional<bool> MaybeCanRename;
   if (!EnableRenaming)
     MaybeCanRename = {false};
 
   SmallPtrSet<const TargetRegisterClass *, 5> RequiredClasses;
for (unsigned Count = 0; MBBI != E && Count < Limit;
 
     // Don't count transient instructions towards the search limit since there
     // may be different numbers of them if e.g. debug information is present.
     if (!MI.isTransient())
       ++Count;
 
     Flags.setSExtIdx(-1);
     if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) &&
+        AArch64InstrInfo::hasLdStOffsetOp(MI) &&
         AArch64InstrInfo::getLdStOffsetOp(MI).isImm()) {
       assert(MI.mayLoadOrStore() && "Expected memory operation.");
       // If we've found another instruction with the same opcode, check to see
       // if the base and offset are compatible with our starting instruction.
       // These instructions all have scaled immediate operands, so we just
       // check for +1/-1. Make sure to check the new instruction offset is
       // actually an immediate and not a symbolic reference destined for
       // a relocation.
       Register MIBaseReg = AArch64InstrInfo::getLdStBaseOp(MI).getReg();
-      int MIOffset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
+      int MIOffset = AArch64InstrInfo::getLdStOffset(MI);
       bool MIIsUnscaled = TII->hasUnscaledLdStOffset(MI);
       if (IsUnscaled != MIIsUnscaled) {
         // We're trying to pair instructions that differ in how they are scaled.
         // If FirstMI is scaled then scale the offset of MI accordingly.
         // Otherwise, do the opposite (i.e., make MI's offset unscaled).
         int MemSize = TII->getMemScale(MI);
         if (MIIsUnscaled) {
           // If the unscaled offset isn't a multiple of the MemSize, we can't
AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I,
   int Scale, MinOffset, MaxOffset;
   getPrePostIndexedMemOpInfo(*I, Scale, MinOffset, MaxOffset);
   if (!AArch64InstrInfo::isPairedLdSt(*I)) {
     // Non-paired instruction.
     MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
               .add(getLdStRegOp(*Update))
               .add(getLdStRegOp(*I))
               .add(AArch64InstrInfo::getLdStBaseOp(*I))
-              .addImm(Value / Scale)
               .setMemRefs(I->memoperands())
               .setMIFlags(I->mergeFlagsWith(*Update));
+    if (AArch64InstrInfo::hasLdStOffsetOp(*I))
+      MIB.addImm(Value / Scale);
   } else {
     // Paired instruction.
     MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
               .add(getLdStRegOp(*Update))
               .add(getLdStRegOp(*I, 0))
               .add(getLdStRegOp(*I, 1))
               .add(AArch64InstrInfo::getLdStBaseOp(*I))
-              .addImm(Value / Scale)
               .setMemRefs(I->memoperands())
               .setMIFlags(I->mergeFlagsWith(*Update));
+    if (AArch64InstrInfo::hasLdStOffsetOp(*I))
+      MIB.addImm(Value / Scale);
   }
   if (CFI != E) {
     MachineBasicBlock *MBB = I->getParent();
     MBB->splice(std::next(MIB.getInstr()->getIterator()), MBB, CFI);
   }
 
   if (IsPreIdx) {
     ++NumPreFolded;

 
 MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
     MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
   MachineBasicBlock::iterator E = I->getParent()->end();
   MachineInstr &MemMI = *I;
   MachineBasicBlock::iterator MBBI = I;
 
   Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MemMI).getReg();
-  int MIUnscaledOffset = AArch64InstrInfo::getLdStOffsetOp(MemMI).getImm() *
-                         TII->getMemScale(MemMI);
+
+  int MIUnscaledOffset =
+      AArch64InstrInfo::getLdStOffset(MemMI) * TII->getMemScale(MemMI);
 
   // Scan forward looking for post-index opportunities.  Updating instructions
   // can't be formed if the memory instruction doesn't have the offset we're
   // looking for.
   if (MIUnscaledOffset != UnscaledOffset)
     return E;
 
   // If the base register overlaps a source/destination register, we can't
MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
     MachineBasicBlock::iterator I, unsigned Limit) {
   MachineBasicBlock::iterator B = I->getParent()->begin();
   MachineBasicBlock::iterator E = I->getParent()->end();
   MachineInstr &MemMI = *I;
   MachineBasicBlock::iterator MBBI = I;
   MachineFunction &MF = *MemMI.getMF();
 
   Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MemMI).getReg();
-  int Offset = AArch64InstrInfo::getLdStOffsetOp(MemMI).getImm();
+  int Offset = AArch64InstrInfo::getLdStOffset(MemMI);
 
   // If the load/store is the first instruction in the block, there's obviously
   // not any matching update. Ditto if the memory offset isn't zero.
   if (MBBI == B || Offset != 0)
     return E;
   // If the base register overlaps a destination register, we can't
   // merge the update.
   if (!isTagStore(MemMI)) {
bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
 
   if (!TII->isCandidateToMergeOrPair(MI))
     return false;
 
   // Early exit if the offset is not possible to match. (6 bits of positive
   // range, plus allow an extra one in case we find a later insn that matches
   // with Offset-1)
   bool IsUnscaled = TII->hasUnscaledLdStOffset(MI);
-  int Offset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
+  int Offset = AArch64InstrInfo::getLdStOffset(MI);
   int OffsetStride = IsUnscaled ? TII->getMemScale(MI) : 1;
   // Allow one more for offset.
   if (Offset > 0)
     Offset -= OffsetStride;
   if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
     return false;
 
   // Look ahead up to LdStLimit instructions for a pairable instruction.
if (Update != E) {
     MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/true);
     return true;
   }
 
   // The immediate in the load/store is scaled by the size of the memory
   // operation. The immediate in the add we're looking for,
   // however, is not, so adjust here.
   int UnscaledOffset =
-      AArch64InstrInfo::getLdStOffsetOp(MI).getImm() * TII->getMemScale(MI);
+      AArch64InstrInfo::getLdStOffset(MI) * TII->getMemScale(MI);
 
   // Look forward to try to find a pre-index instruction. For example,
   // ldr x1, [x0, #64]
   // add x0, x0, #64
   //   merged into:
   // ldr x1, [x0, #64]!
   Update = findMatchingUpdateInsnForward(MBBI, UnscaledOffset, UpdateLimit);
   if (Update != E) {

llvm/test/CodeGen/AArch64/aarch64-rcpc3-ldst.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(lda|ldia|stl|stil)"
+
+; RUN: llc %s -o - -mtriple=aarch64-none-linux-gnu -O1 -mattr=+v8.1a,+rcpc,+rcpc-immo,+rcpc3 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-NO-LSE2
+; RUN: llc %s -o - -mtriple=aarch64-none-linux-gnu -O1 -mattr=+v8.1a,+rcpc,+rcpc-immo,+rcpc3                                   | FileCheck %s --check-prefixes=SDAG-NO-LSE2
+; RUN: llc %s -o - -mtriple=aarch64-none-linux-gnu -O1 -mattr=+v8.1a,+lse2,+rcpc,+rcpc-immo,+rcpc3 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-LSE2
+; RUN: llc %s -o - -mtriple=aarch64-none-linux-gnu -O1 -mattr=+v8.1a,+lse2,+rcpc,+rcpc-immo,+rcpc3                                   | FileCheck %s --check-prefixes=SDAG-LSE2
+; RUN: llc %s -o - -mtriple=aarch64_be-none-linux-gnu -O1 -mattr=+v8.1a,+rcpc,+rcpc-immo,+rcpc3       | FileCheck %s --check-prefixes=SDAG-NO-LSE2
+; RUN: llc %s -o - -mtriple=aarch64_be-none-linux-gnu -O1 -mattr=+v8.1a,+lse2,+rcpc,+rcpc-immo,+rcpc3 | FileCheck %s --check-prefixes=SDAG-LSE2
+
+; TODO:
+; SelDAG implementation
+; merge two ordered load-aqcuires into a load-acquire pair
+; merging of pre/post indexed variants
+; LDIAPP only generated if we have LSE
+; LSE2 makes pair operations single-copy atomic for naturally aligned
+
+; Note:
+; Loading the pointer indirectly results in an update operation (add/sub)
+; which reuses the same src/dst register (x8).
+; Doing getelementptr on the input ptr directly results in `x8 = add blah`.
+
+; RCPC3 lets us merge two 32/64 bit atomic loads into a single one.
+define dso_local void @load_store_2xi32_rcpc3(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: load_store_2xi32_rcpc3:
+; GISEL-NO-LSE2:    ldapr w10, [x8]
+; GISEL-NO-LSE2:    ldapr w11, [x9]
+; GISEL-NO-LSE2:    stlr w10, [x8]
+; GISEL-NO-LSE2:    stlr w11, [x9]
+;
+; SDAG-NO-LSE2-LABEL: load_store_2xi32_rcpc3:
+; SDAG-NO-LSE2:    ldapr w10, [x8]
+; SDAG-NO-LSE2:    ldapr w11, [x9]
+; SDAG-NO-LSE2:    stlr w10, [x8]
+; SDAG-NO-LSE2:    stlr w11, [x9]
+;
+; GISEL-LSE2-LABEL: load_store_2xi32_rcpc3:
+; GISEL-LSE2:    ldapr w10, [x8]
+; GISEL-LSE2:    ldapr w11, [x9]
+; GISEL-LSE2:    stlr w10, [x8]
+; GISEL-LSE2:    stlr w11, [x9]
+;
+; SDAG-LSE2-LABEL: load_store_2xi32_rcpc3:
+; SDAG-LSE2:    ldapr w10, [x8]
+; SDAG-LSE2:    ldapr w11, [x9]
+; SDAG-LSE2:    stlr w10, [x8]
+; SDAG-LSE2:    stlr w11, [x9]
+    %ptr1 = load ptr, ptr %ptr2ptr
+    %ptr2 = getelementptr ptr, ptr %ptr1, i32 1
+
+    %a1 = load atomic i32, ptr %ptr1 acquire, align 8
+    %a2 = load atomic i32, ptr %ptr2 acquire, align 8
+
+    %b1 = add i32 %a1, %a1
+    %b2 = add i32 %a2, %a2
+
+    store atomic i32 %b1, ptr %ptr1 release, align 8
+    store atomic i32 %b2, ptr %ptr2 release, align 8
+
+    ret void
+}
+
+define dso_local void @load_store_2xi64_rcpc3(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: load_store_2xi64_rcpc3:
+; GISEL-NO-LSE2:    ldapr x10, [x8]
+; GISEL-NO-LSE2:    ldapr x11, [x9]
+; GISEL-NO-LSE2:    stlr x10, [x8]
+; GISEL-NO-LSE2:    stlr x11, [x9]
+;
+; SDAG-NO-LSE2-LABEL: load_store_2xi64_rcpc3:
+; SDAG-NO-LSE2:    ldapr x10, [x8]
+; SDAG-NO-LSE2:    ldapr x11, [x9]
+; SDAG-NO-LSE2:    stlr x10, [x8]
+; SDAG-NO-LSE2:    stlr x11, [x9]
+;
+; GISEL-LSE2-LABEL: load_store_2xi64_rcpc3:
+; GISEL-LSE2:    ldapr x10, [x8]
+; GISEL-LSE2:    ldapr x11, [x9]
+; GISEL-LSE2:    stlr x10, [x8]
+; GISEL-LSE2:    stlr x11, [x9]
+;
+; SDAG-LSE2-LABEL: load_store_2xi64_rcpc3:
+; SDAG-LSE2:    ldapr x10, [x8]
+; SDAG-LSE2:    ldapr x11, [x9]
+; SDAG-LSE2:    stlr x10, [x8]
+; SDAG-LSE2:    stlr x11, [x9]
+    %ptr1 = load ptr, ptr %ptr2ptr
+    %ptr2 = getelementptr ptr, ptr %ptr1, i64 1
+
+    %a1 = load atomic i64, ptr %ptr1 acquire, align 8
+    %a2 = load atomic i64, ptr %ptr2 acquire, align 8
+
+    %b1 = add i64 %a1, %a1
+    %b2 = add i64 %a2, %a2
+
+    store atomic i64 %b1, ptr %ptr1 release, align 8
+    store atomic i64 %b2, ptr %ptr2 release, align 8
+
+    ret void
+}
+
+; TODO Same again but with offsets
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Register Pair Ordered - Load Acquire RCpc / Store Release ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; RCPC3 + LSE2 gives us 128-bit single-copy atomics.
+define dso_local void @load_atomic_i128_no_offset(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: load_atomic_i128_no_offset:
+; GISEL-NO-LSE2:    ldiapp xzr, x8, [x8]
+;
+; SDAG-NO-LSE2-LABEL: load_atomic_i128_no_offset:
+; SDAG-NO-LSE2:    ldiapp xzr, x8, [x8]
+;
+; GISEL-LSE2-LABEL: load_atomic_i128_no_offset:
+; GISEL-LSE2:    ldiapp xzr, x8, [x8]
+;
+; SDAG-LSE2-LABEL: load_atomic_i128_no_offset:
+; SDAG-LSE2:    ldiapp xzr, x8, [x8]
+    %ptr = load ptr, ptr %ptr2ptr
+    %a = load atomic i128, ptr %ptr acquire, align 16
+    ret void
+}
+
+define dso_local void @store_atomic_i128_no_offset(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: store_atomic_i128_no_offset:
+; GISEL-NO-LSE2:    stilp x8, xzr, [x9]
+;
+; SDAG-NO-LSE2-LABEL: store_atomic_i128_no_offset:
+; SDAG-NO-LSE2:    stilp x8, xzr, [x9]
+;
+; GISEL-LSE2-LABEL: store_atomic_i128_no_offset:
+; GISEL-LSE2:    stilp x8, xzr, [x9]
+;
+; SDAG-LSE2-LABEL: store_atomic_i128_no_offset:
+; SDAG-LSE2:    stilp x8, xzr, [x9]
+    %ptr = load ptr, ptr %ptr2ptr
+    store atomic i128 1, ptr %ptr release, align 16
+    ret void
+}
+
+; Same again with pre/post indexing.
+define dso_local void @load_atomic_i128_offset_16(ptr %base_ptr) {
+; GISEL-NO-LSE2-LABEL: load_atomic_i128_offset_16:
+; GISEL-NO-LSE2:    ldiapp x8, x9, [x0], #16
+;
+; SDAG-NO-LSE2-LABEL: load_atomic_i128_offset_16:
+; SDAG-NO-LSE2:    ldiapp x8, x9, [x0], #16
+;
+; GISEL-LSE2-LABEL: load_atomic_i128_offset_16:
+; GISEL-LSE2:    ldiapp x8, x9, [x0], #16
+;
+; SDAG-LSE2-LABEL: load_atomic_i128_offset_16:
+; SDAG-LSE2:    ldiapp x8, x9, [x0], #16
+entry:
+  br label %body
+
+body:
+  %ptr = phi ptr [ %offset_ptr, %body ], [ %base_ptr, %entry ]
+
+  ; Machine scheduler has a tendency to move the ADD before the LOAD.
+  ; Hence the loop, to ensure the ADD reuses one register for src and dst.
+  %val = load atomic i128, ptr %ptr acquire, align 16
+  %offset_ptr = getelementptr i64, ptr %ptr, i64 2
+
+  ; %val is used for %cond to ensure the LOAD is not removed.
+  %cond = icmp eq i128 %val, 0
+  br i1 %cond, label %exit, label %body
+
+exit:
+  ret void
+}
+
+define dso_local void @store_atomic_i128_offset_16(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: store_atomic_i128_offset_16:
+; GISEL-NO-LSE2:    stilp x8, xzr, [x9, #-16]!
+;
+; SDAG-NO-LSE2-LABEL: store_atomic_i128_offset_16:
+; SDAG-NO-LSE2:    stilp x8, xzr, [x9, #-16]!
+;
+; GISEL-LSE2-LABEL: store_atomic_i128_offset_16:
+; GISEL-LSE2:    stilp x8, xzr, [x9, #-16]!
+;
+; SDAG-LSE2-LABEL: store_atomic_i128_offset_16:
+; SDAG-LSE2:    stilp x8, xzr, [x9, #-16]!
+    %ptr = load ptr, ptr %ptr2ptr
+    %ptr_wb = getelementptr i64, ptr %ptr, i64 -2
+    store atomic i128 1, ptr %ptr_wb release, align 16
+    ret void
+}
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Single register - Load Acquire RCpc ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; LDAPR <Xt>,[<Xn|SP>],#8 Load Acquire RCpc Register, post index(8) with writeback
+; LDAPR <Wt>,[<Xn|SP>],#4 Load Acquire RCpc Register, post index(4) with writeback
+
+define dso_local i64 @load_atomic_i64_no_offset(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: load_atomic_i64_no_offset:
+; GISEL-NO-LSE2:    ldapr x0, [x8]
+;
+; SDAG-NO-LSE2-LABEL: load_atomic_i64_no_offset:
+; SDAG-NO-LSE2:    ldapr x0, [x8]
+;
+; GISEL-LSE2-LABEL: load_atomic_i64_no_offset:
+; GISEL-LSE2:    ldapr x0, [x8]
+;
+; SDAG-LSE2-LABEL: load_atomic_i64_no_offset:
+; SDAG-LSE2:    ldapr x0, [x8]
+    %ptr = load ptr, ptr %ptr2ptr
+    %a = load atomic i64, ptr %ptr acquire, align 8
+    ret i64 %a
+}
+
+define dso_local i32 @load_atomic_i32_no_offset(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: load_atomic_i32_no_offset:
+; GISEL-NO-LSE2:    ldapr w0, [x8]
+;
+; SDAG-NO-LSE2-LABEL: load_atomic_i32_no_offset:
+; SDAG-NO-LSE2:    ldapr w0, [x8]
+;
+; GISEL-LSE2-LABEL: load_atomic_i32_no_offset:
+; GISEL-LSE2:    ldapr w0, [x8]
+;
+; SDAG-LSE2-LABEL: load_atomic_i32_no_offset:
+; SDAG-LSE2:    ldapr w0, [x8]
+    %ptr = load ptr, ptr %ptr2ptr
+    %a = load atomic i32, ptr %ptr acquire, align 4
+    ret i32 %a
+}
+
+define dso_local void @load_atomic_i64_offset_8(ptr %base_ptr) {
+; GISEL-NO-LSE2-LABEL: load_atomic_i64_offset_8:
+; GISEL-NO-LSE2:    ldapr x8, [x0], #8
+;
+; SDAG-NO-LSE2-LABEL: load_atomic_i64_offset_8:
+; SDAG-NO-LSE2:    ldapr x8, [x0], #8
+;
+; GISEL-LSE2-LABEL: load_atomic_i64_offset_8:
+; GISEL-LSE2:    ldapr x8, [x0], #8
+;
+; SDAG-LSE2-LABEL: load_atomic_i64_offset_8:
+; SDAG-LSE2:    ldapr x8, [x0], #8
+entry:
+  br label %body
+
+body:
+  %ptr = phi ptr [ %offset_ptr, %body ], [ %base_ptr, %entry ]
+
+  ; Machine scheduler has a tendency to move the ADD before the LOAD.
+  ; Hence the loop, to ensure the ADD reuses one register for src and dst.
+  %val = load atomic i64, ptr %ptr acquire, align 8
+  %offset_ptr = getelementptr i64, ptr %ptr, i64 1
+
+  ; %val is used for %cond to ensure the LOAD is not removed.
+  %cond = icmp eq i64 %val, 0
+  br i1 %cond, label %exit, label %body
+
+exit:
+  ret void
+}
+
+define dso_local void @load_atomic_i32_offset_4(ptr %base_ptr) {
+; GISEL-NO-LSE2-LABEL: load_atomic_i32_offset_4:
+; GISEL-NO-LSE2:    ldapr w8, [x0], #4
+;
+; SDAG-NO-LSE2-LABEL: load_atomic_i32_offset_4:
+; SDAG-NO-LSE2:    ldapr w8, [x0], #4
+;
+; GISEL-LSE2-LABEL: load_atomic_i32_offset_4:
+; GISEL-LSE2:    ldapr w8, [x0], #4
+;
+; SDAG-LSE2-LABEL: load_atomic_i32_offset_4:
+; SDAG-LSE2:    ldapr w8, [x0], #4
+entry:
+  br label %body
+
+body:
+  %ptr = phi ptr [ %offset_ptr, %body ], [ %base_ptr, %entry ]
+
+  ; Machine scheduler has a tendency to move the ADD before the LOAD.
+  ; Hence the loop, to ensure the ADD reuses one register for src and dst.
+  %val = load atomic i32, ptr %ptr acquire, align 4
+  %offset_ptr = getelementptr i32, ptr %ptr, i64 1
+
+  ; %val is used for %cond to ensure the LOAD is not removed.
+  %cond = icmp eq i32 %val, 0
+  br i1 %cond, label %exit, label %body
+
+exit:
+  ret void
+}
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Single register – Store Release ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; STLR <Xt>,[<Xn|SP>, #-8]! Store Release Register, negative pre index(8) with writeback
+; STLR <Wt>,[<Xn|SP>, #-4]! Store Release Register, negative pre index(4) with writeback
+
+define dso_local void @store_atomic_i32_no_offset(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: store_atomic_i32_no_offset:
+; GISEL-NO-LSE2:    stlr w8, [x9]
+;
+; SDAG-NO-LSE2-LABEL: store_atomic_i32_no_offset:
+; SDAG-NO-LSE2:    stlr w8, [x9]
+;
+; GISEL-LSE2-LABEL: store_atomic_i32_no_offset:
+; GISEL-LSE2:    stlr w8, [x9]
+;
+; SDAG-LSE2-LABEL: store_atomic_i32_no_offset:
+; SDAG-LSE2:    stlr w8, [x9]
+    %ptr = load ptr, ptr %ptr2ptr
+    store atomic i32 1, ptr %ptr release, align 4
+    ret void
+}
+
+define dso_local void @store_atomic_i64_no_offset(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: store_atomic_i64_no_offset:
+; GISEL-NO-LSE2:    stlr x8, [x9]
+;
+; SDAG-NO-LSE2-LABEL: store_atomic_i64_no_offset:
+; SDAG-NO-LSE2:    stlr x8, [x9]
+;
+; GISEL-LSE2-LABEL: store_atomic_i64_no_offset:
+; GISEL-LSE2:    stlr x8, [x9]
+;
+; SDAG-LSE2-LABEL: store_atomic_i64_no_offset:
+; SDAG-LSE2:    stlr x8, [x9]
+    %ptr = load ptr, ptr %ptr2ptr
+    store atomic i64 1, ptr %ptr release, align 8
+    ret void
+}
+
+
+define dso_local void @store_atomic_i32_offset_4(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: store_atomic_i32_offset_4:
+; GISEL-NO-LSE2:    stlr w8, [x9, #-4]!
+;
+; SDAG-NO-LSE2-LABEL: store_atomic_i32_offset_4:
+; SDAG-NO-LSE2:    stlr w8, [x9, #-4]!
+;
+; GISEL-LSE2-LABEL: store_atomic_i32_offset_4:
+; GISEL-LSE2:    stlr w8, [x9, #-4]!
+;
+; SDAG-LSE2-LABEL: store_atomic_i32_offset_4:
+; SDAG-LSE2:    stlr w8, [x9, #-4]!
+    %ptr = load ptr, ptr %ptr2ptr
+    %ptr_wb = getelementptr i32, ptr %ptr, i64 -1
+    store atomic i32 1, ptr %ptr_wb release, align 4
+    ret void
+}
+
+define dso_local void @store_atomic_i64_offset_8(ptr %ptr2ptr) {
+; GISEL-NO-LSE2-LABEL: store_atomic_i64_offset_8:
+; GISEL-NO-LSE2:    stlr x8, [x9, #-8]!
+;
+; SDAG-NO-LSE2-LABEL: store_atomic_i64_offset_8:
+; SDAG-NO-LSE2:    stlr x8, [x9, #-8]!
+;
+; GISEL-LSE2-LABEL: store_atomic_i64_offset_8:
+; GISEL-LSE2:    stlr x8, [x9, #-8]!
+;
+; SDAG-LSE2-LABEL: store_atomic_i64_offset_8:
+; SDAG-LSE2:    stlr x8, [x9, #-8]!
+    %ptr = load ptr, ptr %ptr2ptr
+    %ptr_wb = getelementptr i64, ptr %ptr, i64 -1
+    store atomic i64 1, ptr %ptr_wb release, align 8
+    ret void
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; 3.1.2 Additions to the Advanced SIMD and floating-point ISA - Register variant ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; LDAPUR <Bt>, [<Xn|SP>{, #<simm>}] // Zeroing
+; LDAPUR <Ht>, [<Xn|SP>{, #<simm>}] // Zeroing
+; LDAPUR <St>, [<Xn|SP>{, #<simm>}] // Zeroing
+; LDAPUR <Dt>, [<Xn|SP>{, #<simm>}] // Zeroing
+; LDAPUR <Qt>, [<Xn|SP>{, #<simm>}]
+; STLUR <Bt>, [<Xn|SP>{, #<simm>}]
+; STLUR <Ht>, [<Xn|SP>{, #<simm>}]
+; STLUR <St>, [<Xn|SP>{, #<simm>}]
+; STLUR <Dt>, [<Xn|SP>{, #<simm>}]
+; STLUR <Qt>, [<Xn|SP>{, #<simm>}]
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; 3.1.2 Additions to the Advanced SIMD and floating-point ISA - Register Index variant ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; LDAP1 {<Vt>.D }[index], [<Xn|SP>] // Merging
+; STL1 {<Vt>.D }[index], [<Xn|SP>]