[AARCH64] Gang up loads and stores (for memcpy) for pairing.

[SirishP]

Summary

This is continuation of https://reviews.llvm.org/D45098. This patch gangs up loads and stores for pairing (in case of memcpy). We address only memcpy and not other memory operations, because in mempy source and destination addresses are disjoint and hence no alias analysis is required to disambiguate loads and stores.

Each target defines numbers of loads and stores to be ganged up. For Aarch64, it is set at 4. Other targets have defaulted to 0, and hence no effect.

[sebpop]

llvm/include/llvm/CodeGen/TargetLowering.h

2519

s/inline/inlined/

2520

s/that you can want//
s/tother/together/

2521

s/vectorazation/vectorization/

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

92

To avoid double negative, which is confusing, please rename this command line option as EnableMem... .

95

remove this second part of the sentence.

5363

Remove empty line.

5397

remove parentheses

5402

The first part is not a sentence, please rewrite in a non-ambiguous way.
s/maybe/may be/
s/mempy/memcpy/

5406

clang-format

5412

clang-format

5424

remove parentheses

5425

remove parentheses

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

582

Let's commit this change https://reviews.llvm.org/D45098 separately after this patch lands.

llvm/test/CodeGen/AArch64/arm64-memset-to-bzero.ll

11

You won't need to change this testcase if you remove the change to "MaxStoresPerMemset = 32;"

89

Do we need to change the testcase here? Why?

llvm/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll

10

Do we need to change the testcase here? Why?

[SirishP]

llvm/test/CodeGen/AArch64/arm64-memset-to-bzero.ll

89

This was done due to change in maxstore per memset. If we change it back to the original value, we don't need this change for now.

[sebpop]

LGTM with some minor changes.

llvm/include/llvm/CodeGen/TargetLowering.h

2520

s/that you can want//

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

5243

Dot at the end of a sentence: "Use LoadToken to chain all loads."

5402

s/mempy/memcpy/

5412

else should be on the same line as close brace:

} else {

5432

Put a dot at end of a sentence.

[sebpop]

Committed in https://reviews.llvm.org/rL332482

[lebedev.ri]

You know you can either just use arc patch, and automagically get a
nice commit msg,
or at least manually add "Differential Revision: link", and it will
get closed automatically?

[sebpop]

In D46477#1108188, @lebedev.ri wrote:

You know you can either just use arc patch, and automagically get a
nice commit msg,
or at least manually add "Differential Revision: link", and it will
get closed automatically?

yup, the problem is when people commit with a message containing the magic keywords and forget a column...

llvm/include/llvm/CodeGen/TargetLowering.h

public:
   /// This function returns the maximum number of store operations permitted
   /// to replace a call to llvm.memcpy. The value is set by the target at the
   /// performance threshold for such a replacement. If OptSize is true,
   /// return the limit for functions that have OptSize attribute.
   unsigned getMaxStoresPerMemcpy(bool OptSize) const {
     return OptSize ? MaxStoresPerMemcpyOptSize : MaxStoresPerMemcpy;
   }
 
+  /// \brief Get maximum # of store operations to be glued together
+  ///
+  /// This function returns the maximum number of store operations permitted
+  /// to glue together during lowering of llvm.memcpy. The value is set by
+  //  the target at the performance threshold for such a replacement.
+  virtual unsigned getMaxGluedStoresPerMemcpy() const {
+    return MaxGluedStoresPerMemcpy;
+  }
+
   /// Get maximum # of load operations permitted for memcmp
   ///
   /// This function returns the maximum number of load operations permitted
   /// to replace a call to memcmp. The value is set by the target at the
   /// performance threshold for such a replacement. If OptSize is true,
   /// return the limit for functions that have OptSize attribute.
   unsigned getMaxExpandSizeMemcmp(bool OptSize) const {
     return OptSize ? MaxLoadsPerMemcmpOptSize : MaxLoadsPerMemcmp;
protected:
   /// should assume that the memcpy will be done using as many of the largest
   /// store operations first, followed by smaller ones, if necessary, per
   /// alignment restrictions. For example, storing 7 bytes on a 32-bit machine
   /// with 32-bit alignment would result in one 4-byte store, a one 2-byte store
   /// and one 1-byte store. This only applies to copying a constant array of
   /// constant size.
   unsigned MaxStoresPerMemcpy;
 
+
+  /// \brief Specify max number of store instructions to glue in inlined memcpy.
+  ///
+  /// When memcpy is inline based on MaxStoresPerMemcpy, specify maximum number
+  /// of store instructions that you can want to keep tother. This helps in
+  /// pairing and vectorazation later on.
+  unsigned MaxGluedStoresPerMemcpy = 0;
+
   /// Maximum number of store operations that may be substituted for a call to
   /// memcpy, used for functions with OptSize attribute.
   unsigned MaxStoresPerMemcpyOptSize;
   unsigned MaxLoadsPerMemcmp;
   unsigned MaxLoadsPerMemcmpOptSize;
 
   /// Specify maximum bytes of store instructions per memmove call.
   ///

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

 }
 
 // Default null implementations of the callbacks.
 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
 
 #define DEBUG_TYPE "selectiondag"
 
+static cl::opt<bool> DisableMemCpyDAGOpt("disable-memcpy-dag-opt",
+       cl::Hidden, cl::init(false),
+       cl::desc("Gang up loads and stores generated by inlining of memcpy "
+                "such that such loads and stores can be paired."));
+
+static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
+       cl::desc("Number limit for gluing ld/st of memcpy."),
+       cl::Hidden, cl::init(0));
+
 static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
   DEBUG(
     dbgs() << Msg;
     V.getNode()->dump(G);
   );
 }
 
 //===----------------------------------------------------------------------===//

 static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
   // On Darwin, -Os means optimize for size without hurting performance, so
   // only really optimize for size when -Oz (MinSize) is used.
   if (MF.getTarget().getTargetTriple().isOSDarwin())
     return MF.getFunction().optForMinSize();
   return MF.getFunction().optForSize();
 }
 
+static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
+                          SmallVector<SDValue, 32> &OutChains, unsigned From,
+                          unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
+                          SmallVector<SDValue, 16> &OutStoreChains) {
+  assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
+  assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
+  SmallVector<SDValue, 16> GluedLoadChains;
+  for (unsigned i = From; i < To; ++i) {
+    OutChains.push_back(OutLoadChains[i]);
+    GluedLoadChains.push_back(OutLoadChains[i]);
+  }
+
+  // Chain for all loads
+  SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
+                                  GluedLoadChains);
+
+  for (unsigned i = From; i < To; ++i) {
+    StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
+    SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
+                                  ST->getBasePtr(), ST->getMemoryVT(),
+                                  ST->getMemOperand());
+    OutChains.push_back(NewStore);
+  }
+}
+
 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
                                        SDValue Chain, SDValue Dst, SDValue Src,
                                        uint64_t Size, unsigned Align,
                                        bool isVol, bool AlwaysInline,
                                        MachinePointerInfo DstPtrInfo,
                                        MachinePointerInfo SrcPtrInfo) {
   // Turn a memcpy of undef to nop.
   if (Src.isUndef())
if (NewAlign > Align) {
       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
       Align = NewAlign;
     }
   }
 
   MachineMemOperand::Flags MMOFlags =
       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
-  SmallVector<SDValue, 8> OutChains;
+  SmallVector<SDValue, 16> OutLoadChains;
+  SmallVector<SDValue, 16> OutStoreChains;
+  SmallVector<SDValue, 32> OutChains;
   unsigned NumMemOps = MemOps.size();
   uint64_t SrcOff = 0, DstOff = 0;
   for (unsigned i = 0; i != NumMemOps; ++i) {
     EVT VT = MemOps[i];
     unsigned VTSize = VT.getSizeInBits() / 8;
     SDValue Value, Store;
 
     if (VTSize > Size) {
if (CopyFromConstant &&
         SubSlice.move(SrcOff);
       } else {
         // This is an out-of-bounds access and hence UB. Pretend we read zero.
         SubSlice.Array = nullptr;
         SubSlice.Offset = 0;
         SubSlice.Length = VTSize;
       }
       Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
-      if (Value.getNode())
+      if (Value.getNode()) {
         Store = DAG.getStore(Chain, dl, Value,
                              DAG.getMemBasePlusOffset(Dst, DstOff, dl),
                              DstPtrInfo.getWithOffset(DstOff), Align,
                              MMOFlags);
+        OutChains.push_back(Store);
+      }
+
     }
 
     if (!Store.getNode()) {
       // The type might not be legal for the target.  This should only happen
       // if the type is smaller than a legal type, as on PPC, so the right
       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
       // to Load/Store if NVT==VT.
       // FIXME does the case above also need this?
       EVT NVT = TLI.getTypeToTransformTo(C, VT);
       assert(NVT.bitsGE(VT));
 
       bool isDereferenceable =
         SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
       MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
       if (isDereferenceable)
         SrcMMOFlags |= MachineMemOperand::MODereferenceable;
 
       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
                              DAG.getMemBasePlusOffset(Src, SrcOff, dl),
                              SrcPtrInfo.getWithOffset(SrcOff), VT,
                              MinAlign(SrcAlign, SrcOff), SrcMMOFlags);
-      OutChains.push_back(Value.getValue(1));
+      OutLoadChains.push_back(Value.getValue(1));
+
       Store = DAG.getTruncStore(
           Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
           DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags);
+      OutStoreChains.push_back(Store);
     }
-    OutChains.push_back(Store);
     SrcOff += VTSize;
     DstOff += VTSize;
     Size -= VTSize;
   }
 
+  unsigned GluedLdStLimit = (MaxLdStGlue == 0 ?
+                                TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue);
+  unsigned NumLdStInMemcpy = OutStoreChains.size();
+
+  if (NumLdStInMemcpy) {
+  // Only if we have load/stores because of memcpy. It maybe that mempy
+  // might be converted to memset if it's memcpy of constants. In that case,
+  // ignore all these.
+    if ((GluedLdStLimit <= 1) || DisableMemCpyDAGOpt) {
+    // If target does not care, just leave as it.
+      for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
+        OutChains.push_back(OutLoadChains[i]);
+        OutChains.push_back(OutStoreChains[i]);
+      }
+    }
+    else {
+      // Ld/St less than/equal limit set by target.
+      if (NumLdStInMemcpy <= GluedLdStLimit) {
+          chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
+                                        NumLdStInMemcpy, OutLoadChains,
+                                        OutStoreChains);
+      } else {
+        unsigned NumberLdChain =  NumLdStInMemcpy / GluedLdStLimit;
+        unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
+        unsigned GlueIter = 0;
+
+        for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
+          unsigned IndexFrom = (NumLdStInMemcpy - GlueIter - GluedLdStLimit);
+          unsigned IndexTo   = (NumLdStInMemcpy - GlueIter);
+
+          chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
+                                       OutLoadChains, OutStoreChains);
+          GlueIter += GluedLdStLimit;
+        }
+
+        // Residual ld/st
+        if (RemainingLdStInMemcpy) {
+          chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
+                                        RemainingLdStInMemcpy, OutLoadChains,
+                                        OutStoreChains);
+        }
+      }
+    }
+  }
   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
 }
 
 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
                                         SDValue Chain, SDValue Dst, SDValue Src,
                                         uint64_t Size, unsigned Align,
                                         bool isVol, bool AlwaysInline,
                                         MachinePointerInfo DstPtrInfo,

llvm/lib/CodeGen/TargetLoweringBase.cpp

 
 /// NOTE: The TargetMachine owns TLOF.
 TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
   initActions();
 
   // Perform these initializations only once.
   MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove =
       MaxLoadsPerMemcmp = 8;
+  MaxGluedStoresPerMemcpy = 0;
   MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize =
       MaxStoresPerMemmoveOptSize = MaxLoadsPerMemcmpOptSize = 4;
   UseUnderscoreSetJmp = false;
   UseUnderscoreLongJmp = false;
   HasMultipleConditionRegisters = false;
   HasExtractBitsInsn = false;
   JumpIsExpensive = JumpIsExpensiveOverride;
   PredictableSelectIsExpensive = false;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::VSELECT);
 
   setTargetDAGCombine(ISD::INTRINSIC_VOID);
   setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
   setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
 
   setTargetDAGCombine(ISD::GlobalAddress);
 
-  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = 8;
-  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = 4;
-  MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 4;
+  MaxStoresPerMemset = 32;
+  MaxStoresPerMemsetOptSize = 8;
+
+  MaxStoresPerMemcpy = 16;
+  MaxStoresPerMemcpyOptSize = 4;
+  MaxGluedStoresPerMemcpy = 4;
+
+  MaxStoresPerMemmove = 16;
+  MaxStoresPerMemmoveOptSize = 4;
 
   setStackPointerRegisterToSaveRestore(AArch64::SP);
 
   setSchedulingPreference(Sched::Hybrid);
 
   EnableExtLdPromotion = true;
 
   // Set required alignment.

llvm/test/CodeGen/AArch64/arm64-2012-05-07-MemcpyAlignBug.ll

 ; RUN: llc < %s -mtriple=arm64-eabi -mcpu=cyclone | FileCheck %s
 ; <rdar://problem/11294426>
 
 @b = private unnamed_addr constant [3 x i32] [i32 1768775988, i32 1685481784, i32 1836253201], align 4
 
 ; The important thing for this test is that we need an unaligned load of `l_b'
 ; ("ldr w2, [x1, #8]" in this case).
 
 ; CHECK:      adrp x[[PAGE:[0-9]+]], {{l_b@PAGE|.Lb}}
 ; CHECK: add  x[[ADDR:[0-9]+]], x[[PAGE]], {{l_b@PAGEOFF|:lo12:.Lb}}
+; CHECK-NEXT: ldr  [[VAL2:x[0-9]+]], [x[[ADDR]]]
 ; CHECK-NEXT: ldr  [[VAL:w[0-9]+]], [x[[ADDR]], #8]
 ; CHECK-NEXT: str  [[VAL]], [x0, #8]
-; CHECK-NEXT: ldr  [[VAL2:x[0-9]+]], [x[[ADDR]]]
 ; CHECK-NEXT: str  [[VAL2]], [x0]
 
 define void @foo(i8* %a) {
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast ([3 x i32]* @b to i8*), i64 12, i1 false)
   ret void
 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind

llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll

 ; CHECK: str [[REG2]],
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 getelementptr inbounds (%struct.x, %struct.x* @dst, i32 0, i32 0), i8* align 8 getelementptr inbounds (%struct.x, %struct.x* @src, i32 0, i32 0), i32 11, i1 false)
   ret i32 0
 }
 
 define void @t1(i8* nocapture %C) nounwind {
 entry:
 ; CHECK-LABEL: t1:
-; CHECK: ldur [[DEST:q[0-9]+]], [x[[BASEREG:[0-9]+]], #15]
-; CHECK: stur [[DEST]], [x0, #15]
 ; CHECK: ldr [[DEST:q[0-9]+]], [x[[BASEREG]]]
-; CHECK: str [[DEST]], [x0]
+; CHECK: ldur [[DEST:q[0-9]+]], [x[[BASEREG:[0-9]+]], #15]
+; CHECK: stur [[DEST:q[0-9]+]], [x0, #15]
+; CHECK: str [[DEST:q[0-9]+]], [x0]
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([31 x i8], [31 x i8]* @.str1, i64 0, i64 0), i64 31, i1 false)
   ret void
 }
 
 define void @t2(i8* nocapture %C) nounwind {
 entry:
 ; CHECK-LABEL: t2:
 ; CHECK: mov [[REG3:w[0-9]+]]
 ; CHECK: movk [[REG3]],
 ; CHECK: str [[REG3]], [x0, #32]
 ; CHECK: ldp [[DEST1:q[0-9]+]], [[DEST2:q[0-9]+]], [x{{[0-9]+}}]
 ; CHECK: stp [[DEST1]], [[DEST2]], [x0]
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([36 x i8], [36 x i8]* @.str2, i64 0, i64 0), i64 36, i1 false)
   ret void
 }
 
 define void @t3(i8* nocapture %C) nounwind {
 entry:
 ; CHECK-LABEL: t3:
+; CHECK: ldr [[DEST:q[0-9]+]], [x[[BASEREG]]]
 ; CHECK: ldr [[REG4:x[0-9]+]], [x[[BASEREG:[0-9]+]], #16]
 ; CHECK: str [[REG4]], [x0, #16]
-; CHECK: ldr [[DEST:q[0-9]+]], [x[[BASEREG]]]
 ; CHECK: str [[DEST]], [x0]
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([24 x i8], [24 x i8]* @.str3, i64 0, i64 0), i64 24, i1 false)
   ret void
 }
 
 define void @t4(i8* nocapture %C) nounwind {
 entry:
 ; CHECK-LABEL: t4:

llvm/test/CodeGen/AArch64/arm64-memset-to-bzero.ll

 ; RUN: llc %s -mtriple=arm64-apple-darwin -o - | \
 ; RUN:   FileCheck --check-prefix=CHECK-DARWIN --check-prefix=CHECK %s
 ; RUN: llc %s -mtriple=arm64-linux-gnu -o - | \
 ; RUN:   FileCheck --check-prefix=CHECK-LINUX --check-prefix=CHECK %s
 ; <rdar://problem/14199482> ARM64: Calls to bzero() replaced with calls to memset()
 
 ; CHECK-LABEL: fct1:
 ; For small size (<= 256), we do not change memset to bzero.
-; CHECK-DARWIN: {{b|bl}} _memset
-; CHECK-LINUX: {{b|bl}} memset
+; Inline memset.
+; CHECK: stp [[R0:q[0-9]+]], [[R0:q[0-9]+]], [x0, #224]
+; CHECK: stp [[R0:q[0-9]+]], [[R0:q[0-9]+]], [x0]
 define void @fct1(i8* nocapture %ptr) {
 entry:
   tail call void @llvm.memset.p0i8.i64(i8* %ptr, i8 0, i64 256, i1 false)
   ret void
 }
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
 

 define void @fct3(i8* nocapture %ptr, i32 %unknown) {
 entry:
   %conv = sext i32 %unknown to i64
   tail call void @llvm.memset.p0i8.i64(i8* %ptr, i8 0, i64 %conv, i1 false)
   ret void
 }
 
 ; CHECK-LABEL: fct4:
-; Size <= 256, no change.
-; CHECK-DARWIN: {{b|bl}} _memset
-; CHECK-LINUX: {{b|bl}} memset
+; Size <= 256, inline memset.
+; CHECK: stp [[R0:q[0-9]+]], [[R0:q[0-9]+]], [x0, #224]
+; CHECK: stp [[R0:q[0-9]+]], [[R0:q[0-9]+]], [x0]
 define void @fct4(i8* %ptr) {
 entry:
   %tmp = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false)
   %call = tail call i8* @__memset_chk(i8* %ptr, i32 0, i64 256, i64 %tmp)
   ret void
 }
 
 declare i8* @__memset_chk(i8*, i32, i64, i64)

 
 ; CHECK-LABEL: fct7:
 ; memset with something that is not a zero, no change.
 ; CHECK-DARWIN: {{b|bl}} _memset
 ; CHECK-LINUX: {{b|bl}} memset
 define void @fct7(i8* %ptr) {
 entry:
   %tmp = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false)
-  %call = tail call i8* @__memset_chk(i8* %ptr, i32 1, i64 256, i64 %tmp)
+  %call = tail call i8* @__memset_chk(i8* %ptr, i32 1, i64 260, i64 %tmp)
   ret void
 }
 
 ; CHECK-LABEL: fct8:
 ; memset with something that is not a zero, no change.
 ; CHECK-DARWIN: {{b|bl}} _memset
 ; CHECK-LINUX: {{b|bl}} memset
 define void @fct8(i8* %ptr) {

llvm/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll

 ; RUN: llc -mtriple=arm64-apple-ios -mattr=+strict-align < %s | FileCheck %s
 
-; Small (16-bytes here) unaligned memcpys should stay memcpy calls if
+; Small (24-bytes here) unaligned memcpys should stay memcpy calls if
 ; strict-alignment is turned on.
 define void @t0(i8* %out, i8* %in) {
 ; CHECK-LABEL: t0:
-; CHECK:         orr w2, wzr, #0x10
+; CHECK:         orr w2, wzr, #0x18
 ; CHECK-NEXT:    bl _memcpy
 entry:
-  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 16, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 24, i1 false)
   ret void
 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1)

llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll

 define void @test_va_copy() {
 ; CHECK-LABEL: test_va_copy:
   %srcaddr = bitcast %va_list* @var to i8*
   %dstaddr = bitcast %va_list* @second_list to i8*
   call void @llvm.va_copy(i8* %dstaddr, i8* %srcaddr)
 
 ; CHECK: add x[[SRC:[0-9]+]], {{x[0-9]+}}, :lo12:var
 
-; CHECK: ldr [[BLOCK:q[0-9]+]], [x[[SRC]]]
+; CHECK: ldp [[BLOCK:q[0-9]+]], [[BLOCK:q[0-9]+]], [x[[SRC]]]
 ; CHECK: add x[[DST:[0-9]+]], {{x[0-9]+}}, :lo12:second_list
-; CHECK: str [[BLOCK]], [x[[DST]]]
-
-; CHECK: ldr [[BLOCK:q[0-9]+]], [x[[SRC]], #16]
-; CHECK: str [[BLOCK]], [x[[DST]], #16]
+; CHECK: stp [[BLOCK:q[0-9]+]], [[BLOCK:q[0-9]+]], [x[[DST]]]
   ret void
 ; CHECK: ret
 }

llvm/test/CodeGen/AArch64/arm64-virtual_base.ll

 %struct.istack_struct = type { %struct.istack_struct*, %struct.istk_entry*, i32 }
 %struct.istk_entry = type { double, [3 x double], [3 x double], %struct.Object_Struct*, i32, i32, double, double, i8* }
 %struct.Transform_Struct = type { [4 x [4 x double]], [4 x [4 x double]] }
 %struct.Bezier_Node_Struct = type { i32, [3 x double], double, i32, i8* }
 
 define void @Precompute_Patch_Values(%struct.Bicubic_Patch_Struct* %Shape) {
 ; CHECK: Precompute_Patch_Values
 ; CHECK: ldr [[VAL:x[0-9]+]], [x0, #288]
-; CHECK-NEXT: str [[VAL]], [sp, #232]
 ; CHECK-NEXT: ldr [[VAL2:q[0-9]+]], [x0, #272]
+; CHECK-NEXT: str [[VAL]], [sp, #232]
 ; CHECK-NEXT: stur [[VAL2]], {{\[}}sp, #216]
 entry:
   %Control_Points = alloca [16 x [3 x double]], align 8
   %arraydecay5.3.1 = getelementptr inbounds [16 x [3 x double]], [16 x [3 x double]]* %Control_Points, i64 0, i64 9, i64 0
   %tmp14 = bitcast double* %arraydecay5.3.1 to i8*
   %arraydecay11.3.1 = getelementptr inbounds %struct.Bicubic_Patch_Struct, %struct.Bicubic_Patch_Struct* %Shape, i64 0, i32 12, i64 1, i64 3, i64 0
   %tmp15 = bitcast double* %arraydecay11.3.1 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp14, i8* %tmp15, i64 24, i1 false)
   ret void
 }
 
 ; Function Attrs: nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1)

llvm/test/CodeGen/AArch64/mergestores_noimplicitfloat.ll

 ; RUN: llc -o - %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-ios10.0.0"
 
 ; PR33475 - Expect 64-bit operations as 128-operations are not legal
+; However, we can generate a paired 64-bit loads and stores, without using
+; floating point registers.
 
 ; CHECK-LABEL: pr33475
-; CHECK-DAG: ldr [[R0:x[0-9]+]], [x1]
-; CHECK-DAG: str [[R0]], [x0]
-; CHECK-DAG: ldr [[R1:x[0-9]+]], [x1, #8]
-; CHECK-DAG: str [[R1]], [x0, #8]
-; CHECK-DAG: ldr [[R2:x[0-9]+]], [x1, #16]
-; CHECK-DAG: str [[R2]], [x0, #16]
-; CHECK-DAG: ldr [[R3:x[0-9]+]], [x1, #24]
-; CHECK-DAG: str [[R3]], [x0, #24]
+; CHECK-DAG: ldp [[R0:x[0-9]+]], [[R0:x[0-9]+]], [x1, #16]
+; CHECK-DAG: ldp [[R0:x[0-9]+]], [[R0:x[0-9]+]], [x1]
+; CHECK-DAG: stp [[R0:x[0-9]+]], [[R0:x[0-9]+]], [x0, #16]
+; CHECK-DAG: stp [[R0:x[0-9]+]], [[R0:x[0-9]+]], [x0]
 
 define void @pr33475(i8* %p0, i8* %p1) noimplicitfloat {
     call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %p0, i8* align 4 %p1, i64 32, i1 false)
     ret void
 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1)

llvm/test/CodeGen/ARM/memcpy-inline.ll

 ; CHECK-T1: bl _memcpy
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([31 x i8], [31 x i8]* @.str1, i64 0, i64 0), i64 31, i1 false)
   ret void
 }
 
 define void @t2(i8* nocapture %C) nounwind {
 entry:
 ; CHECK-LABEL: t2:
-; CHECK: vld1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]!
+; CHECK: vld1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r2]!
+; CHECK: vld1.64 {d{{[0-9]+}}, d{{[0-9]+}}}, [r2]
+; CHECK: vst1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
 ; CHECK: movs [[INC:r[0-9]+]], #32
-; CHECK: add.w   r3, r0, #16
 ; CHECK: vst1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r0], [[INC]]
 ; CHECK: movw [[REG2:r[0-9]+]], #16716
 ; CHECK: movt [[REG2:r[0-9]+]], #72
 ; CHECK: str [[REG2]], [r0]
-; CHECK: vld1.64 {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
-; CHECK: vst1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r3]
 ; CHECK-T1-LABEL: t2:
 ; CHECK-T1: bl _memcpy
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([36 x i8], [36 x i8]* @.str2, i64 0, i64 0), i64 36, i1 false)
   ret void
 }
 
 define void @t3(i8* nocapture %C) nounwind {
 entry: