[AArch64][SVE] Fix the indexed addressing mode when FI = 0.

[sdesmalen]

Summary

This is an alternative fix to D145497, which also addresses

https://github.com/llvm/llvm-project/issues/60918

In D124457 which added the original code for this, @efriedma pointed
out that it wasn't safe to assume that FI #0 would be allocated at offset
0, but that part of the patch went in without any changes.

The downside of this solution is that any access to an object on the
stack that has been allocated at SP + 0, still gets moved to a separate
register first, which degrades performance.

[paulwalker-arm]

We knew when landing the original patch that it'll likely bite us, but at that time there just wasn't a good enough reason to warrant the loss in code quality.

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
   const EVT MemVT = getMemVTFromNode(*(CurDAG->getContext()), Root);
   const DataLayout &DL = CurDAG->getDataLayout();
   const MachineFrameInfo &MFI = MF->getFrameInfo();
 
   if (N.getOpcode() == ISD::FrameIndex) {
     int FI = cast<FrameIndexSDNode>(N)->getIndex();
     // We can only encode VL scaled offsets, so only fold in frame indexes
     // referencing SVE objects.
-    if (FI == 0 || MFI.getStackID(FI) == TargetStackID::ScalableVector) {
+    if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
       Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
       OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);
       return true;
     }
 
     return false;
   }
 
bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
   if (Offset < Min || Offset > Max)
     return false;
 
   Base = N.getOperand(0);
   if (Base.getOpcode() == ISD::FrameIndex) {
     int FI = cast<FrameIndexSDNode>(Base)->getIndex();
     // We can only encode VL scaled offsets, so only fold in frame indexes
     // referencing SVE objects.
-    if (FI == 0 || MFI.getStackID(FI) == TargetStackID::ScalableVector)
+    if (MFI.getStackID(FI) == TargetStackID::ScalableVector)
       Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
   }
 
   OffImm = CurDAG->getTargetConstant(Offset, SDLoc(N), MVT::i64);
   return true;
 }
 
 /// Select register plus register addressing mode for SVE, with scaled

llvm/test/CodeGen/AArch64/sve-fixed-ld2-alloca.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
 declare void @def(ptr)
 
 define void @st1d_fixed(ptr %ptr) #0 {
 ; CHECK-LABEL: st1d_fixed:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #144
-; CHECK-NEXT:    stp x30, x19, [sp, #128] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #160
+; CHECK-NEXT:    stp x20, x19, [sp, #144] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    str x30, [sp, #128] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x20, sp
 ; CHECK-NEXT:    bl def
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    ld2d { z0.d, z1.d }, p0/z, [sp]
+; CHECK-NEXT:    ld2d { z0.d, z1.d }, p0/z, [x20]
+; CHECK-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x19]
-; CHECK-NEXT:    ldp x30, x19, [sp, #128] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #144
+; CHECK-NEXT:    ldp x20, x19, [sp, #144] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #160
 ; CHECK-NEXT:    ret
   %alloc = alloca [16 x double]
   call void @def(ptr %alloc)
   %load = load <8 x double>, ptr %alloc
   %strided.vec = shufflevector <8 x double> %load, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   store <4 x double> %strided.vec, ptr %ptr
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" vscale_range(2,2) nounwind }

llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -debug-only=isel < %s 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
 target triple = "aarch64-unknown-linux-gnu"
 
 ; Ensure that only no offset frame indexes are folded into SVE load/stores when
 ; accessing fixed width objects.
 define void @foo(ptr %a) #0 {
 ; CHECK-LABEL: foo:
-; CHECK:       SelectionDAG has 14 nodes:
+; CHECK:       SelectionDAG has 15 nodes:
 ; CHECK-NEXT:    t0: ch,glue = EntryToken
 ; CHECK-NEXT:    t12: nxv2i1 = PTRUE_D TargetConstant:i32<31>
 ; CHECK-NEXT:    t2: i64,ch = CopyFromReg t0, Register:i64 %0
 ; CHECK-NEXT:    t18: nxv2i64,ch = LD1D_IMM<Mem:(volatile load (s512) from %ir.a)> t12, t2, TargetConstant:i64<0>, t0
 ; CHECK-NEXT:    t8: i64 = ADDXri TargetFrameIndex:i64<1>, TargetConstant:i32<0>, TargetConstant:i32<0>
-; CHECK-NEXT:    t17: ch = ST1D_IMM<Mem:(volatile store (s512) into %ir.r0)> t18, t12, TargetFrameIndex:i64<0>, TargetConstant:i64<0>, t18:1
+; CHECK-NEXT:    t6: i64 = ADDXri TargetFrameIndex:i64<0>, TargetConstant:i32<0>, TargetConstant:i32<0>
+; CHECK-NEXT:    t17: ch = ST1D_IMM<Mem:(volatile store (s512) into %ir.r0)> t18, t12, t6, TargetConstant:i64<0>, t18:1
 ; CHECK-NEXT:    t16: ch = ST1D_IMM<Mem:(volatile store (s512) into %ir.r1)> t18, t12, t8, TargetConstant:i64<0>, t17
 ; CHECK-NEXT:    t10: ch = RET_ReallyLR t16
 ; CHECK-EMPTY:
 entry:
   %r0 = alloca <8 x i64>
   %r1 = alloca <8 x i64>
   %r = load volatile <8 x i64>, ptr %a
   store volatile <8 x i64> %r, ptr %r0
   store volatile <8 x i64> %r, ptr %r1
   ret void
 }
 
 attributes #0 = { nounwind "target-features"="+sve" vscale_range(4,4) }

llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll

 ; CHECK-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-NEXT:    .cfi_offset w30, -8
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    sub x9, sp, #48
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    mov z1.d, z0.d[2]
-; CHECK-NEXT:    mov z2.d, z0.d[3]
-; CHECK-NEXT:    mov x10, v0.d[1]
-; CHECK-NEXT:    fmov x8, d1
-; CHECK-NEXT:    fmov x9, d2
 ; CHECK-NEXT:    fmov x11, d0
+; CHECK-NEXT:    fmov x8, d1
+; CHECK-NEXT:    mov z1.d, z0.d[3]
+; CHECK-NEXT:    fmov x9, d1
+; CHECK-NEXT:    mov x10, v0.d[1]
 ; CHECK-NEXT:    stp x9, x8, [sp, #16]
+; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    stp x10, x11, [sp]
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %tmp1 = load <4 x i64>, ptr %a
   %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   store <4 x i64> %tmp2, ptr %a
   ret void

 ; CHECK-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-NEXT:    .cfi_offset w30, -8
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    sub x9, sp, #48
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK-NEXT:    ptrue p0.s, vl8
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w10, s0
 ; CHECK-NEXT:    mov w9, v0.s[2]
 ; CHECK-NEXT:    mov w11, v0.s[3]
-; CHECK-NEXT:    fmov w10, s0
 ; CHECK-NEXT:    mov z1.s, z0.s[4]
 ; CHECK-NEXT:    mov z2.s, z0.s[5]
 ; CHECK-NEXT:    mov z3.s, z0.s[6]
 ; CHECK-NEXT:    mov z0.s, z0.s[7]
 ; CHECK-NEXT:    stp w8, w10, [sp, #24]
 ; CHECK-NEXT:    fmov w10, s1
 ; CHECK-NEXT:    fmov w8, s2
 ; CHECK-NEXT:    stp w11, w9, [sp, #16]
 ; CHECK-NEXT:    fmov w9, s3
 ; CHECK-NEXT:    fmov w11, s0
 ; CHECK-NEXT:    stp w8, w10, [sp, #8]
+; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    stp w11, w9, [sp]
-; CHECK-NEXT:    ld1w { z0.s }, p0/z, [sp]
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %tmp1 = load <8 x i32>, ptr %a
   %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   store <8 x i32> %tmp2, ptr %a
   ret void

 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    sub x9, sp, #48
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    mov z1.h, z0.h[8]
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    mov z4.h, z0.h[11]
 ; CHECK-NEXT:    mov z5.h, z0.h[12]
 ; CHECK-NEXT:    mov z2.h, z0.h[9]
+; CHECK-NEXT:    strh w8, [sp, #14]
+; CHECK-NEXT:    fmov w8, s4
 ; CHECK-NEXT:    mov z3.h, z0.h[10]
-; CHECK-NEXT:    mov z4.h, z0.h[11]
-; CHECK-NEXT:    fmov w11, s2
 ; CHECK-NEXT:    strh w9, [sp, #30]
 ; CHECK-NEXT:    fmov w9, s5
+; CHECK-NEXT:    mov z16.h, z0.h[15]
+; CHECK-NEXT:    fmov w11, s2
 ; CHECK-NEXT:    fmov w12, s3
-; CHECK-NEXT:    strh w8, [sp, #14]
-; CHECK-NEXT:    fmov w8, s4
+; CHECK-NEXT:    strh w8, [sp, #24]
+; CHECK-NEXT:    fmov w8, s16
 ; CHECK-NEXT:    mov z6.h, z0.h[13]
 ; CHECK-NEXT:    mov z7.h, z0.h[14]
-; CHECK-NEXT:    mov z16.h, z0.h[15]
 ; CHECK-NEXT:    umov w10, v0.h[1]
 ; CHECK-NEXT:    strh w9, [sp, #22]
 ; CHECK-NEXT:    umov w9, v0.h[2]
 ; CHECK-NEXT:    strh w11, [sp, #28]
 ; CHECK-NEXT:    fmov w11, s6
 ; CHECK-NEXT:    strh w12, [sp, #26]
 ; CHECK-NEXT:    fmov w12, s7
-; CHECK-NEXT:    strh w8, [sp, #24]
-; CHECK-NEXT:    fmov w8, s16
+; CHECK-NEXT:    strh w8, [sp, #16]
+; CHECK-NEXT:    umov w8, v0.h[5]
 ; CHECK-NEXT:    strh w10, [sp, #12]
 ; CHECK-NEXT:    strh w11, [sp, #20]
 ; CHECK-NEXT:    umov w11, v0.h[3]
 ; CHECK-NEXT:    strh w12, [sp, #18]
 ; CHECK-NEXT:    umov w12, v0.h[4]
-; CHECK-NEXT:    strh w8, [sp, #16]
-; CHECK-NEXT:    umov w8, v0.h[5]
 ; CHECK-NEXT:    umov w10, v0.h[6]
 ; CHECK-NEXT:    strh w9, [sp, #10]
 ; CHECK-NEXT:    umov w9, v0.h[7]
+; CHECK-NEXT:    strh w8, [sp, #4]
+; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    strh w11, [sp, #8]
 ; CHECK-NEXT:    strh w12, [sp, #6]
-; CHECK-NEXT:    strh w8, [sp, #4]
 ; CHECK-NEXT:    strh w10, [sp, #2]
 ; CHECK-NEXT:    strh w9, [sp]
-; CHECK-NEXT:    ld1h { z0.h }, p0/z, [sp]
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %tmp1 = load <16 x i16>, ptr %a
   %tmp2 = shufflevector <16 x i16> %tmp1, <16 x i16> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
   store <16 x i16> %tmp2, ptr %a
   ret void

 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    sub x9, sp, #48
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ldr q0, [x1]
 ; CHECK-NEXT:    orr x9, x8, #0x1e
 ; CHECK-NEXT:    orr x10, x8, #0x1c
 ; CHECK-NEXT:    ldr q1, [x0]
-; CHECK-NEXT:    orr x12, x8, #0x10
 ; CHECK-NEXT:    orr x11, x8, #0x18
+; CHECK-NEXT:    orr x12, x8, #0x10
 ; CHECK-NEXT:    str h0, [sp, #22]
 ; CHECK-NEXT:    st1 { v0.h }[4], [x9]
 ; CHECK-NEXT:    orr x9, x8, #0xe
 ; CHECK-NEXT:    st1 { v0.h }[5], [x10]
 ; CHECK-NEXT:    orr x10, x8, #0xc
-; CHECK-NEXT:    st1 { v0.h }[3], [x12]
-; CHECK-NEXT:    mov w12, #26
-; CHECK-NEXT:    st1 { v1.h }[4], [x9]
-; CHECK-NEXT:    orr x9, x8, #0x8
 ; CHECK-NEXT:    st1 { v0.h }[7], [x11]
-; CHECK-NEXT:    orr x11, x8, #0x2
+; CHECK-NEXT:    orr x11, x8, #0x8
+; CHECK-NEXT:    st1 { v1.h }[4], [x9]
+; CHECK-NEXT:    orr x9, x8, #0x4
 ; CHECK-NEXT:    st1 { v1.h }[5], [x10]
-; CHECK-NEXT:    orr x10, x8, #0x4
-; CHECK-NEXT:    st1 { v1.h }[7], [x9]
+; CHECK-NEXT:    mov w10, #26
+; CHECK-NEXT:    orr x10, x8, x10
+; CHECK-NEXT:    st1 { v0.h }[3], [x12]
+; CHECK-NEXT:    st1 { v1.h }[1], [x9]
+; CHECK-NEXT:    orr x9, x8, #0x2
+; CHECK-NEXT:    st1 { v1.h }[7], [x11]
+; CHECK-NEXT:    mov w11, #20
+; CHECK-NEXT:    mov w12, #18
+; CHECK-NEXT:    st1 { v0.h }[6], [x10]
+; CHECK-NEXT:    mov w10, #10
+; CHECK-NEXT:    orr x11, x8, x11
+; CHECK-NEXT:    st1 { v1.h }[2], [x9]
 ; CHECK-NEXT:    orr x9, x8, x12
-; CHECK-NEXT:    st1 { v1.h }[2], [x11]
-; CHECK-NEXT:    mov w11, #10
-; CHECK-NEXT:    st1 { v1.h }[1], [x10]
-; CHECK-NEXT:    mov w10, #18
-; CHECK-NEXT:    st1 { v0.h }[6], [x9]
-; CHECK-NEXT:    mov w9, #20
-; CHECK-NEXT:    orr x9, x8, x9
 ; CHECK-NEXT:    orr x10, x8, x10
 ; CHECK-NEXT:    st1 { v1.h }[3], [x8]
-; CHECK-NEXT:    orr x8, x8, x11
-; CHECK-NEXT:    str h1, [sp, #6]
+; CHECK-NEXT:    st1 { v0.h }[1], [x11]
 ; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    st1 { v0.h }[1], [x9]
-; CHECK-NEXT:    st1 { v0.h }[2], [x10]
-; CHECK-NEXT:    st1 { v1.h }[6], [x8]
-; CHECK-NEXT:    ld1h { z0.h }, p0/z, [sp]
+; CHECK-NEXT:    st1 { v0.h }[2], [x9]
+; CHECK-NEXT:    st1 { v1.h }[6], [x10]
+; CHECK-NEXT:    str h1, [sp, #6]
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %tmp1 = load <8 x i16>, ptr %a
   %tmp2 = load <8 x i16>, ptr %b
   %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   store <16 x i16> %tmp3, ptr %c
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }
 attributes #1 = { "target-features"="+sve" vscale_range(2,2) }
 attributes #2 = { "target-features"="+sve2p1" }

llvm/test/CodeGen/AArch64/sve-fixed-length-permute-zip-uzp-trn.ll

 ; VBITS_EQ_512-NEXT:    .cfi_def_cfa_offset 16
 ; VBITS_EQ_512-NEXT:    mov x29, sp
 ; VBITS_EQ_512-NEXT:    .cfi_def_cfa w29, 16
 ; VBITS_EQ_512-NEXT:    .cfi_offset w30, -8
 ; VBITS_EQ_512-NEXT:    .cfi_offset w29, -16
 ; VBITS_EQ_512-NEXT:    sub x9, sp, #48
 ; VBITS_EQ_512-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; VBITS_EQ_512-NEXT:    ptrue p0.d, vl4
+; VBITS_EQ_512-NEXT:    mov x8, sp
 ; VBITS_EQ_512-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; VBITS_EQ_512-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; VBITS_EQ_512-NEXT:    mov z2.d, z1.d[3]
 ; VBITS_EQ_512-NEXT:    mov z3.d, z0.d[3]
 ; VBITS_EQ_512-NEXT:    stp d3, d2, [sp, #16]
 ; VBITS_EQ_512-NEXT:    mov z2.d, z1.d[2]
 ; VBITS_EQ_512-NEXT:    mov z3.d, z0.d[2]
 ; VBITS_EQ_512-NEXT:    zip1 z0.d, z0.d, z1.d
 ; VBITS_EQ_512-NEXT:    stp d3, d2, [sp]
-; VBITS_EQ_512-NEXT:    ld1d { z2.d }, p0/z, [sp]
+; VBITS_EQ_512-NEXT:    ld1d { z2.d }, p0/z, [x8]
 ; VBITS_EQ_512-NEXT:    fadd z0.d, p0/m, z0.d, z2.d
 ; VBITS_EQ_512-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_EQ_512-NEXT:    mov sp, x29
 ; VBITS_EQ_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; VBITS_EQ_512-NEXT:    ret
   %tmp1 = load <4 x double>, ptr %a
   %tmp2 = load <4 x double>, ptr %b
   %tmp3 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    mov x29, sp
 ; CHECK-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-NEXT:    .cfi_offset w30, -8
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    sub x9, sp, #48
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    mov z2.d, z1.d[3]
 ; CHECK-NEXT:    mov z3.d, z0.d[3]
 ; CHECK-NEXT:    stp d3, d2, [sp, #16]
 ; CHECK-NEXT:    mov z2.d, z1.d[2]
 ; CHECK-NEXT:    mov z3.d, z0.d[2]
 ; CHECK-NEXT:    zip1 z0.d, z0.d, z1.d
 ; CHECK-NEXT:    stp d3, d2, [sp]
-; CHECK-NEXT:    ld1d { z2.d }, p0/z, [sp]
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x8]
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, z2.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %tmp1 = load <4 x double>, ptr %a
   %tmp2 = load <4 x double>, ptr %b
   %tmp3 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>

llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle.ll

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    mov x29, sp
 ; CHECK-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-NEXT:    .cfi_offset w30, -8
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    sub x9, sp, #48
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    mov z2.d, z1.d[1]
 ; CHECK-NEXT:    stp d1, d2, [sp, #16]
 ; CHECK-NEXT:    mov z1.d, z0.d[3]
 ; CHECK-NEXT:    mov z0.d, z0.d[2]
 ; CHECK-NEXT:    stp d0, d1, [sp]
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %ret = shufflevector <4 x double> %op1, <4 x double> %op2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   store <4 x double> %ret, ptr %a
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
 declare void @def(ptr)
 
 define void @alloc_v4i8(ptr %st_ptr) #0 {
 ; CHECK-LABEL: alloc_v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #32
-; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    add x20, sp, #28
 ; CHECK-NEXT:    bl def
-; CHECK-NEXT:    add x8, sp, #12
 ; CHECK-NEXT:    ptrue p0.b, vl2
-; CHECK-NEXT:    ld2b { z0.b, z1.b }, p0/z, [x8]
+; CHECK-NEXT:    ld2b { z0.b, z1.b }, p0/z, [x20]
 ; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov z2.b, z0.b[1]
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    fmov w9, s2
-; CHECK-NEXT:    stp w8, w9, [sp]
-; CHECK-NEXT:    ldr d0, [sp]
+; CHECK-NEXT:    stp w8, w9, [sp, #8]
+; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    st1b { z0.s }, p0, [x19]
-; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
   %alloc = alloca [4 x i8]
   call void @def(ptr %alloc)
   %load = load <4 x i8>, ptr %alloc
   %strided.vec = shufflevector <4 x i8> %load, <4 x i8> poison, <2 x i32> <i32 0, i32 2>
   store <2 x i8> %strided.vec, ptr %st_ptr
   ret void
 }
 
 define void @alloc_v6i8(ptr %st_ptr) #0 {
 ; CHECK-LABEL: alloc_v6i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    add x20, sp, #24
 ; CHECK-NEXT:    bl def
-; CHECK-NEXT:    add x8, sp, #24
 ; CHECK-NEXT:    ptrue p0.b, vl3
-; CHECK-NEXT:    ld2b { z0.b, z1.b }, p0/z, [x8]
+; CHECK-NEXT:    ld2b { z0.b, z1.b }, p0/z, [x20]
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    fmov w8, s1
 ; CHECK-NEXT:    mov z2.b, z1.b[3]
 ; CHECK-NEXT:    mov z3.b, z1.b[2]
 ; CHECK-NEXT:    mov z0.b, z1.b[1]
 ; CHECK-NEXT:    fmov w9, s2
 ; CHECK-NEXT:    fmov w10, s3
-; CHECK-NEXT:    strh w8, [sp, #8]
+; CHECK-NEXT:    strh w8, [sp]
 ; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    strh w9, [sp, #14]
-; CHECK-NEXT:    strh w10, [sp, #12]
-; CHECK-NEXT:    strh w8, [sp, #10]
-; CHECK-NEXT:    add x8, sp, #20
-; CHECK-NEXT:    ldr d0, [sp, #8]
+; CHECK-NEXT:    strh w9, [sp, #6]
+; CHECK-NEXT:    strh w10, [sp, #4]
+; CHECK-NEXT:    strh w8, [sp, #2]
+; CHECK-NEXT:    add x8, sp, #12
+; CHECK-NEXT:    ldr d0, [sp]
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x8]
-; CHECK-NEXT:    ldrh w8, [sp, #20]
+; CHECK-NEXT:    ldrh w8, [sp, #12]
 ; CHECK-NEXT:    strb w10, [x19, #2]
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    strh w8, [x19]
-; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
   %alloc = alloca [6 x i8]
   call void @def(ptr %alloc)
   %load = load <6 x i8>, ptr %alloc
   %strided.vec = shufflevector <6 x i8> %load, <6 x i8> poison, <3 x i32> <i32 1, i32 3, i32 5>
   store <3 x i8> %strided.vec, ptr %st_ptr
   ret void

 ; CHECK-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    mov x20, sp
 ; CHECK-NEXT:    bl def
 ; CHECK-NEXT:    mov x8, #4
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    ld2d { z0.d, z1.d }, p0/z, [sp]
+; CHECK-NEXT:    ld2d { z0.d, z1.d }, p0/z, [x20]
 ; CHECK-NEXT:    ld2d { z2.d, z3.d }, p0/z, [x20, x8, lsl #3]
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    stp q0, q2, [x19]
 ; CHECK-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %alloc = alloca [8 x double]
   call void @def(ptr %alloc)
   %load = load <8 x double>, ptr %alloc
   %strided.vec = shufflevector <8 x double> %load, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   store <4 x double> %strided.vec, ptr %st_ptr
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" nounwind}