[AArch64] Implement passing SVE vectors by ref for AAPCS.

[sdesmalen]

Summary

This patch implements the part of the calling convention
where SVE Vectors are passed by reference. This means the
caller must allocate stack space for these objects and
pass the address to the callee.

[efriedma]

LGTM

[sdesmalen]

• Added two fixes to DAGCombine where we need to bail out if the type is scalable (the tests ran into some assertion failures otherwise).

[efriedma]

Please add a comment explaining each isScalableVector(), to make it clear why we're excluding them. Otherwise LGTM

llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll

10

I just thought of this, but should scalable arguments be allowed for calls using the regular C calling convetion? If they are, do we need any additional changes to make that work?

[sdesmalen]

In D71216#1817936, @efriedma wrote:

Please add a comment explaining each isScalableVector(), to make it clear why we're excluding them. Otherwise LGTM

Thanks, I'll add some comments for these cases (and do this for future cases I find as well).

llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll

10

The code that derives aarch64_sve_vector_pcs from the function signature (it takes and/or returns SVE vectors) already exists (in AArch64ISelLowering::LowerCall), so the aarch64_sve_vector_pcs is unnecessary here.

[sdesmalen]

• Added a few comments to the patch around isScalableVector().

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

bool DAGCombiner::SliceUpLoad(SDNode *N) {
   if (Level < AfterLegalizeDAG)
     return false;
 
   LoadSDNode *LD = cast<LoadSDNode>(N);
   if (!LD->isSimple() || !ISD::isNormalLoad(LD) ||
       !LD->getValueType(0).isInteger())
     return false;
 
+  // The algorithm to split up a load of a scalable vector into individual
+  // elements currently requires knowing the length of the loaded type,
+  // so will need adjusting to work on scalable vectors.
+  if (LD->getValueType(0).isScalableVector())
+    return false;
+
   // Keep track of already used bits to detect overlapping values.
   // In that case, we will just abort the transformation.
   APInt UsedBits(LD->getValueSizeInBits(0), 0);
 
   SmallVector<LoadedSlice, 4> LoadedSlices;
 
   // Check if this load is used as several smaller chunks of bits.
   // Basically, look for uses in trunc or trunc(lshr) and record a new chain
if (ST->isUnindexed() && ST->isSimple() &&
       if (ST1->getBasePtr() == Ptr && ST1->getValue() == Value &&
           ST->getMemoryVT() == ST1->getMemoryVT()) {
         // If this is a store followed by a store with the same value to the
         // same location, then the store is dead/noop.
         return Chain;
       }
 
       if (OptLevel != CodeGenOpt::None && ST1->hasOneUse() &&
-          !ST1->getBasePtr().isUndef()) {
+          !ST1->getBasePtr().isUndef() &&
+          // BaseIndexOffset and the code below requires knowing the size
+          // of a vector, so bail out if MemoryVT is scalable.
+          !ST1->getMemoryVT().isScalableVector()) {
         const BaseIndexOffset STBase = BaseIndexOffset::match(ST, DAG);
         const BaseIndexOffset ChainBase = BaseIndexOffset::match(ST1, DAG);
         unsigned STBitSize = ST->getMemoryVT().getSizeInBits();
         unsigned ChainBitSize = ST1->getMemoryVT().getSizeInBits();
         // If this is a store who's preceding store to a subset of the current
         // location and no one other node is chained to that store we can
         // effectively drop the store. Do not remove stores to undef as they may
         // be used as data sinks.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 //===----------------------------------------------------------------------===//
 
 /// Selects the correct CCAssignFn for a given CallingConvention value.
 CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
                                                      bool IsVarArg) const {
   switch (CC) {
   default:
     report_fatal_error("Unsupported calling convention.");
-  case CallingConv::AArch64_SVE_VectorCall:
-    // Calling SVE functions is currently not yet supported.
-    report_fatal_error("Unsupported calling convention.");
   case CallingConv::WebKit_JS:
     return CC_AArch64_WebKit_JS;
   case CallingConv::GHC:
     return CC_AArch64_GHC;
   case CallingConv::C:
   case CallingConv::Fast:
   case CallingConv::PreserveMost:
   case CallingConv::CXX_FAST_TLS:
   case CallingConv::Swift:
     if (Subtarget->isTargetWindows() && IsVarArg)
       return CC_AArch64_Win64_VarArg;
     if (!Subtarget->isTargetDarwin())
       return CC_AArch64_AAPCS;
     if (!IsVarArg)
       return CC_AArch64_DarwinPCS;
     return Subtarget->isTargetILP32() ? CC_AArch64_DarwinPCS_ILP32_VarArg
                                       : CC_AArch64_DarwinPCS_VarArg;
    case CallingConv::Win64:
     return IsVarArg ? CC_AArch64_Win64_VarArg : CC_AArch64_AAPCS;
    case CallingConv::CFGuard_Check:
      return CC_AArch64_Win64_CFGuard_Check;
    case CallingConv::AArch64_VectorCall:
+   case CallingConv::AArch64_SVE_VectorCall:
      return CC_AArch64_AAPCS;
   }
 }
 
 CCAssignFn *
 AArch64TargetLowering::CCAssignFnForReturn(CallingConv::ID CC) const {
   return CC == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS
                                       : RetCC_AArch64_AAPCS;
if (VA.isRegLoc()) {
       switch (VA.getLocInfo()) {
       default:
         llvm_unreachable("Unknown loc info!");
       case CCValAssign::Full:
         break;
       case CCValAssign::Indirect:
         assert(VA.getValVT().isScalableVector() &&
                "Only scalable vectors can be passed indirectly");
-        llvm_unreachable("Spilling of SVE vectors not yet implemented");
+        break;
       case CCValAssign::BCvt:
         ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue);
         break;
       case CCValAssign::AExt:
       case CCValAssign::SExt:
       case CCValAssign::ZExt:
         break;
       case CCValAssign::AExtUpper:
         ArgValue = DAG.getNode(ISD::SRL, DL, RegVT, ArgValue,
                                DAG.getConstant(32, DL, RegVT));
         ArgValue = DAG.getZExtOrTrunc(ArgValue, DL, VA.getValVT());
         break;
       }
     } else { // VA.isRegLoc()
       assert(VA.isMemLoc() && "CCValAssign is neither reg nor mem");
       unsigned ArgOffset = VA.getLocMemOffset();
-      unsigned ArgSize = VA.getValVT().getSizeInBits() / 8;
+      unsigned ArgSize = (VA.getLocInfo() == CCValAssign::Indirect
+                              ? VA.getLocVT().getSizeInBits()
+                              : VA.getValVT().getSizeInBits()) / 8;
 
       uint32_t BEAlign = 0;
       if (!Subtarget->isLittleEndian() && ArgSize < 8 &&
           !Ins[i].Flags.isInConsecutiveRegs())
         BEAlign = 8 - ArgSize;
 
       int FI = MFI.CreateFixedObject(ArgSize, ArgOffset + BEAlign, true);
 
if (VA.isRegLoc()) {
         break;
       case CCValAssign::Trunc:
       case CCValAssign::BCvt:
         MemVT = VA.getLocVT();
         break;
       case CCValAssign::Indirect:
         assert(VA.getValVT().isScalableVector() &&
                "Only scalable vectors can be passed indirectly");
-        llvm_unreachable("Spilling of SVE vectors not yet implemented");
+        MemVT = VA.getLocVT();
+        break;
       case CCValAssign::SExt:
         ExtType = ISD::SEXTLOAD;
         break;
       case CCValAssign::ZExt:
         ExtType = ISD::ZEXTLOAD;
         break;
       case CCValAssign::AExt:
         ExtType = ISD::EXTLOAD;
         break;
       }
 
       ArgValue = DAG.getExtLoad(
           ExtType, DL, VA.getLocVT(), Chain, FIN,
           MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
           MemVT);
 
     }
+
+    if (VA.getLocInfo() == CCValAssign::Indirect) {
+      assert(VA.getValVT().isScalableVector() &&
+           "Only scalable vectors can be passed indirectly");
+      // If value is passed via pointer - do a load.
+      ArgValue =
+          DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, MachinePointerInfo());
+    }
+
     if (Subtarget->isTargetILP32() && Ins[i].Flags.isPointer())
       ArgValue = DAG.getNode(ISD::AssertZext, DL, ArgValue.getValueType(),
                              ArgValue, DAG.getValueType(MVT::i32));
     InVals.push_back(ArgValue);
   }
 
   // varargs
   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
bool AArch64TargetLowering::isEligibleForTailCallOptimization(
 
   SmallVector<CCValAssign, 16> ArgLocs;
   CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
 
   CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
 
   const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
 
+  // If any of the arguments is passed indirectly, it must be SVE, so the
+  // 'getBytesInStackArgArea' is not sufficient to determine whether we need to
+  // allocate space on the stack. That is why we determine this explicitly here
+  // the call cannot be a tailcall.
+  if (llvm::any_of(ArgLocs, [](CCValAssign &A) {
+        assert((A.getLocInfo() != CCValAssign::Indirect ||
+                A.getValVT().isScalableVector()) &&
+               "Expected value to be scalable");
+        return A.getLocInfo() == CCValAssign::Indirect;
+      }))
+    return false;
+
   // If the stack arguments for this call do not fit into our own save area then
   // the call cannot be made tail.
   if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
     return false;
 
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
     return false;
case CCValAssign::Trunc:
       Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
       break;
     case CCValAssign::FPExt:
       Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
       break;
     case CCValAssign::Indirect:
       assert(VA.getValVT().isScalableVector() &&
              "Only scalable vectors can be passed indirectly");
-      llvm_unreachable("Spilling of SVE vectors not yet implemented");
+      MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
+      Type *Ty = EVT(VA.getValVT()).getTypeForEVT(*DAG.getContext());
+      unsigned Align = DAG.getDataLayout().getPrefTypeAlignment(Ty);
+      int FI = MFI.CreateStackObject(
+          VA.getValVT().getStoreSize().getKnownMinSize(), Align, false);
+      MFI.setStackID(FI, TargetStackID::SVEVector);
+
+      SDValue SpillSlot = DAG.getFrameIndex(
+          FI, DAG.getTargetLoweringInfo().getFrameIndexTy(DAG.getDataLayout()));
+      Chain = DAG.getStore(
+          Chain, DL, Arg, SpillSlot,
+          MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
+      Arg = SpillSlot;
+      break;
     }
 
     if (VA.isRegLoc()) {
       if (i == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
           Outs[0].VT == MVT::i64) {
         assert(VA.getLocVT() == MVT::i64 &&
                "unexpected calling convention register assignment");
         assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&
if (VA.isRegLoc()) {
       assert(VA.isMemLoc());
 
       SDValue DstAddr;
       MachinePointerInfo DstInfo;
 
       // FIXME: This works on big-endian for composite byvals, which are the
       // common case. It should also work for fundamental types too.
       uint32_t BEAlign = 0;
-      unsigned OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
-                                        : VA.getValVT().getSizeInBits();
+      unsigned OpSize;
+      if (VA.getLocInfo() == CCValAssign::Indirect)
+        OpSize = VA.getLocVT().getSizeInBits();
+      else
+        OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
       OpSize = (OpSize + 7) / 8;
       if (!Subtarget->isLittleEndian() && !Flags.isByVal() &&
           !Flags.isInConsecutiveRegs()) {
         if (OpSize < 8)
           BEAlign = 8 - OpSize;
       }
       unsigned LocMemOffset = VA.getLocMemOffset();
       int32_t Offset = LocMemOffset + BEAlign;

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

   defm : pred_store<nxv16i8, nxv16i1, non_temporal_store, STNT1B_ZRI>;
   defm : pred_store<nxv8i16, nxv8i1,  non_temporal_store, STNT1H_ZRI>;
   defm : pred_store<nxv4i32, nxv4i1,  non_temporal_store, STNT1W_ZRI>;
   defm : pred_store<nxv2i64, nxv2i1,  non_temporal_store, STNT1D_ZRI>;
 
   multiclass unpred_store<ValueType Ty, Instruction RegImmInst, Instruction PTrue> {
     def _fi : Pat<(store (Ty ZPR:$val), (am_sve_fi GPR64sp:$base, simm4s1:$offset)),
                        (RegImmInst ZPR:$val, (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
+
+    def _default : Pat<(store (Ty ZPR:$val), GPR64:$base),
+                       (RegImmInst ZPR:$val, (PTrue 31), GPR64:$base, (i64 0))>;
   }
 
   defm Pat_ST1B        : unpred_store<nxv16i8, ST1B_IMM, PTRUE_B>;
   defm Pat_ST1H        : unpred_store<nxv8i16, ST1H_IMM, PTRUE_H>;
   defm Pat_ST1W        : unpred_store<nxv4i32, ST1W_IMM, PTRUE_S>;
   defm Pat_ST1D        : unpred_store<nxv2i64, ST1D_IMM, PTRUE_D>;
   defm Pat_ST1H_float16: unpred_store<nxv8f16, ST1H_IMM, PTRUE_H>;
   defm Pat_ST1W_float  : unpred_store<nxv4f32, ST1W_IMM, PTRUE_S>;
   defm Pat_ST1D_double : unpred_store<nxv2f64, ST1D_IMM, PTRUE_D>;
 
   multiclass unpred_load<ValueType Ty, Instruction RegImmInst, Instruction PTrue> {
     def _fi : Pat<(Ty (load  (am_sve_fi GPR64sp:$base, simm4s1:$offset))),
                   (RegImmInst (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
+
+    def _default : Pat<(Ty (load GPR64:$base)),
+                       (RegImmInst (PTrue 31), GPR64:$base, (i64 0))>;
   }
 
   defm Pat_LD1B        : unpred_load<nxv16i8, LD1B_IMM, PTRUE_B>;
   defm Pat_LD1H        : unpred_load<nxv8i16, LD1H_IMM, PTRUE_H>;
   defm Pat_LD1W        : unpred_load<nxv4i32, LD1W_IMM, PTRUE_S>;
   defm Pat_LD1D        : unpred_load<nxv2i64, LD1D_IMM, PTRUE_D>;
   defm Pat_LD1H_float16: unpred_load<nxv8f16, LD1H_IMM, PTRUE_H>;
   defm Pat_LD1W_float  : unpred_load<nxv4f32, LD1W_IMM, PTRUE_S>;
   defm Pat_LD1D_double : unpred_load<nxv2f64, LD1D_IMM, PTRUE_D>;
 
   multiclass unpred_store_predicate<ValueType Ty, Instruction Store> {
     def _fi : Pat<(store (Ty PPR:$val), (am_sve_fi GPR64sp:$base, simm9:$offset)),
                   (Store PPR:$val, GPR64sp:$base, simm9:$offset)>;
+
+    def _default : Pat<(store (Ty PPR:$Val), GPR64:$base),
+                  (Store PPR:$Val, GPR64:$base, (i64 0))>;
   }
 
   defm Pat_Store_P16 : unpred_store_predicate<nxv16i1, STR_PXI>;
   defm Pat_Store_P8  : unpred_store_predicate<nxv8i1, STR_PXI>;
   defm Pat_Store_P4  : unpred_store_predicate<nxv4i1, STR_PXI>;
   defm Pat_Store_P2  : unpred_store_predicate<nxv2i1, STR_PXI>;
 
   multiclass unpred_load_predicate<ValueType Ty, Instruction Load> {
     def _fi : Pat<(Ty (load (am_sve_fi GPR64sp:$base, simm9:$offset))),
                   (Load GPR64sp:$base, simm9:$offset)>;
+
+    def _default : Pat<(Ty (load GPR64:$base)),
+                  (Load GPR64:$base, (i64 0))>;
   }
 
   defm Pat_Load_P16 : unpred_load_predicate<nxv16i1, LDR_PXI>;
   defm Pat_Load_P8  : unpred_load_predicate<nxv8i1, LDR_PXI>;
   defm Pat_Load_P4  : unpred_load_predicate<nxv4i1, LDR_PXI>;
   defm Pat_Load_P2  : unpred_load_predicate<nxv2i1, LDR_PXI>;
 
   multiclass ldnf1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT> {

llvm/test/CodeGen/AArch64/sve-callbyref-notailcall.ll

+; Because some arguments are passed by reference (through stack),
+; the compiler should not do tail-call optimization.
+; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s
+
+; CHECK-LABEL: caller:
+; CHECK:       addvl sp, sp, #-[[STACKSIZE:[0-9]+]]
+; CHECK-NOT:   addvl sp
+; CHECK:       bl callee
+; CHECK:       addvl sp, sp, #[[STACKSIZE]]
+; CHECK:       ret
+define <vscale x 16 x i8> @caller(<vscale x 16 x i8> %v) {
+  %1 = tail call <vscale x 16 x i8> @callee(<vscale x 16 x i8> %v, <vscale x 16 x i8> %v, <vscale x 16 x i8> %v, <vscale x 16 x i8> %v, <vscale x 16 x i8> %v, <vscale x 16 x i8> %v, <vscale x 16 x i8> %v, <vscale x 16 x i8> %v, <vscale x 16 x i8> %v)
+  ret <vscale x 16 x i8> %1
+}
+
+declare <vscale x 16 x i8> @callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+
+; CHECK-LABEL: caller_pred:
+; CHECK:       addvl sp, sp, #-[[STACKSIZE:[0-9]+]]
+; CHECK-NOT:   addvl sp
+; CHECK:       bl callee_pred
+; CHECK:       addvl sp, sp, #[[STACKSIZE]]
+; CHECK:       ret
+define <vscale x 16 x i1> @caller_pred(<vscale x 16 x i1> %v) {
+  %1 = tail call <vscale x 16 x i1> @callee_pred(<vscale x 16 x i1> %v, <vscale x 16 x i1> %v, <vscale x 16 x i1> %v, <vscale x 16 x i1> %v, <vscale x 16 x i1> %v)
+  ret <vscale x 16 x i1> %1
+}
+
+declare <vscale x 16 x i1> @callee_pred(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)

llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -stop-after=finalize-isel < %s | FileCheck %s
+
+; Test that z8 and z9, passed in by reference, are correctly loaded from x0 and x1.
+; i.e. z0 =  %z0
+;         :
+;      z7 =  %z7
+;      x0 = &%z8
+;      x1 = &%z9
+define aarch64_sve_vector_pcs <vscale x 4 x i32> @callee_with_many_sve_arg(<vscale x 4 x i32> %z0, <vscale x 4 x i32> %z1, <vscale x 4 x i32> %z2, <vscale x 4 x i32> %z3, <vscale x 4 x i32> %z4, <vscale x 4 x i32> %z5, <vscale x 4 x i32> %z6, <vscale x 4 x i32> %z7, <vscale x 4 x i32> %z8, <vscale x 4 x i32> %z9) {
+; CHECK: name: callee_with_many_sve_arg
+; CHECK-DAG: [[BASE:%[0-9]+]]:gpr64common = COPY $x1
+; CHECK-DAG: [[PTRUE:%[0-9]+]]:ppr_3b = PTRUE_S 31
+; CHECK-DAG: [[RES:%[0-9]+]]:zpr = LD1W_IMM killed [[PTRUE]], [[BASE]]
+; CHECK-DAG: $z0 = COPY [[RES]]
+; CHECK:     RET_ReallyLR implicit $z0
+  ret <vscale x 4 x i32> %z9
+}
+
+; Test that z8 and z9 are passed by reference.
+define aarch64_sve_vector_pcs <vscale x 4 x i32> @caller_with_many_sve_arg(<vscale x 4 x i32> %z) {
+; CHECK: name: caller_with_many_sve_arg
+; CHECK: stack:
+; CHECK:      - { id: 0, name: '', type: default, offset: 0, size: 16, alignment: 16,
+; CHECK-NEXT:     stack-id: sve-vec
+; CHECK:      - { id: 1, name: '', type: default, offset: 0, size: 16, alignment: 16,
+; CHECK-NEXT:     stack-id: sve-vec
+; CHECK-DAG:  [[PTRUE:%[0-9]+]]:ppr_3b = PTRUE_S 31
+; CHECK-DAG:  ST1W_IMM %{{[0-9]+}}, [[PTRUE]], %stack.1, 0
+; CHECK-DAG:  ST1W_IMM %{{[0-9]+}}, [[PTRUE]], %stack.0, 0
+; CHECK-DAG:  [[BASE2:%[0-9]+]]:gpr64sp = ADDXri %stack.1, 0
+; CHECK-DAG:  [[BASE1:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0
+; CHECK-DAG:  $x0 = COPY [[BASE1]]
+; CHECK-DAG:  $x1 = COPY [[BASE2]]
+; CHECK-NEXT: BL @callee_with_many_sve_arg
+; CHECK:      RET_ReallyLR implicit $z0
+  %ret = call aarch64_sve_vector_pcs <vscale x 4 x i32> @callee_with_many_sve_arg(<vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z)
+  ret <vscale x 4 x i32> %ret
+}
+
+; Test that p4 and p5, passed in by reference, are correctly loaded from register x0 and x1.
+; i.e. p0 =  %p0
+;         :
+;      p3 =  %p3
+;      x0 = &%p4
+;      x1 = &%p5
+define aarch64_sve_vector_pcs <vscale x 4 x i1> @callee_with_many_svepred_arg(<vscale x 4 x i1> %p0, <vscale x 4 x i1> %p1, <vscale x 4 x i1> %p2, <vscale x 4 x i1> %p3, <vscale x 4 x i1> %p4, <vscale x 4 x i1> %p5) {
+; CHECK: name: callee_with_many_svepred_arg
+; CHECK-DAG: [[BASE:%[0-9]+]]:gpr64common = COPY $x1
+; CHECK-DAG: [[RES:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 0
+; CHECK-DAG: $p0 = COPY [[RES]]
+; CHECK:     RET_ReallyLR implicit $p0
+  ret <vscale x 4 x i1> %p5
+}
+
+; Test that p4 and p5 are passed by reference.
+define aarch64_sve_vector_pcs <vscale x 4 x i1> @caller_with_many_svepred_arg(<vscale x 4 x i1> %p) {
+; CHECK: name: caller_with_many_svepred_arg
+; CHECK: stack:
+; CHECK:      - { id: 0, name: '', type: default, offset: 0, size: 1, alignment: 4,
+; CHECK-NEXT:     stack-id: sve-vec
+; CHECK:      - { id: 1, name: '', type: default, offset: 0, size: 1, alignment: 4,
+; CHECK-NEXT:     stack-id: sve-vec
+; CHECK-DAG: STR_PXI %{{[0-9]+}}, %stack.0, 0
+; CHECK-DAG: STR_PXI %{{[0-9]+}}, %stack.1, 0
+; CHECK-DAG: [[BASE1:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0
+; CHECK-DAG: [[BASE2:%[0-9]+]]:gpr64sp = ADDXri %stack.1, 0
+; CHECK-DAG: $x0 = COPY [[BASE1]]
+; CHECK-DAG: $x1 = COPY [[BASE2]]
+; CHECK-NEXT: BL @callee_with_many_svepred_arg
+; CHECK:     RET_ReallyLR implicit $p0
+  %ret = call aarch64_sve_vector_pcs <vscale x 4 x i1> @callee_with_many_svepred_arg(<vscale x 4 x i1> %p, <vscale x 4 x i1> %p, <vscale x 4 x i1> %p, <vscale x 4 x i1> %p, <vscale x 4 x i1> %p, <vscale x 4 x i1> %p)
+  ret <vscale x 4 x i1> %ret
+}
+
+; Test that z8 and z9, passed by reference, are loaded from a location that is passed on the stack.
+; i.e.     x0 =   %x0
+;             :
+;          x7 =   %x7
+;          z0 =   %z0
+;             :
+;          z7 =   %z7
+;        [sp] =  &%z8
+;      [sp+8] =  &%z9
+;
+define aarch64_sve_vector_pcs <vscale x 4 x i32> @callee_with_many_gpr_sve_arg(i64 %x0, i64 %x1, i64 %x2, i64 %x3, i64 %x4, i64 %x5, i64 %x6, i64 %x7, <vscale x 4 x i32> %z0, <vscale x 4 x i32> %z1, <vscale x 4 x i32> %z2, <vscale x 4 x i32> %z3, <vscale x 4 x i32> %z4, <vscale x 4 x i32> %z5, <vscale x 4 x i32> %z6, <vscale x 4 x i32> %z7, <vscale x 2 x i64> %z8, <vscale x 4 x i32> %z9) {
+; CHECK: name: callee_with_many_gpr_sve_arg
+; CHECK: fixedStack:
+; CHECK:      - { id: 0, type: default, offset: 8, size: 8, alignment: 8, stack-id: default,
+; CHECK-DAG: [[BASE:%[0-9]+]]:gpr64common = LDRXui %fixed-stack.0, 0
+; CHECK-DAG: [[PTRUE:%[0-9]+]]:ppr_3b = PTRUE_S 31
+; CHECK-DAG: [[RES:%[0-9]+]]:zpr = LD1W_IMM killed [[PTRUE]], killed [[BASE]]
+; CHECK-DAG: $z0 = COPY [[RES]]
+; CHECK: RET_ReallyLR implicit $z0
+  ret <vscale x 4 x i32> %z9
+}
+
+; Test that z8 and z9 are passed by reference, where reference is passed on the stack.
+define aarch64_sve_vector_pcs <vscale x 4 x i32> @caller_with_many_gpr_sve_arg(i64 %x, <vscale x 4 x i32> %z, <vscale x 2 x i64> %z2) {
+; CHECK: name: caller_with_many_gpr_sve_arg
+; CHECK: stack:
+; CHECK:      - { id: 0, name: '', type: default, offset: 0, size: 16, alignment: 16,
+; CHECK-NEXT:     stack-id: sve-vec
+; CHECK:      - { id: 1, name: '', type: default, offset: 0, size: 16, alignment: 16,
+; CHECK-NEXT:     stack-id: sve-vec
+; CHECK-DAG: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31
+; CHECK-DAG: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31
+; CHECK-DAG: ST1D_IMM %{{[0-9]+}}, killed [[PTRUE_D]], %stack.0, 0
+; CHECK-DAG: ST1W_IMM %{{[0-9]+}}, killed [[PTRUE_S]], %stack.1, 0
+; CHECK-DAG: [[BASE1:%[0-9]+]]:gpr64common = ADDXri %stack.0, 0
+; CHECK-DAG: [[BASE2:%[0-9]+]]:gpr64common = ADDXri %stack.1, 0
+; CHECK-DAG: [[SP:%[0-9]+]]:gpr64sp = COPY $sp
+; CHECK-DAG: STRXui killed [[BASE1]], [[SP]], 0
+; CHECK-DAG: STRXui killed [[BASE2]], [[SP]], 1
+; CHECK:     BL @callee_with_many_gpr_sve_arg
+; CHECK:     RET_ReallyLR implicit $z0
+  %ret = call aarch64_sve_vector_pcs <vscale x 4 x i32> @callee_with_many_gpr_sve_arg(i64 %x, i64 %x, i64 %x, i64 %x, i64 %x, i64 %x, i64 %x, i64 %x, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 4 x i32> %z, <vscale x 2 x i64> %z2, <vscale x 4 x i32> %z)
+  ret <vscale x 4 x i32> %ret
+}