[RISCV] Spilling for RISC-V V extension.

[frasercrmck]

llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp

97

Should hasFP not pessimistically return true if V is enabled?

[HsiangKai]

llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp

97

Do you mean to move Subtarget.hasStdExtV() into hasFP()? To consider spilling, it seems reasonable.

[frasercrmck]

llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp

97

Yeah, that's what I meant. I believe I had a thought a few hours after that comment why you might not want hasStdExtV in hasFP, but I've forgotten it now.

[craig.topper]

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

851

Use if/else here. The conditional operator with no assignment looks kind of strange.

858

Same here

llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp

97

I don't see an equivalent of this in AArch64 when SVE is enabled. How are the avoiding needing a frame pointer?

[HsiangKai]

llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp

97

Indeed, it is possible to access frame objects without reserving frame pointer when there is any RVV objects on the stack. If we do not leverage frame pointer when there is any RVV objects, all frame object accesses will need do some polynomial calculation.

Now, there is a new type to represent two dimensions stack offset. I could try to implement it without reserving the frame pointer.

[lenary]

llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp

97

Being deep within hasFP code right now, and seeing that it has some nasty edge-cases when introducing ilp32e, it seems prudent to start with a conservatively correct approach for the initial implementation, and then optimise to avoid using frame pointers if necessary?

That should also make reviewer's lives easier :)

[HsiangKai]

Review D95148 instead of this one.

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/EHPersonalities.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/GCMetadata.h"
 #include "llvm/CodeGen/GCMetadataPrinter.h"
 #include "llvm/CodeGen/GCStrategy.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
static void emitComments(const MachineInstr &MI, raw_ostream &CommentOS) {
 
   // We assume a single instruction only has a spill or reload, not
   // both.
   Optional<unsigned> Size;
   if ((Size = MI.getRestoreSize(TII))) {
     CommentOS << *Size << "-byte Reload\n";
   } else if ((Size = MI.getFoldedRestoreSize(TII))) {
     if (*Size)
+      (*Size == static_cast<unsigned>(MemoryLocation::UnknownSize)) ?
+        CommentOS << "Unknown size Folded Reload\n" :
         CommentOS << *Size << "-byte Folded Reload\n";
   } else if ((Size = MI.getSpillSize(TII))) {
     CommentOS << *Size << "-byte Spill\n";
   } else if ((Size = MI.getFoldedSpillSize(TII))) {
     if (*Size)
+      (*Size == static_cast<unsigned>(MemoryLocation::UnknownSize)) ?
+        CommentOS << "Unknown size Folded Spill\n" :
         CommentOS << *Size << "-byte Folded Spill\n";
   }
 
   // Check for spill-induced copies
   if (MI.getAsmPrinterFlag(MachineInstr::ReloadReuse))
     CommentOS << " Reload Reuse\n";
 }
 
 /// emitImplicitDef - This method emits the specified machine instruction

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

 //===-- RISCVInstrInfo.cpp - RISCV Instruction Information ------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains the RISCV implementation of the TargetInstrInfo class.
 //
 //===----------------------------------------------------------------------===//
 
 #include "RISCVInstrInfo.h"
 #include "RISCV.h"
+#include "RISCVMachineFunctionInfo.h"
 #include "RISCVSubtarget.h"
 #include "RISCVTargetMachine.h"
 #include "Utils/RISCVMatInt.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                          Register SrcReg, bool IsKill, int FI,
                                          const TargetRegisterClass *RC,
                                          const TargetRegisterInfo *TRI) const {
   DebugLoc DL;
   if (I != MBB.end())
     DL = I->getDebugLoc();
 
   MachineFunction *MF = MBB.getParent();
-  const MachineFrameInfo &MFI = MF->getFrameInfo();
-  MachineMemOperand *MMO = MF->getMachineMemOperand(
-      MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
-      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
+  MachineFrameInfo &MFI = MF->getFrameInfo();
 
   unsigned Opcode;
+  bool IsScalableVector = false;
   if (RISCV::GPRRegClass.hasSubClassEq(RC))
     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
              RISCV::SW : RISCV::SD;
   else if (RISCV::FPR16RegClass.hasSubClassEq(RC))
     Opcode = RISCV::FSH;
   else if (RISCV::FPR32RegClass.hasSubClassEq(RC))
     Opcode = RISCV::FSW;
   else if (RISCV::FPR64RegClass.hasSubClassEq(RC))
     Opcode = RISCV::FSD;
-  else
+  else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::PseudoVSPILL_M1;
+    IsScalableVector = true;
+  } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::PseudoVSPILL_M2;
+    IsScalableVector = true;
+  } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::PseudoVSPILL_M4;
+    IsScalableVector = true;
+  } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::PseudoVSPILL_M8;
+    IsScalableVector = true;
+  } else
     llvm_unreachable("Can't store this register to stack slot");
 
+  if (IsScalableVector) {
+    MachineMemOperand *MMO = MF->getMachineMemOperand(
+        MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
+        MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
+
+    MFI.setStackID(FI, TargetStackID::RISCVVector);
+    BuildMI(MBB, I, DL, get(Opcode))
+        .addReg(SrcReg, getKillRegState(IsKill))
+        .addFrameIndex(FI)
+        .addMemOperand(MMO);
+  } else {
+    MachineMemOperand *MMO = MF->getMachineMemOperand(
+        MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
+        MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
+
     BuildMI(MBB, I, DL, get(Opcode))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI)
         .addImm(0)
         .addMemOperand(MMO);
   }
+}
 
 void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator I,
                                           Register DstReg, int FI,
                                           const TargetRegisterClass *RC,
                                           const TargetRegisterInfo *TRI) const {
   DebugLoc DL;
   if (I != MBB.end())
     DL = I->getDebugLoc();
 
   MachineFunction *MF = MBB.getParent();
-  const MachineFrameInfo &MFI = MF->getFrameInfo();
-  MachineMemOperand *MMO = MF->getMachineMemOperand(
-      MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
-      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
+  MachineFrameInfo &MFI = MF->getFrameInfo();
 
   unsigned Opcode;
+  bool IsScalableVector = false;
   if (RISCV::GPRRegClass.hasSubClassEq(RC))
     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
              RISCV::LW : RISCV::LD;
   else if (RISCV::FPR16RegClass.hasSubClassEq(RC))
     Opcode = RISCV::FLH;
   else if (RISCV::FPR32RegClass.hasSubClassEq(RC))
     Opcode = RISCV::FLW;
   else if (RISCV::FPR64RegClass.hasSubClassEq(RC))
     Opcode = RISCV::FLD;
-  else
+  else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::PseudoVRELOAD_M1;
+    IsScalableVector = true;
+  } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::PseudoVRELOAD_M2;
+    IsScalableVector = true;
+  } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::PseudoVRELOAD_M4;
+    IsScalableVector = true;
+  } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::PseudoVRELOAD_M8;
+    IsScalableVector = true;
+  } else
     llvm_unreachable("Can't load this register from stack slot");
 
+  if (IsScalableVector) {
+    MachineMemOperand *MMO = MF->getMachineMemOperand(
+        MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
+        MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
+
+    MFI.setStackID(FI, TargetStackID::RISCVVector);
+    BuildMI(MBB, I, DL, get(Opcode), DstReg)
+        .addFrameIndex(FI)
+        .addMemOperand(MMO);
+  } else {
+    MachineMemOperand *MMO = MF->getMachineMemOperand(
+        MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
+        MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
+
     BuildMI(MBB, I, DL, get(Opcode), DstReg)
         .addFrameIndex(FI)
         .addImm(0)
         .addMemOperand(MMO);
   }
+}
 
 void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator MBBI,
                             const DebugLoc &DL, Register DstReg, uint64_t Val,
                             MachineInstr::MIFlag Flag) const {
   MachineFunction *MF = MBB.getParent();
   MachineRegisterInfo &MRI = MF->getRegInfo();
   bool IsRV64 = MF->getSubtarget<RISCVSubtarget>().is64Bit();

llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td

let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
   def PseudoVMV4R_V : VPseudo<VMV4R_V, V_M4, (outs VRM4:$vd), (ins VRM4:$vs2)>;
   def PseudoVMV8R_V : VPseudo<VMV8R_V, V_M8, (outs VRM8:$vd), (ins VRM8:$vs2)>;
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1 in {
   def PseudoReadVLENB : Pseudo<(outs GPR:$rd), (ins), []>;
 }
 
+let hasSideEffects = 0, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1 in {
+  def PseudoVSPILL_M1 : VPseudo<VS1R_V, V_M1, (outs), (ins VR:$rs1, GPR:$rs2)>;
+  def PseudoVSPILL_M2 : VPseudo<VS2R_V, V_M2, (outs), (ins VRM2:$rs1, GPR:$rs2)>;
+  def PseudoVSPILL_M4 : VPseudo<VS4R_V, V_M4, (outs), (ins VRM4:$rs1, GPR:$rs2)>;
+  def PseudoVSPILL_M8 : VPseudo<VS8R_V, V_M8, (outs), (ins VRM8:$rs1, GPR:$rs2)>;
+}
+
+let hasSideEffects = 0, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1 in {
+  def PseudoVRELOAD_M1 : VPseudo<VL1RE8_V, V_M1, (outs VR:$rs1), (ins GPR:$rs2)>;
+  def PseudoVRELOAD_M2 : VPseudo<VL2RE8_V, V_M2, (outs VRM2:$rs1), (ins GPR:$rs2)>;
+  def PseudoVRELOAD_M4 : VPseudo<VL4RE8_V, V_M4, (outs VRM4:$rs1), (ins GPR:$rs2)>;
+  def PseudoVRELOAD_M8 : VPseudo<VL8RE8_V, V_M8, (outs VRM8:$rs1), (ins GPR:$rs2)>;
+}
+
 //===----------------------------------------------------------------------===//
 // 6. Configuration-Setting Instructions
 //===----------------------------------------------------------------------===//
 
 // Pseudos.
 let hasSideEffects = 1, mayLoad = 0, mayStore = 0, Defs = [VL, VTYPE] in {
 def PseudoVSETVLI : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei), []>;
 

llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp

RISCVRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   case RISCVABI::ABI_LP64D:
     return CSR_ILP32D_LP64D_SaveList;
   }
 }
 
 BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   const RISCVFrameLowering *TFI = getFrameLowering(MF);
   BitVector Reserved(getNumRegs());
+  const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
 
   // Mark any registers requested to be reserved as such
   for (size_t Reg = 0; Reg < getNumRegs(); Reg++) {
     if (MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(Reg))
       markSuperRegs(Reserved, Reg);
   }
 
   // Use markSuperRegs to ensure any register aliases are also reserved
   markSuperRegs(Reserved, RISCV::X0); // zero
   markSuperRegs(Reserved, RISCV::X2); // sp
   markSuperRegs(Reserved, RISCV::X3); // gp
   markSuperRegs(Reserved, RISCV::X4); // tp
-  if (TFI->hasFP(MF))
+  // If there is need to spill RVV object as register allocation, SP will
+  // be adjusted by a variable length at runtime. We need to reserve FP to
+  // access callee-saved registers. However, we have no information for RVV
+  // spilling after ISel lowering. Here, we reserve FP if V extension is
+  // enabled. See llvm/test/CodeGen/RISCV/rvv/spill-vector-csr.ll.
+  if (TFI->hasFP(MF) || Subtarget.hasStdExtV())
     markSuperRegs(Reserved, RISCV::X8); // fp
   // Reserve the base register if we need to realign the stack and allocate
   // variable-sized objects at runtime.
   if (TFI->hasBP(MF))
     markSuperRegs(Reserved, RISCVABI::getBPReg()); // bp
 
   // V registers for code generation. We handle them manually.
   markSuperRegs(Reserved, RISCV::VL);

llvm/test/CodeGen/RISCV/rvv/spill-vector-csr.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -mattr=+d -O0 < %s \
+; RUN:    | FileCheck --check-prefix=SPILL-O0 %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -mattr=+d -O2 < %s \
+; RUN:    | FileCheck --check-prefix=SPILL-O2 %s
+
+@.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
+
+define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double> %b, <vscale x 1 x double> %c, i64 %gvl) nounwind
+; SPILL-O0-LABEL: foo:
+; SPILL-O0:       # %bb.0:
+; SPILL-O0-NEXT:    addi sp, sp, -48
+; SPILL-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    addi s0, sp, 48
+; SPILL-O0-NEXT:    csrr a1, vlenb
+; SPILL-O0-NEXT:    sub sp, sp, a1
+; SPILL-O0-NEXT:    sd sp, -32(s0) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sub sp, sp, a1
+; SPILL-O0-NEXT:    sd sp, -40(s0) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd a0, -24(s0) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    ld a1, -32(s0)
+; SPILL-O0-NEXT:    vs1r.v v16, (a1) # Unknown size Folded Spill
+; SPILL-O0-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vfadd.vv v25, v16, v17
+; SPILL-O0-NEXT:    ld a0, -40(s0)
+; SPILL-O0-NEXT:    vs1r.v v25, (a0) # Unknown size Folded Spill
+; SPILL-O0-NEXT:    lui a0, %hi(.L.str)
+; SPILL-O0-NEXT:    addi a0, a0, %lo(.L.str)
+; SPILL-O0-NEXT:    call puts@plt
+; SPILL-O0-NEXT:    ld a1, -40(s0)
+; SPILL-O0-NEXT:    vl1r.v v25, (a1) # Unknown size Folded Reload
+; SPILL-O0-NEXT:    ld a1, -32(s0)
+; SPILL-O0-NEXT:    vl1r.v v16, (a1) # Unknown size Folded Reload
+; SPILL-O0-NEXT:    # kill: def $x11 killed $x10
+; SPILL-O0-NEXT:    ld a0, -24(s0) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vfadd.vv v16, v16, v25
+; SPILL-O0-NEXT:    addi sp, s0, -48
+; SPILL-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    addi sp, sp, 48
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: foo:
+; SPILL-O2:       # %bb.0:
+; SPILL-O2-NEXT:    addi sp, sp, -48
+; SPILL-O2-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    addi s0, sp, 48
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    sub sp, sp, a1
+; SPILL-O2-NEXT:    sd sp, -32(s0) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sub sp, sp, a1
+; SPILL-O2-NEXT:    sd sp, -40(s0) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    mv s1, a0
+; SPILL-O2-NEXT:    ld a1, -40(s0)
+; SPILL-O2-NEXT:    vs1r.v v16, (a1) # Unknown size Folded Spill
+; SPILL-O2-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
+; SPILL-O2-NEXT:    vfadd.vv v25, v16, v17
+; SPILL-O2-NEXT:    ld a0, -32(s0)
+; SPILL-O2-NEXT:    vs1r.v v25, (a0) # Unknown size Folded Spill
+; SPILL-O2-NEXT:    lui a0, %hi(.L.str)
+; SPILL-O2-NEXT:    addi a0, a0, %lo(.L.str)
+; SPILL-O2-NEXT:    call puts@plt
+; SPILL-O2-NEXT:    vsetvli a0, s1, e64,m1,ta,mu
+; SPILL-O2-NEXT:    ld a0, -32(s0)
+; SPILL-O2-NEXT:    vl1r.v v25, (a0) # Unknown size Folded Reload
+; SPILL-O2-NEXT:    ld a0, -40(s0)
+; SPILL-O2-NEXT:    vl1r.v v26, (a0) # Unknown size Folded Reload
+; SPILL-O2-NEXT:    vfadd.vv v16, v26, v25
+; SPILL-O2-NEXT:    addi sp, s0, -48
+; SPILL-O2-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    addi sp, sp, 48
+; SPILL-O2-NEXT:    ret
+{
+   %x = call <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b, i64 %gvl)
+   %call = call signext i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+   %z = call <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %x, i64 %gvl)
+   ret <vscale x 1 x double> %z
+}
+
+declare <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b, i64 %gvl)
+declare i32 @puts(i8*);

llvm/test/CodeGen/RISCV/rvv/spill-vector.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -O0 < %s \
+; RUN:    | FileCheck --check-prefix=SPILL-O0 %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -O2 < %s \
+; RUN:    | FileCheck --check-prefix=SPILL-O2 %s
+
+define <vscale x 1 x i64> @spill_lmul_1(<vscale x 1 x i64> %va) nounwind {
+; SPILL-O0-LABEL: spill_lmul_1:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -32
+; SPILL-O0-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    addi s0, sp, 32
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    sub sp, sp, a0
+; SPILL-O0-NEXT:    sd sp, -24(s0) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    ld a0, -24(s0)
+; SPILL-O0-NEXT:    vs1r.v v16, (a0) # Unknown size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    ld a0, -24(s0)
+; SPILL-O0-NEXT:    vl1r.v v16, (a0) # Unknown size Folded Reload
+; SPILL-O0-NEXT:    addi sp, s0, -32
+; SPILL-O0-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    addi sp, sp, 32
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_lmul_1:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -32
+; SPILL-O2-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    addi s0, sp, 32
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    sub sp, sp, a0
+; SPILL-O2-NEXT:    sd sp, -24(s0) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    ld a0, -24(s0)
+; SPILL-O2-NEXT:    vs1r.v v16, (a0) # Unknown size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    ld a0, -24(s0)
+; SPILL-O2-NEXT:    vl1r.v v16, (a0) # Unknown size Folded Reload
+; SPILL-O2-NEXT:    addi sp, s0, -32
+; SPILL-O2-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    addi sp, sp, 32
+; SPILL-O2-NEXT:    ret
+entry:
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+
+  ret <vscale x 1 x i64> %va
+}
+
+define <vscale x 2 x i64> @spill_lmul_2(<vscale x 2 x i64> %va) nounwind {
+; SPILL-O0-LABEL: spill_lmul_2:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -32
+; SPILL-O0-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    addi s0, sp, 32
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 1
+; SPILL-O0-NEXT:    sub sp, sp, a0
+; SPILL-O0-NEXT:    sd sp, -24(s0) # 16-byte Folded Spill
+; SPILL-O0-NEXT:    ld a0, -24(s0)
+; SPILL-O0-NEXT:    vs2r.v v16, (a0) # Unknown size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    ld a0, -24(s0)
+; SPILL-O0-NEXT:    vl2r.v v16, (a0) # Unknown size Folded Reload
+; SPILL-O0-NEXT:    addi sp, s0, -32
+; SPILL-O0-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    addi sp, sp, 32
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_lmul_2:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -32
+; SPILL-O2-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    addi s0, sp, 32
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 1
+; SPILL-O2-NEXT:    sub sp, sp, a0
+; SPILL-O2-NEXT:    sd sp, -24(s0) # 16-byte Folded Spill
+; SPILL-O2-NEXT:    ld a0, -24(s0)
+; SPILL-O2-NEXT:    vs2r.v v16, (a0) # Unknown size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    ld a0, -24(s0)
+; SPILL-O2-NEXT:    vl2r.v v16, (a0) # Unknown size Folded Reload
+; SPILL-O2-NEXT:    addi sp, s0, -32
+; SPILL-O2-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    addi sp, sp, 32
+; SPILL-O2-NEXT:    ret
+entry:
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+
+  ret <vscale x 2 x i64> %va
+}
+
+define <vscale x 4 x i64> @spill_lmul_4(<vscale x 4 x i64> %va) nounwind {
+; SPILL-O0-LABEL: spill_lmul_4:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -32
+; SPILL-O0-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    addi s0, sp, 32
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 2
+; SPILL-O0-NEXT:    sub sp, sp, a0
+; SPILL-O0-NEXT:    sd sp, -24(s0) # 32-byte Folded Spill
+; SPILL-O0-NEXT:    ld a0, -24(s0)
+; SPILL-O0-NEXT:    vs4r.v v16, (a0) # Unknown size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    ld a0, -24(s0)
+; SPILL-O0-NEXT:    vl4r.v v16, (a0) # Unknown size Folded Reload
+; SPILL-O0-NEXT:    addi sp, s0, -32
+; SPILL-O0-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    addi sp, sp, 32
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_lmul_4:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -32
+; SPILL-O2-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    addi s0, sp, 32
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 2
+; SPILL-O2-NEXT:    sub sp, sp, a0
+; SPILL-O2-NEXT:    sd sp, -24(s0) # 32-byte Folded Spill
+; SPILL-O2-NEXT:    ld a0, -24(s0)
+; SPILL-O2-NEXT:    vs4r.v v16, (a0) # Unknown size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    ld a0, -24(s0)
+; SPILL-O2-NEXT:    vl4r.v v16, (a0) # Unknown size Folded Reload
+; SPILL-O2-NEXT:    addi sp, s0, -32
+; SPILL-O2-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    addi sp, sp, 32
+; SPILL-O2-NEXT:    ret
+entry:
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+
+  ret <vscale x 4 x i64> %va
+}
+
+define <vscale x 8 x i64> @spill_lmul_8(<vscale x 8 x i64> %va) nounwind {
+; SPILL-O0-LABEL: spill_lmul_8:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -32
+; SPILL-O0-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    addi s0, sp, 32
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 3
+; SPILL-O0-NEXT:    sub sp, sp, a0
+; SPILL-O0-NEXT:    sd sp, -24(s0) # 64-byte Folded Spill
+; SPILL-O0-NEXT:    ld a0, -24(s0)
+; SPILL-O0-NEXT:    vs8r.v v16, (a0) # Unknown size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    ld a0, -24(s0)
+; SPILL-O0-NEXT:    vl8r.v v16, (a0) # Unknown size Folded Reload
+; SPILL-O0-NEXT:    addi sp, s0, -32
+; SPILL-O0-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    addi sp, sp, 32
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_lmul_8:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -32
+; SPILL-O2-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    addi s0, sp, 32
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 3
+; SPILL-O2-NEXT:    sub sp, sp, a0
+; SPILL-O2-NEXT:    sd sp, -24(s0) # 64-byte Folded Spill
+; SPILL-O2-NEXT:    ld a0, -24(s0)
+; SPILL-O2-NEXT:    vs8r.v v16, (a0) # Unknown size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    ld a0, -24(s0)
+; SPILL-O2-NEXT:    vl8r.v v16, (a0) # Unknown size Folded Reload
+; SPILL-O2-NEXT:    addi sp, s0, -32
+; SPILL-O2-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    addi sp, sp, 32
+; SPILL-O2-NEXT:    ret
+entry:
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+
+  ret <vscale x 8 x i64> %va
+}