[VE] Add lvm/svm intrinsic instructions

[kaz7]

Summary

Add lvm/svm intrinsic instructions and a regression test. Change
RegisterInfo to specify that VM0/VMP0 are constant and reserved
registers. This modifies a vst regression test, so update it.
Also add pseudo instructions for VM512 register classes
and mechanism to expand them after register allocation.

[simoll]

llvm/lib/Target/VE/VEInstrInfo.cpp

743–744

Looks like a good application for the Register type, here and for Src, VMX, ..

[kaz7]

Rebased to the upstream/master and change to use Register.

llvm/include/llvm/IR/IntrinsicsVEVL.gen.td

 let TargetPrefix = "ve" in def int_ve_vl_vstl2dncot_vssl : GCCBuiltin<"__builtin_ve_vl_vstl2dncot_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vstl2dncot_vssml : GCCBuiltin<"__builtin_ve_vl_vstl2dncot_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_pfchv_ssl : GCCBuiltin<"__builtin_ve_vl_pfchv_ssl">, Intrinsic<[], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrInaccessibleMemOrArgMemOnly]>;
 let TargetPrefix = "ve" in def int_ve_vl_pfchvnc_ssl : GCCBuiltin<"__builtin_ve_vl_pfchvnc_ssl">, Intrinsic<[], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrInaccessibleMemOrArgMemOnly]>;
 let TargetPrefix = "ve" in def int_ve_vl_lsv_vvss : GCCBuiltin<"__builtin_ve_vl_lsv_vvss">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i64>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_lvsl_svs : GCCBuiltin<"__builtin_ve_vl_lvsl_svs">, Intrinsic<[LLVMType<i64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_lvsd_svs : GCCBuiltin<"__builtin_ve_vl_lvsd_svs">, Intrinsic<[LLVMType<f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_lvss_svs : GCCBuiltin<"__builtin_ve_vl_lvss_svs">, Intrinsic<[LLVMType<f32>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_lvm_mmss : GCCBuiltin<"__builtin_ve_vl_lvm_mmss">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<i64>, LLVMType<i64>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_lvm_MMss : GCCBuiltin<"__builtin_ve_vl_lvm_MMss">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<i64>, LLVMType<i64>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_svm_sms : GCCBuiltin<"__builtin_ve_vl_svm_sms">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i64>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_svm_sMs : GCCBuiltin<"__builtin_ve_vl_svm_sMs">, Intrinsic<[LLVMType<i64>], [LLVMType<v512i1>, LLVMType<i64>], [IntrNoMem]>;

llvm/lib/Target/VE/VEInstrInfo.cpp

Register VEInstrInfo::getGlobalBaseReg(MachineFunction *MF) const {
   MachineBasicBlock &FirstMBB = MF->front();
   MachineBasicBlock::iterator MBBI = FirstMBB.begin();
   DebugLoc dl;
   BuildMI(FirstMBB, MBBI, dl, get(VE::GETGOT), GlobalBaseReg);
   VEFI->setGlobalBaseReg(GlobalBaseReg);
   return GlobalBaseReg;
 }
 
+static Register getVM512Upper(Register reg) {
+  return (reg - VE::VMP0) * 2 + VE::VM0;
+}
+
+static Register getVM512Lower(Register reg) { return getVM512Upper(reg) + 1; }
+
 bool VEInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   switch (MI.getOpcode()) {
   case VE::EXTEND_STACK: {
     return expandExtendStackPseudo(MI);
   }
   case VE::EXTEND_STACK_GUARD: {
     MI.eraseFromParent(); // The pseudo instruction is gone now.
     return true;
   }
   case VE::GETSTACKTOP: {
     return expandGetStackTopPseudo(MI);
   }
+
+  case VE::LVMyir:
+  case VE::LVMyim:
+  case VE::LVMyir_y:
+  case VE::LVMyim_y: {
+    Register VMXu = getVM512Upper(MI.getOperand(0).getReg());
+    Register VMXl = getVM512Lower(MI.getOperand(0).getReg());
+    int64_t Imm = MI.getOperand(1).getImm();
+    bool IsSrcReg =
+        MI.getOpcode() == VE::LVMyir || MI.getOpcode() == VE::LVMyir_y;
+    Register Src = IsSrcReg ? MI.getOperand(2).getReg() : VE::NoRegister;
+    int64_t MImm = IsSrcReg ? 0 : MI.getOperand(2).getImm();
+    bool KillSrc = IsSrcReg ? MI.getOperand(2).isKill() : false;
+    Register VMX = VMXl;
+    if (Imm >= 4) {
+      VMX = VMXu;
+      Imm -= 4;
+    }
+    MachineBasicBlock *MBB = MI.getParent();
+    DebugLoc DL = MI.getDebugLoc();
+    switch (MI.getOpcode()) {
+    case VE::LVMyir:
+      BuildMI(*MBB, MI, DL, get(VE::LVMir))
+          .addDef(VMX)
+          .addImm(Imm)
+          .addReg(Src, getKillRegState(KillSrc));
+      break;
+    case VE::LVMyim:
+      BuildMI(*MBB, MI, DL, get(VE::LVMim))
+          .addDef(VMX)
+          .addImm(Imm)
+          .addImm(MImm);
+      break;
+    case VE::LVMyir_y:
+      assert(MI.getOperand(0).getReg() == MI.getOperand(3).getReg() &&
+             "LVMyir_y has different register in 3rd operand");
+      BuildMI(*MBB, MI, DL, get(VE::LVMir_m))
+          .addDef(VMX)
+          .addImm(Imm)
+          .addReg(Src, getKillRegState(KillSrc))
+          .addReg(VMX);
+      break;
+    case VE::LVMyim_y:
+      assert(MI.getOperand(0).getReg() == MI.getOperand(3).getReg() &&
+             "LVMyim_y has different register in 3rd operand");
+      BuildMI(*MBB, MI, DL, get(VE::LVMim_m))
+          .addDef(VMX)
+          .addImm(Imm)
+          .addImm(MImm)
+          .addReg(VMX);
+      break;
+    }
+    MI.eraseFromParent();
+    return true;
+  }
+  case VE::SVMyi: {
+    Register Dest = MI.getOperand(0).getReg();
+    Register VMZu = getVM512Upper(MI.getOperand(1).getReg());
+    Register VMZl = getVM512Lower(MI.getOperand(1).getReg());
+    bool KillSrc = MI.getOperand(1).isKill();
+    int64_t Imm = MI.getOperand(2).getImm();
+    Register VMZ = VMZl;
+    if (Imm >= 4) {
+      VMZ = VMZu;
+      Imm -= 4;
+    }
+    MachineBasicBlock *MBB = MI.getParent();
+    DebugLoc DL = MI.getDebugLoc();
+    MachineInstrBuilder MIB =
+        BuildMI(*MBB, MI, DL, get(VE::SVMmi), Dest).addReg(VMZ).addImm(Imm);
+    MachineInstr *Inst = MIB.getInstr();
+    MI.eraseFromParent();
+    if (KillSrc) {
+      const TargetRegisterInfo *TRI = &getRegisterInfo();
+      Inst->addRegisterKilled(MI.getOperand(1).getReg(), TRI, true);
+    }
+    return true;
+  }
   }
   return false;
 }
 
 bool VEInstrInfo::expandExtendStackPseudo(MachineInstr &MI) const {
   MachineBasicBlock &MBB = *MI.getParent();
   MachineFunction &MF = *MBB.getParent();
   const VESubtarget &STI = MF.getSubtarget<VESubtarget>();

llvm/lib/Target/VE/VEInstrIntrinsicVL.gen.td

 def : Pat<(int_ve_vl_vstl2dncot_vssl v256f64:$vx, i64:$sy, i64:$sz, i32:$vl), (VSTL2DNCOTrrvl i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
 def : Pat<(int_ve_vl_vstl2dncot_vssl v256f64:$vx, simm7:$I, i64:$sz, i32:$vl), (VSTL2DNCOTirvl (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
 def : Pat<(int_ve_vl_vstl2dncot_vssml v256f64:$vx, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSTL2DNCOTrrvml i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
 def : Pat<(int_ve_vl_vstl2dncot_vssml v256f64:$vx, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSTL2DNCOTirvml (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
 def : Pat<(int_ve_vl_pfchv_ssl i64:$sy, i64:$sz, i32:$vl), (PFCHVrrl i64:$sy, i64:$sz, i32:$vl)>;
 def : Pat<(int_ve_vl_pfchv_ssl simm7:$I, i64:$sz, i32:$vl), (PFCHVirl (LO7 $I), i64:$sz, i32:$vl)>;
 def : Pat<(int_ve_vl_pfchvnc_ssl i64:$sy, i64:$sz, i32:$vl), (PFCHVNCrrl i64:$sy, i64:$sz, i32:$vl)>;
 def : Pat<(int_ve_vl_pfchvnc_ssl simm7:$I, i64:$sz, i32:$vl), (PFCHVNCirl (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_lvm_mmss v256i1:$ptm, uimm6:$N, i64:$sz), (LVMir_m (ULO7 $N), i64:$sz, v256i1:$ptm)>;
+def : Pat<(int_ve_vl_lvm_MMss v512i1:$ptm, uimm6:$N, i64:$sz), (LVMyir_y (ULO7 $N), i64:$sz, v512i1:$ptm)>;
+def : Pat<(int_ve_vl_svm_sms v256i1:$vmz, uimm6:$N), (SVMmi v256i1:$vmz, (ULO7 $N))>;
+def : Pat<(int_ve_vl_svm_sMs v512i1:$vmz, uimm6:$N), (SVMyi v512i1:$vmz, (ULO7 $N))>;

llvm/lib/Target/VE/VEInstrVec.td

 //===----------------------------------------------------------------------===//
 // Vector Instructions
 //===----------------------------------------------------------------------===//
 
+// Pseudo instructions for VM512 modifications
+//
+// Specifies hasSideEffects = 0 to disable UnmodeledSideEffects.
+
+let hasSideEffects = 0 in {
+  let Constraints = "$vx = $vd", DisableEncoding = "$vd" in {
+    def LVMyir_y : Pseudo<(outs VM512:$vx), (ins uimm3:$sy, I64:$sz, VM512:$vd),
+                          "# pseudo LVM $vx, $sy, $sz, $vd">;
+    def LVMyim_y : Pseudo<(outs VM512:$vx),
+                          (ins uimm3:$sy, mimm:$sz, VM512:$vd),
+                          "# pseudo LVM $vx, $sy, $sz, $vd">;
+  }
+  def LVMyir : Pseudo<(outs VM512:$vx), (ins uimm3:$sy, I64:$sz),
+                      "# pseudo LVM $vx, $sy, $sz">;
+  def LVMyim : Pseudo<(outs VM512:$vx), (ins uimm3:$sy, mimm:$sz),
+                      "# pseudo LVM $vx, $sy, $sz">;
+  def SVMyi : Pseudo<(outs I64:$sx), (ins VM512:$vz, uimm3:$sy),
+                     "# pseudo SVM $sx, $vz, $sy">;
+}
+
 //===----------------------------------------------------------------------===//
 // Instructions
 //
 // Define all vector instructions defined in SX-Aurora TSUBASA Architecture
 // Guide here.  As those mnemonics, we use mnemonics defined in Vector Engine
 // Assembly Language Reference Manual.
 //
 // Some instructions can update existing data by following instructions

llvm/lib/Target/VE/VERegisterInfo.cpp

const Register ReservedRegs[] = {
                 // sx34-sx63 are temporary registers
   };
 
   for (auto R : ReservedRegs)
     for (MCRegAliasIterator ItAlias(R, this, true); ItAlias.isValid();
          ++ItAlias)
       Reserved.set(*ItAlias);
 
+  // Reserve constant registers.
+  Reserved.set(VE::VM0);
+  Reserved.set(VE::VMP0);
+
   return Reserved;
 }
 
-bool VERegisterInfo::isConstantPhysReg(MCRegister PhysReg) const { return false; }
+bool VERegisterInfo::isConstantPhysReg(MCRegister PhysReg) const {
+  switch (PhysReg) {
+  case VE::VM0:
+  case VE::VMP0:
+    return true;
+  default:
+    return false;
+  }
+}
 
 const TargetRegisterClass *
 VERegisterInfo::getPointerRegClass(const MachineFunction &MF,
                                    unsigned Kind) const {
   return &VE::I64RegClass;
 }
 
 static void replaceFI(MachineFunction &MF, MachineBasicBlock::iterator II,

llvm/test/CodeGen/VE/VELIntrinsics/lvm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test load/save vector mask intrinsic instructions
+;;;
+;;; Note:
+;;;   We test LVMir_m, LVMyir_y, SVMmi, and SVMyi instructions.
+
+; Function Attrs: nounwind readnone
+define i64 @lvm_mmss(i8* nocapture readnone %0, i64 %1) {
+; CHECK-LABEL: lvm_mmss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lvm %vm1, 3, %s1
+; CHECK-NEXT:    svm %s0, %vm1, 3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.lvm.mmss(<256 x i1> undef, i64 3, i64 %1)
+  %4 = tail call i64 @llvm.ve.vl.svm.sms(<256 x i1> %3, i64 3)
+  ret i64 %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.lvm.mmss(<256 x i1>, i64, i64)
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.ve.vl.svm.sms(<256 x i1>, i64)
+
+; Function Attrs: nounwind readnone
+define i64 @lvml_MMss(i8* nocapture readnone %0, i64 %1) {
+; CHECK-LABEL: lvml_MMss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lvm %vm2, 1, %s1
+; CHECK-NEXT:    svm %s0, %vm3, 3
+; CHECK-NEXT:    svm %s1, %vm2, 2
+; CHECK-NEXT:    adds.l %s0, %s1, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.lvm.MMss(<512 x i1> undef, i64 5, i64 %1)
+  %4 = tail call i64 @llvm.ve.vl.svm.sMs(<512 x i1> %3, i64 3)
+  %5 = tail call i64 @llvm.ve.vl.svm.sMs(<512 x i1> %3, i64 6)
+  %6 = add i64 %5, %4
+  ret i64 %6
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.lvm.MMss(<512 x i1>, i64, i64)
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.ve.vl.svm.sMs(<512 x i1>, i64)

llvm/test/CodeGen/VE/VELIntrinsics/vst.ll

 
 ; Function Attrs: nounwind
 define void @vst_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vst_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vst %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vst %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vst.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vst_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vst_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vst %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vst %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstnc_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstnc_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstnc_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vst.nc %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vst.nc %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstnc.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstnc.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstnc_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstnc_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vst.nc %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vst.nc %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstnc.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vst.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vst.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vst.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vst.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstncot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstncot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstncot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vst.nc.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vst.nc.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstncot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstncot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstncot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstncot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vst.nc.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vst.nc.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstncot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstu_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstu_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstu_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstu %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstu %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstu.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstu_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstu_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstu %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstu %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstunc_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstunc_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstunc_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstu.nc %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstu.nc %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstunc.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstunc.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstunc_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstunc_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstu.nc %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstu.nc %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstunc.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstuot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstuot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstuot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstu.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstu.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstuot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstuot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstuot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstuot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstu.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstu.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstuot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstuncot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstuncot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstuncot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstu.nc.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstu.nc.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstuncot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstuncot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstuncot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstuncot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstu.nc.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstu.nc.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstuncot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstl_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstl_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstl_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstl %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstl %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstl.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstl_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstl_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstl %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstl %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstlnc_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstlnc_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstlnc_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstl.nc %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstl.nc %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstlnc.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstlnc.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstlnc_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstlnc_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstl.nc %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstl.nc %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstlnc.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstlot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstlot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstlot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstl.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstl.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstlot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstlot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstlot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstlot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstl.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstl.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstlot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstlncot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstlncot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstlncot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstl.nc.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstl.nc.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstlncot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstlncot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstlncot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstlncot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstl.nc.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstl.nc.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstlncot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vst2d_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vst2d_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vst2d_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vst2d %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vst2d %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst2d.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vst2d.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vst2d_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vst2d_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vst2d %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vst2d %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst2d.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vst2dnc_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vst2dnc_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vst2dnc_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vst2d.nc %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vst2d.nc %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst2dnc.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vst2dnc.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vst2dnc_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vst2dnc_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vst2d.nc %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vst2d.nc %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst2dnc.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vst2dot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vst2dot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vst2dot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vst2d.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vst2d.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst2dot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vst2dot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vst2dot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vst2dot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vst2d.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vst2d.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst2dot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vst2dncot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vst2dncot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vst2dncot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vst2d.nc.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vst2d.nc.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst2dncot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vst2dncot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vst2dncot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vst2dncot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vst2d.nc.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vst2d.nc.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vst2dncot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstu2d_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstu2d_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstu2d_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstu2d %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstu2d %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu2d.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstu2d.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstu2d_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstu2d_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstu2d %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstu2d %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu2d.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstu2dnc_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstu2dnc_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstu2dnc_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstu2d.nc %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstu2d.nc %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu2dnc.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstu2dnc.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstu2dnc_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstu2dnc_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstu2d.nc %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstu2d.nc %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu2dnc.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstu2dot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstu2dot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstu2dot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstu2d.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstu2d.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu2dot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstu2dot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstu2dot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstu2dot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstu2d.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstu2d.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu2dot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstu2dncot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstu2dncot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstu2dncot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstu2d.nc.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstu2d.nc.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu2dncot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstu2dncot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstu2dncot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstu2dncot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstu2d.nc.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstu2d.nc.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstu2dncot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstl2d_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstl2d_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstl2d_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstl2d %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstl2d %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl2d.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstl2d.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstl2d_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstl2d_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstl2d %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstl2d %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl2d.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstl2dnc_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstl2dnc_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstl2dnc_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstl2d.nc %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstl2d.nc %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl2dnc.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstl2dnc.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstl2dnc_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstl2dnc_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstl2d.nc %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstl2d.nc %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl2dnc.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstl2dot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstl2dot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstl2dot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstl2d.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstl2d.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl2dot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstl2dot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstl2dot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstl2dot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstl2d.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstl2d.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl2dot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind
 define void @vstl2dncot_vssl(i8* %0, i64 %1) {

 
 ; Function Attrs: nounwind
 define void @vstl2dncot_vssml(i8* %0, i64 %1) {
 ; CHECK-LABEL: vstl2dncot_vssml:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, %s1, %s0
-; CHECK-NEXT:    vstl2d.nc.ot %v0, %s1, %s0, %vm0
+; CHECK-NEXT:    vstl2d.nc.ot %v0, %s1, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 %1, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl2dncot.vssml(<256 x double> %3, i64 %1, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }
 
 ; Function Attrs: nounwind writeonly
 declare void @llvm.ve.vl.vstl2dncot.vssml(<256 x double>, i64, i8*, <256 x i1>, i32)

 
 ; Function Attrs: nounwind
 define void @vstl2dncot_vssml_imm(i8* %0) {
 ; CHECK-LABEL: vstl2dncot_vssml_imm:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    vstl2d.nc.ot %v0, 8, %s0, %vm0
+; CHECK-NEXT:    vstl2d.nc.ot %v0, 8, %s0, %vm1
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
   tail call void @llvm.ve.vl.vstl2dncot.vssml(<256 x double> %2, i64 8, i8* %0, <256 x i1> undef, i32 256)
   ret void
 }