[VE] Add vector load/store instructions

[kaz7]

Summary

Add vector registers and vector load/store instructions. Add
regression tests for vector load/store instructions too.

[simoll]

llvm/lib/Target/VE/VEISelLowering.cpp

637–672

I wonder why there are vector types that do not map directly to vector registers.

[kaz7]

Reply to an inline comment.

llvm/lib/Target/VE/VEISelLowering.cpp

637–672

Which types are not mapped as you are asked? I think all vector types from v2f64 to v256f64 are mapped.

[simoll]

llvm/lib/Target/VE/VEISelLowering.cpp

637–672

Yes, they are mapped now, but why are there more vector types being added than necessary? For example, v16f64 is not a vector register type for VE. If it was not mapped, LLVM could still legalize/expand it into a legal v256f64 type.
I was expecting to see only v256i64, v256f64, v512i32,v512f32,v256i1 and v512i1 in this list but there may be a good reason to have all other vector types here too.

[kaz7]

llvm/lib/Target/VE/VEISelLowering.cpp

637–672

I've understood your point. The reason doing this is to support regular SIMD fixed-sized-vector optimization on VE in the future. I would like to support both existing SIMD optimization and vector predication optimization. This SIMD optimization for VE is partially implemented locally.

On the other hand, there was a discussion like that we should support not only V64 (16384) register but also V16384, V8192, V4096, ..., V64 registers. Nobody except me wants it, so not implemented yet, though.

[simoll]

We can keep those register mappings for now and refine them later.

llvm/lib/Target/VE/Disassembler/VEDisassembler.cpp

 //===- VEDisassembler.cpp - Disassembler for VE -----------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is part of the VE Disassembler.
 //
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/VEMCTargetDesc.h"
 #include "TargetInfo/VETargetInfo.h"
 #include "VE.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCFixedLenDisassembler.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/Support/TargetRegistry.h"
static const unsigned F32RegDecoderTable[] = {
     VE::SF63};
 
 static const unsigned F128RegDecoderTable[] = {
     VE::Q0,  VE::Q1,  VE::Q2,  VE::Q3,  VE::Q4,  VE::Q5,  VE::Q6,  VE::Q7,
     VE::Q8,  VE::Q9,  VE::Q10, VE::Q11, VE::Q12, VE::Q13, VE::Q14, VE::Q15,
     VE::Q16, VE::Q17, VE::Q18, VE::Q19, VE::Q20, VE::Q21, VE::Q22, VE::Q23,
     VE::Q24, VE::Q25, VE::Q26, VE::Q27, VE::Q28, VE::Q29, VE::Q30, VE::Q31};
 
+static const unsigned V64RegDecoderTable[] = {
+    VE::V0,  VE::V1,  VE::V2,  VE::V3,  VE::V4,  VE::V5,  VE::V6,  VE::V7,
+    VE::V8,  VE::V9,  VE::V10, VE::V11, VE::V12, VE::V13, VE::V14, VE::V15,
+    VE::V16, VE::V17, VE::V18, VE::V19, VE::V20, VE::V21, VE::V22, VE::V23,
+    VE::V24, VE::V25, VE::V26, VE::V27, VE::V28, VE::V29, VE::V30, VE::V31,
+    VE::V32, VE::V33, VE::V34, VE::V35, VE::V36, VE::V37, VE::V38, VE::V39,
+    VE::V40, VE::V41, VE::V42, VE::V43, VE::V44, VE::V45, VE::V46, VE::V47,
+    VE::V48, VE::V49, VE::V50, VE::V51, VE::V52, VE::V53, VE::V54, VE::V55,
+    VE::V56, VE::V57, VE::V58, VE::V59, VE::V60, VE::V61, VE::V62, VE::V63};
+
+static const unsigned VMRegDecoderTable[] = {
+    VE::VM0,  VE::VM1,  VE::VM2,  VE::VM3, VE::VM4,  VE::VM5,
+    VE::VM6,  VE::VM7,  VE::VM8,  VE::VM9, VE::VM10, VE::VM11,
+    VE::VM12, VE::VM13, VE::VM14, VE::VM15};
+
+static const unsigned VM512RegDecoderTable[] = {VE::VMP0, VE::VMP1, VE::VMP2,
+                                                VE::VMP3, VE::VMP4, VE::VMP5,
+                                                VE::VMP6, VE::VMP7};
+
 static const unsigned MiscRegDecoderTable[] = {
     VE::USRCC,      VE::PSW,        VE::SAR,        VE::NoRegister,
     VE::NoRegister, VE::NoRegister, VE::NoRegister, VE::PMMR,
     VE::PMCR0,      VE::PMCR1,      VE::PMCR2,      VE::PMCR3,
     VE::NoRegister, VE::NoRegister, VE::NoRegister, VE::NoRegister,
     VE::PMC0,       VE::PMC1,       VE::PMC2,       VE::PMC3,
     VE::PMC4,       VE::PMC5,       VE::PMC6,       VE::PMC7,
     VE::PMC8,       VE::PMC9,       VE::PMC10,      VE::PMC11,
static DecodeStatus DecodeF128RegisterClass(MCInst &Inst, unsigned RegNo,
                                             const void *Decoder) {
   if (RegNo % 2 || RegNo > 63)
     return MCDisassembler::Fail;
   unsigned Reg = F128RegDecoderTable[RegNo / 2];
   Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
+static DecodeStatus DecodeV64RegisterClass(MCInst &Inst, unsigned RegNo,
+                                           uint64_t Address,
+                                           const void *Decoder) {
+  unsigned Reg = VE::NoRegister;
+  if (RegNo == 255)
+    Reg = VE::VIX;
+  else if (RegNo > 63)
+    return MCDisassembler::Fail;
+  else
+    Reg = V64RegDecoderTable[RegNo];
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeVMRegisterClass(MCInst &Inst, unsigned RegNo,
+                                          uint64_t Address,
+                                          const void *Decoder) {
+  if (RegNo > 15)
+    return MCDisassembler::Fail;
+  unsigned Reg = VMRegDecoderTable[RegNo];
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus DecodeVM512RegisterClass(MCInst &Inst, unsigned RegNo,
+                                             uint64_t Address,
+                                             const void *Decoder) {
+  if (RegNo % 2 || RegNo > 15)
+    return MCDisassembler::Fail;
+  unsigned Reg = VM512RegDecoderTable[RegNo / 2];
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
 static DecodeStatus DecodeMISCRegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
   if (RegNo > 30)
     return MCDisassembler::Fail;
   unsigned Reg = MiscRegDecoderTable[RegNo];
   if (Reg == VE::NoRegister)
     return MCDisassembler::Fail;

llvm/lib/Target/VE/VEISelLowering.cpp

VETargetLowering::VETargetLowering(const TargetMachine &TM,
 
   // Set up the register classes.
   addRegisterClass(MVT::i32, &VE::I32RegClass);
   addRegisterClass(MVT::i64, &VE::I64RegClass);
   addRegisterClass(MVT::f32, &VE::F32RegClass);
   addRegisterClass(MVT::f64, &VE::I64RegClass);
   addRegisterClass(MVT::f128, &VE::F128RegClass);
 
+  addRegisterClass(MVT::v2i32, &VE::V64RegClass);
+  addRegisterClass(MVT::v4i32, &VE::V64RegClass);
+  addRegisterClass(MVT::v8i32, &VE::V64RegClass);
+  addRegisterClass(MVT::v16i32, &VE::V64RegClass);
+  addRegisterClass(MVT::v32i32, &VE::V64RegClass);
+  addRegisterClass(MVT::v64i32, &VE::V64RegClass);
+  addRegisterClass(MVT::v128i32, &VE::V64RegClass);
+  addRegisterClass(MVT::v256i32, &VE::V64RegClass);
+  addRegisterClass(MVT::v512i32, &VE::V64RegClass);
+
+  addRegisterClass(MVT::v2i64, &VE::V64RegClass);
+  addRegisterClass(MVT::v4i64, &VE::V64RegClass);
+  addRegisterClass(MVT::v8i64, &VE::V64RegClass);
+  addRegisterClass(MVT::v16i64, &VE::V64RegClass);
+  addRegisterClass(MVT::v32i64, &VE::V64RegClass);
+  addRegisterClass(MVT::v64i64, &VE::V64RegClass);
+  addRegisterClass(MVT::v128i64, &VE::V64RegClass);
+  addRegisterClass(MVT::v256i64, &VE::V64RegClass);
+
+  addRegisterClass(MVT::v2f32, &VE::V64RegClass);
+  addRegisterClass(MVT::v4f32, &VE::V64RegClass);
+  addRegisterClass(MVT::v8f32, &VE::V64RegClass);
+  addRegisterClass(MVT::v16f32, &VE::V64RegClass);
+  addRegisterClass(MVT::v32f32, &VE::V64RegClass);
+  addRegisterClass(MVT::v64f32, &VE::V64RegClass);
+  addRegisterClass(MVT::v128f32, &VE::V64RegClass);
+  addRegisterClass(MVT::v256f32, &VE::V64RegClass);
+  addRegisterClass(MVT::v512f32, &VE::V64RegClass);
+
+  addRegisterClass(MVT::v2f64, &VE::V64RegClass);
+  addRegisterClass(MVT::v4f64, &VE::V64RegClass);
+  addRegisterClass(MVT::v8f64, &VE::V64RegClass);
+  addRegisterClass(MVT::v16f64, &VE::V64RegClass);
+  addRegisterClass(MVT::v32f64, &VE::V64RegClass);
+  addRegisterClass(MVT::v64f64, &VE::V64RegClass);
+  addRegisterClass(MVT::v128f64, &VE::V64RegClass);
+  addRegisterClass(MVT::v256f64, &VE::V64RegClass);
+
+  addRegisterClass(MVT::v256i1, &VE::VMRegClass);
+  addRegisterClass(MVT::v512i1, &VE::VM512RegClass);
+
   /// Load & Store {
   for (MVT FPVT : MVT::fp_valuetypes()) {
     for (MVT OtherFPVT : MVT::fp_valuetypes()) {
       // Turn FP extload into load/fpextend
       setLoadExtAction(ISD::EXTLOAD, FPVT, OtherFPVT, Expand);
 
       // Turn FP truncstore into trunc + store.
       setTruncStoreAction(FPVT, OtherFPVT, Expand);

llvm/lib/Target/VE/VEInstrFormats.td

class InstVE<dag outs, dag ins, string asmstr, list<dag> pattern>
   bits<8> op;
   let Inst{63-56} = op;
 
   dag OutOperandList = outs;
   dag InOperandList = ins;
   let AsmString   = asmstr;
   let Pattern = pattern;
 
+  bits<1> VE_Vector = 0;
+  bits<1> VE_VLInUse = 0;
+  bits<3> VE_VLIndex = 0;
+  bits<1> VE_VLWithMask = 0;
+
+  /// These fields correspond to the fields in VEInstrInfo.h.  Any changes to
+  /// these must be reflected there!  See comments there for what these are.
+  ///
+  /// VLIndex is the index of VL register in MI's operands.  The HW instruction
+  /// doesn't have that field, but we add is in MI for the ease of optimization.
+  /// For example, the index of VL of (VST $sy, $sz, $sx, $vl) is 3 (beginning
+  /// from 0), and the index of VL of (VST $sy, $sz, $sx, $vm, $vl) is 4.  We
+  /// define vector instructions hierarchically, so use VE_VLIndex which is
+  /// defined by the type of instruction and VE_VLWithMask which is defined
+  /// whether the insturction use mask or not.
+  let TSFlags{0}   = VE_Vector;
+  let TSFlags{1}   = VE_VLInUse;
+  let TSFlags{4-2} = !add(VE_VLIndex, VE_VLWithMask);
+
   let DecoderNamespace = "VE";
   field bits<64> SoftFail = 0;
 }
 
 //-----------------------------------------------------------------------------
 // Section 5.1 RM Type
 //
 // RM type has sx, sy, sz, and imm32.

 }
 
 //-----------------------------------------------------------------------------
 // Section 5.5 RW Type
 //-----------------------------------------------------------------------------
 
 //-----------------------------------------------------------------------------
 // Section 5.6 RVM Type
+//
+// RVM type is for vector transfer instructions.
 //-----------------------------------------------------------------------------
 
+class RVM<bits<8>opVal, dag outs, dag ins, string asmstr,
+          list<dag> pattern = []>
+   : InstVE<outs, ins, asmstr, pattern> {
+  bits<1>  cx = 0;
+  bits<1>  vc = 0;
+  bits<1>  cs = 0;
+  bits<4>  m = 0;
+  bits<1>  cy = 1;
+  bits<7>  sy;
+  bits<1>  cz = 1;
+  bits<7>  sz;
+  bits<8>  vx;
+  bits<8>  vy = 0;
+  bits<7>  sw = 0;
+  let op = opVal;
+  let Inst{55} = cx;
+  let Inst{54} = vc;
+  let Inst{53} = cs;
+  let Inst{52} = 0;
+  let Inst{51-48} = m;
+  let Inst{47} = cy;
+  let Inst{46-40} = sy;
+  let Inst{39} = cz;
+  let Inst{38-32} = sz;
+  let Inst{31-24} = vx;
+  let Inst{23-16} = vy;
+  let Inst{15-8} = 0;
+  let Inst{7} = 0;
+  let Inst{6-0} = sw;
+
+  let VE_Vector = 1;
+}
+
 //-----------------------------------------------------------------------------
 // Section 5.7 RV Type
 //-----------------------------------------------------------------------------
 
 // Pseudo instructions.
 class Pseudo<dag outs, dag ins, string asmstr, list<dag> pattern = []>
    : InstVE<outs, ins, asmstr, pattern> {
   let isCodeGenOnly = 1;
   let isPseudo = 1;
 }

llvm/lib/Target/VE/VEInstrInfo.h

 
 #define GET_INSTRINFO_HEADER
 #include "VEGenInstrInfo.inc"
 
 namespace llvm {
 
 class VESubtarget;
 
+/// VEII - This namespace holds all of the Aurora VE target-specific
+/// per-instruction flags.  These must match the corresponding definitions in
+/// VEInstrFormats.td.
+namespace VEII {
+enum {
+  // Aurora VE Instruction Flags.  These flags describe the characteristics of
+  // the Aurora VE instructions for vector handling.
+
+  /// VE_Vector - This instruction is Vector Instruction.
+  VE_Vector = 0x1,
+
+  /// VE_VLInUse - This instruction has a vector register in its operands.
+  VE_VLInUse = 0x2,
+
+  /// VE_VLMask/Shift - This is a bitmask that selects the index number where
+  /// an instruction holds vector length informatio (0 to 6, 7 means undef).n
+  VE_VLShift = 2,
+  VE_VLMask = 0x07 << VE_VLShift,
+};
+
+#define HAS_VLINDEX(TSF) ((TSF)&VEII::VE_VLInUse)
+#define GET_VLINDEX(TSF)                                                       \
+  (HAS_VLINDEX(TSF) ? (int)(((TSF)&VEII::VE_VLMask) >> VEII::VE_VLShift) : -1)
+} // end namespace VEII
+
 class VEInstrInfo : public VEGenInstrInfo {
   const VERegisterInfo RI;
   virtual void anchor();
 
 public:
   explicit VEInstrInfo(VESubtarget &ST);
 
   /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info.  As

llvm/lib/Target/VE/VEInstrInfo.td

def : Pat<(i32 (and i32:$val, 0xff)),
               (ANDrm (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $val, sub_i32),
                      !add(56, 64)), sub_i32)>;
 def : Pat<(i32 (and i32:$val, 0xffff)),
           (EXTRACT_SUBREG
               (ANDrm (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $val, sub_i32),
                      !add(48, 64)), sub_i32)>;
 def : Pat<(i64 (and i64:$val, 0xffffffff)),
           (ANDrm $val, !add(32, 64))>;
+
+// Vector instructions.
+include "VEInstrVec.td"

llvm/lib/Target/VE/VEInstrVec.td

+//===----------------------------------------------------------------------===//
+// Vector Instructions
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//
+// Define all vector instructions defined in SX-Aurora TSUBASA Architecture
+// Guide here.  As those mnemonics, we use mnemonics defined in Vector Engine
+// Assembly Language Reference Manual.
+//
+// Some instructions can update existing data by following instructions
+// sequence.
+//
+//   lea %s0, 256
+//   lea %s1, 128
+//   lvl %s0
+//   vbrd %v0, 2 # v0 = { 2, 2, 2, ..., 2, 2, 2 }
+//   lvl %s1
+//   vbrd %v0, 3 # v0 = { 3, 3, 3, ..., 3, 2, 2, 2, ..., 2, 2, 2 }
+//
+// In order to represent above with a virtual register, we defines instructions
+// with an additional base register and `_v` suffiex in mnemonic.
+//
+//   lea t0, 256
+//   lea t1, 128
+//   lea t0
+//   vbrd tv0, 2
+//   lvl t1
+//   vbrd_v tv1, 2, tv0
+//
+// We also have some instructions uses VL register with an pseudo VL value
+// with following suffixes in mnemonic.
+//
+//   l: have an additional I32 register to represent the VL value.
+//   L: have an additional VL register to represent the VL value.
+//===----------------------------------------------------------------------===//
+
+//-----------------------------------------------------------------------------
+// Section 8.9 - Vector Load/Store and Move Instructions
+//-----------------------------------------------------------------------------
+
+// Multiclass for VLD instructions
+let mayLoad = 1, hasSideEffects = 0, Uses = [VL] in
+multiclass VLDbm<string opcStr, bits<8>opc, RegisterClass RC, dag dag_in,
+                 string disEnc = ""> {
+  let DisableEncoding = disEnc in
+  def "" : RVM<opc, (outs RC:$vx), dag_in,
+               !strconcat(opcStr, " $vx, $sy, $sz")>;
+  let Constraints = "$vx = $base", DisableEncoding = disEnc#"$base",
+      isCodeGenOnly = 1 in
+  def _v : RVM<opc, (outs RC:$vx), !con(dag_in, (ins RC:$base)),
+               !strconcat(opcStr, " $vx, $sy, $sz")>;
+}
+multiclass VLDlm<string opcStr, bits<8>opc, RegisterClass RC, dag dag_in> {
+  defm "" : VLDbm<opcStr, opc, RC, dag_in>;
+  let isCodeGenOnly = 1, VE_VLInUse = 1 in {
+    defm l : VLDbm<opcStr, opc, RC, !con(dag_in, (ins I32:$vl)), "$vl,">;
+    defm L : VLDbm<opcStr, opc, RC, !con(dag_in, (ins VLS:$vl)), "$vl,">;
+  }
+}
+let VE_VLIndex = 3 in
+multiclass VLDtgm<string opcStr, bits<8>opc, RegisterClass RC> {
+  defm rr : VLDlm<opcStr, opc, RC, (ins I64:$sy, I64:$sz)>;
+  let cy = 0 in
+  defm ir : VLDlm<opcStr, opc, RC, (ins simm7:$sy, I64:$sz)>;
+  let cz = 0 in
+  defm rz : VLDlm<opcStr, opc, RC, (ins I64:$sy, zero:$sz)>;
+  let cy = 0, cz = 0 in
+  defm iz : VLDlm<opcStr, opc, RC, (ins simm7:$sy, zero:$sz)>;
+}
+multiclass VLDm<string opcStr, bits<8>opc, RegisterClass RC> {
+  let vc = 1 in defm "" : VLDtgm<opcStr, opc, RC>;
+  let vc = 0 in defm NC : VLDtgm<opcStr#".nc", opc, RC>;
+}
+
+// Section 8.9.1 - VLD (Vector Load)
+defm VLD : VLDm<"vld", 0x81, V64>;
+
+// Section 8.9.2 - VLDU (Vector Load Upper)
+defm VLDU : VLDm<"vldu", 0x82, V64>;
+
+// Section 8.9.3 - VLDL (Vector Load Lower)
+defm VLDLSX : VLDm<"vldl.sx", 0x83, V64>;
+let cx = 1 in defm VLDLZX : VLDm<"vldl.zx", 0x83, V64>;
+
+// Section 8.9.4 - VLD2D (Vector Load 2D)
+defm VLD2D : VLDm<"vld2d", 0xc1, V64>;
+
+// Section 8.9.5 - VLDU2D (Vector Load Upper 2D)
+defm VLDU2D : VLDm<"vldu2d", 0xc2, V64>;
+
+// Section 8.9.6 - VLDL2D (Vector Load Lower 2D)
+defm VLDL2DSX : VLDm<"vldl2d.sx", 0xc3, V64>;
+let cx = 1 in defm VLDL2DZX : VLDm<"vldl2d.zx", 0xc3, V64>;
+
+// Multiclass for VST instructions
+let mayStore = 1, hasSideEffects = 0, Uses = [VL] in
+multiclass VSTbm<string opcStr, string argStr, bits<8>opc, dag dag_in> {
+  def "" : RVM<opc, (outs), dag_in, !strconcat(opcStr, argStr)>;
+  let DisableEncoding = "$vl", isCodeGenOnly = 1, VE_VLInUse = 1 in {
+    def l : RVM<opc, (outs), !con(dag_in, (ins I32:$vl)),
+                !strconcat(opcStr, argStr)>;
+    def L : RVM<opc, (outs), !con(dag_in, (ins VLS:$vl)),
+                !strconcat(opcStr, argStr)>;
+  }
+}
+multiclass VSTmm<string opcStr, bits<8>opc, dag dag_in> {
+  defm "" : VSTbm<opcStr, " $vx, $sy, $sz", opc, dag_in>;
+  let m = ?, VE_VLWithMask = 1 in
+  defm m : VSTbm<opcStr, " $vx, $sy, $sz, $m", opc, !con(dag_in, (ins VM:$m))>;
+}
+let VE_VLIndex = 3 in
+multiclass VSTtgm<string opcStr, bits<8>opc, RegisterClass RC> {
+  defm rrv : VSTmm<opcStr, opc, (ins I64:$sy, I64:$sz, RC:$vx)>;
+  let cy = 0 in
+  defm irv : VSTmm<opcStr, opc, (ins simm7:$sy, I64:$sz, RC:$vx)>;
+  let cz = 0 in
+  defm rzv : VSTmm<opcStr, opc, (ins I64:$sy, zero:$sz, RC:$vx)>;
+  let cy = 0, cz = 0 in
+  defm izv : VSTmm<opcStr, opc, (ins simm7:$sy, zero:$sz, RC:$vx)>;
+}
+multiclass VSTm<string opcStr, bits<8>opc, RegisterClass RC> {
+  let vc = 1, cx = 0 in defm "" : VSTtgm<opcStr, opc, RC>;
+  let vc = 0, cx = 0 in defm NC : VSTtgm<opcStr#".nc", opc, RC>;
+  let vc = 1, cx = 1 in defm OT : VSTtgm<opcStr#".ot", opc, RC>;
+  let vc = 0, cx = 1 in defm NCOT : VSTtgm<opcStr#".nc.ot", opc, RC>;
+}
+
+// Section 8.9.7 - VST (Vector Store)
+defm VST : VSTm<"vst", 0x91, V64>;
+
+// Section 8.9.8 - VST (Vector Store Upper)
+defm VSTU : VSTm<"vstu", 0x92, V64>;
+
+// Section 8.9.9 - VSTL (Vector Store Lower)
+defm VSTL : VSTm<"vstl", 0x93, V64>;
+
+// Section 8.9.10 - VST2D (Vector Store 2D)
+defm VST2D : VSTm<"vst2d", 0xd1, V64>;
+
+// Section 8.9.11 - VSTU2D (Vector Store Upper 2D)
+defm VSTU2D : VSTm<"vstu2d", 0xd2, V64>;
+
+// Section 8.9.12 - VSTL2D (Vector Store Lower 2D)
+defm VSTL2D : VSTm<"vstl2d", 0xd3, V64>;

llvm/lib/Target/VE/VERegisterInfo.td

 }
 
 class VEMiscReg<bits<6> enc, string n>: Register<n> {
   let HWEncoding{15-6} = 0;
   let HWEncoding{5-0} = enc;
   let Namespace = "VE";
 }
 
+class VEVecReg<bits<8> enc, string n, list<Register> subregs = [],
+               list<string> altNames = [], list<Register> aliases = []>
+        : Register<n, altNames> {
+  let HWEncoding{15-8} = 0;
+  let HWEncoding{7-0} = enc;
+  let Namespace = "VE";
+  let SubRegs = subregs;
+  let Aliases = aliases;
+}
+
+class VEMaskReg<bits<4> enc, string n, list<Register> subregs = [],
+                list<string> altNames = [], list<Register> aliases = []>
+        : Register<n, altNames> {
+  let HWEncoding{15-4} = 0;
+  let HWEncoding{3-0} = enc;
+  let Namespace = "VE";
+  let SubRegs = subregs;
+  let Aliases = aliases;
+}
+
 let Namespace = "VE" in {
   def sub_i32     : SubRegIndex<32, 32>;        // Low 32 bit (32..63)
   def sub_f32     : SubRegIndex<32>;            // High 32 bit (0..31)
   def sub_even    : SubRegIndex<64>;            // High 64 bit (0..63)
   def sub_odd     : SubRegIndex<64, 64>;        // Low 64 bit (64..127)
+  def sub_vm_even : SubRegIndex<256>;           // High 256 bit (0..255)
+  def sub_vm_odd  : SubRegIndex<256, 256>;      // Low 256 bit (256..511)
   def AsmName     : RegAltNameIndex;
 }
 
 //-----------------------------------------------------------------------------
 // Miscellaneous Registers
 //-----------------------------------------------------------------------------
 
 def USRCC : VEMiscReg<0, "usrcc">;      // User clock counter

 
 //-----------------------------------------------------------------------------
 // Instruction Counter Register
 //-----------------------------------------------------------------------------
 
 def IC : VEMiscReg<62, "ic">;
 
 //-----------------------------------------------------------------------------
+// Vector Length Register
+//-----------------------------------------------------------------------------
+
+def VL : VEMiscReg<63, "vl">;
+
+// Register classes.
+def VLS : RegisterClass<"VE", [i32], 64, (add VL)>;
+
+//-----------------------------------------------------------------------------
 // Generic Registers
 //-----------------------------------------------------------------------------
 
 let RegAltNameIndices = [AsmName] in {
 
 // Generic integer registers - 32 bits wide
 foreach I = 0-63 in
   def SW#I : VEReg<I, "sw"#I, [], ["s"#I]>, DwarfRegNum<[I]>;

 //   ...
 let SubRegIndices = [sub_even, sub_odd], CoveredBySubRegs = 1 in
 foreach I = 0-31 in
   def Q#I : VEReg<!shl(I,1), "q"#I,
                   [!cast<VEReg>("SX"#!shl(I,1)),
                    !cast<VEReg>("SX"#!add(!shl(I,1),1))],
                   ["s"#!shl(I,1)]>;
 
+// Vector registers - 64 bits wide 256 elements
+foreach I = 0-63 in
+  def V#I : VEVecReg<I, "v"#I, [], ["v"#I]>, DwarfRegNum<[!add(64,I)]>;
+
+// Vector Index Register
+def VIX : VEVecReg<255, "vix", [], ["vix"]>;
+
+// Vector mask registers - 256 bits wide
+foreach I = 0-15 in
+  def VM#I : VEMaskReg<I, "vm"#I, [], ["vm"#I]>, DwarfRegNum<[!add(128,I)]>;
+
+// Aliases of VMs to use as a pair of two VM for packed instructions
+let SubRegIndices = [sub_vm_even, sub_vm_odd], CoveredBySubRegs = 1 in
+foreach I = 0-7 in
+  def VMP#I : VEMaskReg<!shl(I,1), "vmp"#I,
+                        [!cast<VEMaskReg>("VM"#!shl(I,1)),
+                         !cast<VEMaskReg>("VM"#!add(!shl(I,1),1))],
+                        ["vm"#!shl(I,1)]>;
+
 } // RegAltNameIndices = [AsmName]
 
 // Register classes.
 //
 // The register order is defined in terms of the preferred
 // allocation order.
 def I32 : RegisterClass<"VE", [i32], 32,
                         (add (sequence "SW%u", 0, 7),
                              (sequence "SW%u", 34, 63),
                              (sequence "SW%u", 8, 33))>;
 def I64 : RegisterClass<"VE", [i64, f64], 64,
                         (add (sequence "SX%u", 0, 7),
                              (sequence "SX%u", 34, 63),
                              (sequence "SX%u", 8, 33))>;
 def F32 : RegisterClass<"VE", [f32], 32,
                         (add (sequence "SF%u", 0, 7),
                              (sequence "SF%u", 34, 63),
                              (sequence "SF%u", 8, 33))>;
 def F128 : RegisterClass<"VE", [f128], 128,
                         (add (sequence "Q%u", 0, 3),
                              (sequence "Q%u", 17, 31),
                              (sequence "Q%u", 4, 16))>;
+
+def V64 : RegisterClass<"VE",
+                        [v512i32, v512f32,
+                         v256i64, v256i32, v256f32, v256f64,
+                         v128i64, v128i32, v128f32, v128f64,
+                         v64i64,  v64i32,  v64f32,  v64f64,
+                         v32i64,  v32i32,  v32f32,  v32f64,
+                         v16i64,  v16i32,  v16f32,  v16f64,
+                         v8i64,   v8i32,   v8f32,   v8f64,
+                         v4i64,   v4i32,   v4f32,   v4f64,
+                         v2i64,   v2i32,   v2f32,   v2f64], 64,
+                        (add (sequence "V%u", 0, 63),
+                             VIX)>;
+
+// vm0 is reserved for always true
+def VM : RegisterClass<"VE", [v256i1], 64, (sequence "VM%u", 0, 15)>;
+def VM512 : RegisterClass<"VE", [v512i1], 64, (sequence "VMP%u", 0, 7)>;

llvm/test/MC/VE/VLD.s

+# RUN: llvm-mc -triple=ve --show-encoding < %s \
+# RUN:     | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: llvm-mc -triple=ve -filetype=obj < %s | llvm-objdump -d - \
+# RUN:     | FileCheck %s --check-prefixes=CHECK-INST
+
+# CHECK-INST: vld %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0x81]
+vld %v11, 23, %s12
+
+# CHECK-INST: vld.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0x81]
+vld.nc %vix, 63, %s22
+
+# CHECK-INST: vldu %v63, -64, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0x40,0x82]
+vldu %v63, -64, %s63
+
+# CHECK-INST: vldu.nc %v12, %s12, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x00,0x82]
+vldu.nc %v12, %s12, 0
+
+# CHECK-INST: vldl.sx %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0x83]
+vldl.sx %v11, 23, %s12
+
+# CHECK-INST: vldl.sx.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0x83]
+vldl.sx.nc %vix, 63, %s22
+
+# CHECK-INST: vldl.zx %v63, -64, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0x00,0x40,0xc0,0x83]
+vldl.zx %v63, -64, 0
+
+# CHECK-INST: vldl.zx.nc %v12, %s12, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0xbf,0x8c,0x80,0x83]
+vldl.zx.nc %v12, %s12, %s63
+
+# CHECK-INST: vld2d %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0xc1]
+vld2d %v11, 23, %s12
+
+# CHECK-INST: vld2d.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0xc1]
+vld2d.nc %vix, 63, %s22
+
+# CHECK-INST: vldu2d %v63, -64, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0x40,0xc2]
+vldu2d %v63, -64, %s63
+
+# CHECK-INST: vldu2d.nc %v12, %s12, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x00,0xc2]
+vldu2d.nc %v12, %s12, 0
+
+# CHECK-INST: vldl2d.sx %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0xc3]
+vldl2d.sx %v11, 23, %s12
+
+# CHECK-INST: vldl2d.sx.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0xc3]
+vldl2d.sx.nc %vix, 63, %s22
+
+# CHECK-INST: vldl2d.zx %v63, -64, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0x00,0x40,0xc0,0xc3]
+vldl2d.zx %v63, -64, 0
+
+# CHECK-INST: vldl2d.zx.nc %v12, %s12, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0xbf,0x8c,0x80,0xc3]
+vldl2d.zx.nc %v12, %s12, %s63

llvm/test/MC/VE/VST.s

+# RUN: llvm-mc -triple=ve --show-encoding < %s \
+# RUN:     | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: llvm-mc -triple=ve -filetype=obj < %s | llvm-objdump -d - \
+# RUN:     | FileCheck %s --check-prefixes=CHECK-INST
+
+# CHECK-INST: vst %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0x91]
+vst %v11, 23, %s12
+
+# CHECK-INST: vst.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0x91]
+vst.nc %vix, 63, %s22
+
+# CHECK-INST: vst.ot %v63, -64, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xc0,0x91]
+vst.ot %v63, -64, %s63
+
+# CHECK-INST: vst.nc.ot %v12, %s12, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x80,0x91]
+vst.nc.ot %v12, %s12, 0
+
+# CHECK-INST: vst %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0x91]
+vst %v11, 23, %s12, %vm0
+
+# CHECK-INST: vst.nc %vix, 63, %s22, %vm1
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x01,0x91]
+vst.nc %vix, 63, %s22, %vm1
+
+# CHECK-INST: vst.ot %v63, -64, %s63, %vm15
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xcf,0x91]
+vst.ot %v63, -64, %s63, %vm15
+
+# CHECK-INST: vst.nc.ot %v12, %s12, 0, %vm8
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x88,0x91]
+vst.nc.ot %v12, %s12, 0, %vm8
+
+# CHECK-INST: vstu %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0x92]
+vstu %v11, 23, %s12
+
+# CHECK-INST: vstu.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0x92]
+vstu.nc %vix, 63, %s22
+
+# CHECK-INST: vstu.ot %v63, -64, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xc0,0x92]
+vstu.ot %v63, -64, %s63
+
+# CHECK-INST: vstu.nc.ot %v12, %s12, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x80,0x92]
+vstu.nc.ot %v12, %s12, 0
+
+# CHECK-INST: vstu %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0x92]
+vstu %v11, 23, %s12, %vm0
+
+# CHECK-INST: vstu.nc %vix, 63, %s22, %vm1
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x01,0x92]
+vstu.nc %vix, 63, %s22, %vm1
+
+# CHECK-INST: vstu.ot %v63, -64, %s63, %vm15
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xcf,0x92]
+vstu.ot %v63, -64, %s63, %vm15
+
+# CHECK-INST: vstu.nc.ot %v12, %s12, 0, %vm8
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x88,0x92]
+vstu.nc.ot %v12, %s12, 0, %vm8
+
+# CHECK-INST: vstl %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0x93]
+vstl %v11, 23, %s12
+
+# CHECK-INST: vstl.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0x93]
+vstl.nc %vix, 63, %s22
+
+# CHECK-INST: vstl.ot %v63, -64, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xc0,0x93]
+vstl.ot %v63, -64, %s63
+
+# CHECK-INST: vstl.nc.ot %v12, %s12, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x80,0x93]
+vstl.nc.ot %v12, %s12, 0
+
+# CHECK-INST: vstl %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0x93]
+vstl %v11, 23, %s12, %vm0
+
+# CHECK-INST: vstl.nc %vix, 63, %s22, %vm1
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x01,0x93]
+vstl.nc %vix, 63, %s22, %vm1
+
+# CHECK-INST: vstl.ot %v63, -64, %s63, %vm15
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xcf,0x93]
+vstl.ot %v63, -64, %s63, %vm15
+
+# CHECK-INST: vstl.nc.ot %v12, %s12, 0, %vm8
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x88,0x93]
+vstl.nc.ot %v12, %s12, 0, %vm8
+
+# CHECK-INST: vst2d %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0xd1]
+vst2d %v11, 23, %s12
+
+# CHECK-INST: vst2d.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0xd1]
+vst2d.nc %vix, 63, %s22
+
+# CHECK-INST: vst2d.ot %v63, -64, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xc0,0xd1]
+vst2d.ot %v63, -64, %s63
+
+# CHECK-INST: vst2d.nc.ot %v12, %s12, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x80,0xd1]
+vst2d.nc.ot %v12, %s12, 0
+
+# CHECK-INST: vst2d %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0xd1]
+vst2d %v11, 23, %s12, %vm0
+
+# CHECK-INST: vst2d.nc %vix, 63, %s22, %vm1
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x01,0xd1]
+vst2d.nc %vix, 63, %s22, %vm1
+
+# CHECK-INST: vst2d.ot %v63, -64, %s63, %vm15
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xcf,0xd1]
+vst2d.ot %v63, -64, %s63, %vm15
+
+# CHECK-INST: vst2d.nc.ot %v12, %s12, 0, %vm8
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x88,0xd1]
+vst2d.nc.ot %v12, %s12, 0, %vm8
+
+# CHECK-INST: vstu2d %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0xd2]
+vstu2d %v11, 23, %s12
+
+# CHECK-INST: vstu2d.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0xd2]
+vstu2d.nc %vix, 63, %s22
+
+# CHECK-INST: vstu2d.ot %v63, -64, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xc0,0xd2]
+vstu2d.ot %v63, -64, %s63
+
+# CHECK-INST: vstu2d.nc.ot %v12, %s12, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x80,0xd2]
+vstu2d.nc.ot %v12, %s12, 0
+
+# CHECK-INST: vstu2d %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0xd2]
+vstu2d %v11, 23, %s12, %vm0
+
+# CHECK-INST: vstu2d.nc %vix, 63, %s22, %vm1
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x01,0xd2]
+vstu2d.nc %vix, 63, %s22, %vm1
+
+# CHECK-INST: vstu2d.ot %v63, -64, %s63, %vm15
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xcf,0xd2]
+vstu2d.ot %v63, -64, %s63, %vm15
+
+# CHECK-INST: vstu2d.nc.ot %v12, %s12, 0, %vm8
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x88,0xd2]
+vstu2d.nc.ot %v12, %s12, 0, %vm8
+
+# CHECK-INST: vstl2d %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0xd3]
+vstl2d %v11, 23, %s12
+
+# CHECK-INST: vstl2d.nc %vix, 63, %s22
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x00,0xd3]
+vstl2d.nc %vix, 63, %s22
+
+# CHECK-INST: vstl2d.ot %v63, -64, %s63
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xc0,0xd3]
+vstl2d.ot %v63, -64, %s63
+
+# CHECK-INST: vstl2d.nc.ot %v12, %s12, 0
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x80,0xd3]
+vstl2d.nc.ot %v12, %s12, 0
+
+# CHECK-INST: vstl2d %v11, 23, %s12
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0b,0x8c,0x17,0x40,0xd3]
+vstl2d %v11, 23, %s12, %vm0
+
+# CHECK-INST: vstl2d.nc %vix, 63, %s22, %vm1
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0xff,0x96,0x3f,0x01,0xd3]
+vstl2d.nc %vix, 63, %s22, %vm1
+
+# CHECK-INST: vstl2d.ot %v63, -64, %s63, %vm15
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x3f,0xbf,0x40,0xcf,0xd3]
+vstl2d.ot %v63, -64, %s63, %vm15
+
+# CHECK-INST: vstl2d.nc.ot %v12, %s12, 0, %vm8
+# CHECK-ENCODING: encoding: [0x00,0x00,0x00,0x0c,0x00,0x8c,0x88,0xd3]
+vstl2d.nc.ot %v12, %s12, 0, %vm8