[VE] Add logical mask intrinsic instructions

[kaz7]

Summary

Add andm, orm, xorm, eqvm, nndm, negm, pcvm, lzvm, and tovm intrinsic
instructions, a few pseudo instructions to expand logical intrinsic
using VM512, a mechnism to expand such pseudo instructions, and
regression tests. Also, assign vector mask types and vector mask
register classes correctly. This is required to use VM512 registers
as function arguments.

[simoll]

Just this one thing, else, LGTM.

llvm/lib/Target/VE/VEISelLowering.cpp

87–89

Can we make this a separate patch?

[kaz7]

Hmm, not sure why it needs. These new tests shows the correctness of code modifications enough, IMO.

llvm/lib/Target/VE/VEISelLowering.cpp

87–89

I condider that, but it's difficult to do so. In order to do so, a regression test is required. Making such a regression test requires additional implementation of vector mask register copy function. I'm not sure what is required to implement that.

Probablly, I should ask them when you add above code. I asked tests for V64 registers but forgot to ask tests of VM registers.

Anywa, this time I add new intrinsic instructions supporting vm512 and hit this bug. You can see crashes by disabling above modifications and run new regression tests that this patch adds. Then, you can see no crash after enabling above modifications and run the identical tests again. Isn't that enough?

[kaz7]

Update patch following clang-tidy suggestions.

[simoll]

llvm/lib/Target/VE/VEISelLowering.cpp

87–89

You can fix this in a separate patch without additional tests since it is obviously incorrect. I am asking you to do this because this patch adds new functionality first - we shouldn't bundle it up with bug fixes.

[kaz7]

Separate this bug fixes to https://reviews.llvm.org/D93212.

llvm/lib/Target/VE/VEISelLowering.cpp

87–89

Ok. If you don't require additional test for bug fix, I can do it.

[kaz7]

Split a part of this patch into https://reviews.llvm.org/D93212.

llvm/include/llvm/IR/IntrinsicsVEVL.gen.td

 let TargetPrefix = "ve" in def int_ve_vl_vscl_vvssl : GCCBuiltin<"__builtin_ve_vl_vscl_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vscl_vvssml : GCCBuiltin<"__builtin_ve_vl_vscl_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsclnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsclnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclnc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsclot_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsclot_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsclncot_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsclncot_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_andm_mmm : GCCBuiltin<"__builtin_ve_vl_andm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_andm_MMM : GCCBuiltin<"__builtin_ve_vl_andm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_orm_mmm : GCCBuiltin<"__builtin_ve_vl_orm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_orm_MMM : GCCBuiltin<"__builtin_ve_vl_orm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_xorm_mmm : GCCBuiltin<"__builtin_ve_vl_xorm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_xorm_MMM : GCCBuiltin<"__builtin_ve_vl_xorm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_eqvm_mmm : GCCBuiltin<"__builtin_ve_vl_eqvm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_eqvm_MMM : GCCBuiltin<"__builtin_ve_vl_eqvm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_nndm_mmm : GCCBuiltin<"__builtin_ve_vl_nndm_mmm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>, LLVMType<v256i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_nndm_MMM : GCCBuiltin<"__builtin_ve_vl_nndm_MMM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>, LLVMType<v512i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_negm_mm : GCCBuiltin<"__builtin_ve_vl_negm_mm">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_negm_MM : GCCBuiltin<"__builtin_ve_vl_negm_MM">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v512i1>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pcvm_sml : GCCBuiltin<"__builtin_ve_vl_pcvm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_lzvm_sml : GCCBuiltin<"__builtin_ve_vl_lzvm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_tovm_sml : GCCBuiltin<"__builtin_ve_vl_tovm_sml">, Intrinsic<[LLVMType<i64>], [LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;

llvm/lib/Target/VE/VEISelLowering.cpp

bool VETargetLowering::CanLowerReturn(
   SmallVector<CCValAssign, 16> RVLocs;
   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
   return CCInfo.CheckReturn(Outs, RetCC);
 }
 
 static const MVT AllVectorVTs[] = {MVT::v256i32, MVT::v512i32, MVT::v256i64,
                                    MVT::v256f32, MVT::v512f32, MVT::v256f64};
 
-static const MVT AllMaskVTs[] = {MVT::v256i1, MVT::v512i1};
-
 void VETargetLowering::initRegisterClasses() {
   // Set up the register classes.
   addRegisterClass(MVT::i32, &VE::I32RegClass);
   addRegisterClass(MVT::i64, &VE::I64RegClass);
   addRegisterClass(MVT::f32, &VE::F32RegClass);
   addRegisterClass(MVT::f64, &VE::I64RegClass);
   addRegisterClass(MVT::f128, &VE::F128RegClass);
 
   if (Subtarget->enableVPU()) {
     for (MVT VecVT : AllVectorVTs)
       addRegisterClass(VecVT, &VE::V64RegClass);
-    for (MVT MaskVT : AllMaskVTs)
-      addRegisterClass(MaskVT, &VE::VMRegClass);
+    addRegisterClass(MVT::v256i1, &VE::VMRegClass);
+    addRegisterClass(MVT::v512i1, &VE::VM512RegClass);
   }
 }
 
 void VETargetLowering::initSPUActions() {
   const auto &TM = getTargetMachine();
   /// Load & Store {
 
   // VE doesn't have i1 sign extending load.
   for (MVT VT : MVT::integer_valuetypes()) {

llvm/lib/Target/VE/VEInstrInfo.cpp

 }
 
 static Register getVM512Upper(Register reg) {
   return (reg - VE::VMP0) * 2 + VE::VM0;
 }
 
 static Register getVM512Lower(Register reg) { return getVM512Upper(reg) + 1; }
 
+// Expand pseudo logical vector instructions for VM512 registers.
+static void expandPseudoLogM(MachineInstr &MI, const MCInstrDesc &MCID) {
+  MachineBasicBlock *MBB = MI.getParent();
+  DebugLoc dl = MI.getDebugLoc();
+
+  unsigned VMXu = getVM512Upper(MI.getOperand(0).getReg());
+  unsigned VMXl = getVM512Lower(MI.getOperand(0).getReg());
+  unsigned VMYu = getVM512Upper(MI.getOperand(1).getReg());
+  unsigned VMYl = getVM512Lower(MI.getOperand(1).getReg());
+
+  switch (MI.getOpcode()) {
+  default: {
+    unsigned VMZu = getVM512Upper(MI.getOperand(2).getReg());
+    unsigned VMZl = getVM512Lower(MI.getOperand(2).getReg());
+    BuildMI(*MBB, MI, dl, MCID).addDef(VMXu).addUse(VMYu).addUse(VMZu);
+    BuildMI(*MBB, MI, dl, MCID).addDef(VMXl).addUse(VMYl).addUse(VMZl);
+    break;
+  }
+  case VE::NEGMy:
+    BuildMI(*MBB, MI, dl, MCID).addDef(VMXu).addUse(VMYu);
+    BuildMI(*MBB, MI, dl, MCID).addDef(VMXl).addUse(VMYl);
+    break;
+  }
+  MI.eraseFromParent();
+}
+
 static void addOperandsForVFMK(MachineInstrBuilder &MIB, MachineInstr &MI,
                                bool Upper) {
   // VM512
   MIB.addReg(Upper ? getVM512Upper(MI.getOperand(0).getReg())
                    : getVM512Lower(MI.getOperand(0).getReg()));
 
   switch (MI.getNumExplicitOperands()) {
   default:
bool VEInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   case VE::EXTEND_STACK_GUARD: {
     MI.eraseFromParent(); // The pseudo instruction is gone now.
     return true;
   }
   case VE::GETSTACKTOP: {
     return expandGetStackTopPseudo(MI);
   }
 
+  case VE::ANDMyy:
+    expandPseudoLogM(MI, get(VE::ANDMmm));
+    return true;
+  case VE::ORMyy:
+    expandPseudoLogM(MI, get(VE::ORMmm));
+    return true;
+  case VE::XORMyy:
+    expandPseudoLogM(MI, get(VE::XORMmm));
+    return true;
+  case VE::EQVMyy:
+    expandPseudoLogM(MI, get(VE::EQVMmm));
+    return true;
+  case VE::NNDMyy:
+    expandPseudoLogM(MI, get(VE::NNDMmm));
+    return true;
+  case VE::NEGMy:
+    expandPseudoLogM(MI, get(VE::NEGMm));
+    return true;
+
   case VE::LVMyir:
   case VE::LVMyim:
   case VE::LVMyir_y:
   case VE::LVMyim_y: {
     Register VMXu = getVM512Upper(MI.getOperand(0).getReg());
     Register VMXl = getVM512Lower(MI.getOperand(0).getReg());
     int64_t Imm = MI.getOperand(1).getImm();
     bool IsSrcReg =

llvm/lib/Target/VE/VEInstrIntrinsicVL.gen.td

 def : Pat<(int_ve_vl_vsclncot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCLNCOTvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
 def : Pat<(int_ve_vl_vsclncot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCLNCOTvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
 def : Pat<(int_ve_vl_vsclncot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCLNCOTvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
 def : Pat<(int_ve_vl_vsclncot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCLNCOTvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
 def : Pat<(int_ve_vl_vsclncot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCLNCOTvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
 def : Pat<(int_ve_vl_vsclncot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCLNCOTvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
 def : Pat<(int_ve_vl_vsclncot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCLNCOTvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
 def : Pat<(int_ve_vl_vsclncot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCLNCOTvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_andm_mmm v256i1:$vmy, v256i1:$vmz), (ANDMmm v256i1:$vmy, v256i1:$vmz)>;
+def : Pat<(int_ve_vl_andm_MMM v512i1:$vmy, v512i1:$vmz), (ANDMyy v512i1:$vmy, v512i1:$vmz)>;
+def : Pat<(int_ve_vl_orm_mmm v256i1:$vmy, v256i1:$vmz), (ORMmm v256i1:$vmy, v256i1:$vmz)>;
+def : Pat<(int_ve_vl_orm_MMM v512i1:$vmy, v512i1:$vmz), (ORMyy v512i1:$vmy, v512i1:$vmz)>;
+def : Pat<(int_ve_vl_xorm_mmm v256i1:$vmy, v256i1:$vmz), (XORMmm v256i1:$vmy, v256i1:$vmz)>;
+def : Pat<(int_ve_vl_xorm_MMM v512i1:$vmy, v512i1:$vmz), (XORMyy v512i1:$vmy, v512i1:$vmz)>;
+def : Pat<(int_ve_vl_eqvm_mmm v256i1:$vmy, v256i1:$vmz), (EQVMmm v256i1:$vmy, v256i1:$vmz)>;
+def : Pat<(int_ve_vl_eqvm_MMM v512i1:$vmy, v512i1:$vmz), (EQVMyy v512i1:$vmy, v512i1:$vmz)>;
+def : Pat<(int_ve_vl_nndm_mmm v256i1:$vmy, v256i1:$vmz), (NNDMmm v256i1:$vmy, v256i1:$vmz)>;
+def : Pat<(int_ve_vl_nndm_MMM v512i1:$vmy, v512i1:$vmz), (NNDMyy v512i1:$vmy, v512i1:$vmz)>;
+def : Pat<(int_ve_vl_negm_mm v256i1:$vmy), (NEGMm v256i1:$vmy)>;
+def : Pat<(int_ve_vl_negm_MM v512i1:$vmy), (NEGMy v512i1:$vmy)>;
+def : Pat<(int_ve_vl_pcvm_sml v256i1:$vmy, i32:$vl), (PCVMml v256i1:$vmy, i32:$vl)>;
+def : Pat<(int_ve_vl_lzvm_sml v256i1:$vmy, i32:$vl), (LZVMml v256i1:$vmy, i32:$vl)>;
+def : Pat<(int_ve_vl_tovm_sml v256i1:$vmy, i32:$vl), (TOVMml v256i1:$vmy, i32:$vl)>;

llvm/lib/Target/VE/VEInstrVec.td

let hasSideEffects = 0, isCodeGenOnly = 1, DisableEncoding = "$vl" in {
   def VFMKSyvl  : Pseudo<(outs VM512:$vmx),
                          (ins CCOp:$cf, V64:$vz, I32:$vl),
                          "# pseudo-vfmk.s.$cf $vmx, $vz">;
   def VFMKSyvyl : Pseudo<(outs VM512:$vmx),
                          (ins CCOp:$cf, V64:$vz, VM512:$vm, I32:$vl),
                          "# pseudo-vfmk.s.$cf $vmx, $vz, $vm">;
 }
 
+// ANDM/ORM/XORM/EQVM/NNDM/NEGM instructions using VM512
+let hasSideEffects = 0, isCodeGenOnly = 1 in {
+  def ANDMyy : Pseudo<(outs VM512:$vmx), (ins VM512:$vmy, VM512:$vmz),
+                      "# andm $vmx, $vmy, $vmz">;
+  def ORMyy : Pseudo<(outs VM512:$vmx), (ins VM512:$vmy, VM512:$vmz),
+                     "# orm $vmx, $vmy, $vmz">;
+  def XORMyy : Pseudo<(outs VM512:$vmx), (ins VM512:$vmy, VM512:$vmz),
+                      "# xorm $vmx, $vmy, $vmz">;
+  def EQVMyy : Pseudo<(outs VM512:$vmx), (ins VM512:$vmy, VM512:$vmz),
+                      "# eqvm $vmx, $vmy, $vmz">;
+  def NNDMyy : Pseudo<(outs VM512:$vmx), (ins VM512:$vmy, VM512:$vmz),
+                      "# nndm $vmx, $vmy, $vmz">;
+  def NEGMy : Pseudo<(outs VM512:$vmx), (ins VM512:$vmy),
+                     "# negm $vmx, $vmy">;
+}
+
 //===----------------------------------------------------------------------===//
 // Instructions
 //
 // Define all vector instructions defined in SX-Aurora TSUBASA Architecture
 // Guide here.  As those mnemonics, we use mnemonics defined in Vector Engine
 // Assembly Language Reference Manual.
 //
 // Some instructions can update existing data by following instructions

llvm/test/CodeGen/VE/VELIntrinsics/andm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test and vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test ANDM*mm and ANDM*yy instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @andm_mmm(<256 x i1> %0, <256 x i1> %1) {
+; CHECK-LABEL: andm_mmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andm %vm1, %vm1, %vm2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.andm.mmm(<256 x i1> %0, <256 x i1> %1)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.andm.mmm(<256 x i1>, <256 x i1>)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @andm_MMM(<512 x i1> %0, <512 x i1> %1) {
+; CHECK-LABEL: andm_MMM:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andm %vm2, %vm2, %vm4
+; CHECK-NEXT:    andm %vm3, %vm3, %vm5
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.andm.MMM(<512 x i1> %0, <512 x i1> %1)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.andm.MMM(<512 x i1>, <512 x i1>)

llvm/test/CodeGen/VE/VELIntrinsics/eqvm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test equivalence vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test EQVM*mm and EQVM*yy instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @eqvm_mmm(<256 x i1> %0, <256 x i1> %1) {
+; CHECK-LABEL: eqvm_mmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    eqvm %vm1, %vm1, %vm2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.eqvm.mmm(<256 x i1> %0, <256 x i1> %1)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.eqvm.mmm(<256 x i1>, <256 x i1>)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @eqvm_MMM(<512 x i1> %0, <512 x i1> %1) {
+; CHECK-LABEL: eqvm_MMM:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    eqvm %vm2, %vm2, %vm4
+; CHECK-NEXT:    eqvm %vm3, %vm3, %vm5
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.eqvm.MMM(<512 x i1> %0, <512 x i1> %1)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.eqvm.MMM(<512 x i1>, <512 x i1>)

llvm/test/CodeGen/VE/VELIntrinsics/lzvm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test leading zero of vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test LZVM*ml instruction.
+
+; Function Attrs: nounwind readnone
+define fastcc i64 @lzvm_sml(<256 x i1> %0) {
+; CHECK-LABEL: lzvm_sml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    lzvm %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call i64 @llvm.ve.vl.lzvm.sml(<256 x i1> %0, i32 256)
+  ret i64 %2
+}
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.ve.vl.lzvm.sml(<256 x i1>, i32)

llvm/test/CodeGen/VE/VELIntrinsics/negm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test negate vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test NEGM*m and NEGM*y instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @negm_mm(<256 x i1> %0) {
+; CHECK-LABEL: negm_mm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    negm %vm1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.negm.mm(<256 x i1> %0)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.negm.mm(<256 x i1>)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @negm_MM(<512 x i1> %0) {
+; CHECK-LABEL: negm_MM:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    negm %vm2, %vm2
+; CHECK-NEXT:    negm %vm3, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.negm.MM(<512 x i1> %0)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.negm.MM(<512 x i1>)

llvm/test/CodeGen/VE/VELIntrinsics/nndm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test negate and vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test NNDM*mm and NNDM*yy instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @nndm_mmm(<256 x i1> %0, <256 x i1> %1) {
+; CHECK-LABEL: nndm_mmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nndm %vm1, %vm1, %vm2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.nndm.mmm(<256 x i1> %0, <256 x i1> %1)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.nndm.mmm(<256 x i1>, <256 x i1>)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @nndm_MMM(<512 x i1> %0, <512 x i1> %1) {
+; CHECK-LABEL: nndm_MMM:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nndm %vm2, %vm2, %vm4
+; CHECK-NEXT:    nndm %vm3, %vm3, %vm5
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.nndm.MMM(<512 x i1> %0, <512 x i1> %1)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.nndm.MMM(<512 x i1>, <512 x i1>)

llvm/test/CodeGen/VE/VELIntrinsics/orm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test or vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test ORM*mm and ORM*yy instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @orm_mmm(<256 x i1> %0, <256 x i1> %1) {
+; CHECK-LABEL: orm_mmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orm %vm1, %vm1, %vm2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.orm.mmm(<256 x i1> %0, <256 x i1> %1)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.orm.mmm(<256 x i1>, <256 x i1>)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @orm_MMM(<512 x i1> %0, <512 x i1> %1) {
+; CHECK-LABEL: orm_MMM:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orm %vm2, %vm2, %vm4
+; CHECK-NEXT:    orm %vm3, %vm3, %vm5
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.orm.MMM(<512 x i1> %0, <512 x i1> %1)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.orm.MMM(<512 x i1>, <512 x i1>)

llvm/test/CodeGen/VE/VELIntrinsics/pcvm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test population count of vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test PCVM*ml instruction.
+
+; Function Attrs: nounwind readnone
+define fastcc i64 @pcvm_sml(<256 x i1> %0) {
+; CHECK-LABEL: pcvm_sml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pcvm %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call i64 @llvm.ve.vl.pcvm.sml(<256 x i1> %0, i32 256)
+  ret i64 %2
+}
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.ve.vl.pcvm.sml(<256 x i1>, i32)

llvm/test/CodeGen/VE/VELIntrinsics/tovm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test trailing one of vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test TOVM*ml instruction.
+
+; Function Attrs: nounwind readnone
+define fastcc i64 @tovm_sml(<256 x i1> %0) {
+; CHECK-LABEL: tovm_sml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    tovm %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call i64 @llvm.ve.vl.tovm.sml(<256 x i1> %0, i32 256)
+  ret i64 %2
+}
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.ve.vl.tovm.sml(<256 x i1>, i32)

llvm/test/CodeGen/VE/VELIntrinsics/xorm.ll

+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test exclusive or vm intrinsic instructions
+;;;
+;;; Note:
+;;;   We test XORM*mm and XORM*yy instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @xorm_mmm(<256 x i1> %0, <256 x i1> %1) {
+; CHECK-LABEL: xorm_mmm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorm %vm1, %vm1, %vm2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.xorm.mmm(<256 x i1> %0, <256 x i1> %1)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.xorm.mmm(<256 x i1>, <256 x i1>)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @xorm_MMM(<512 x i1> %0, <512 x i1> %1) {
+; CHECK-LABEL: xorm_MMM:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorm %vm2, %vm2, %vm4
+; CHECK-NEXT:    xorm %vm3, %vm3, %vm5
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.xorm.MMM(<512 x i1> %0, <512 x i1> %1)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.xorm.MMM(<512 x i1>, <512 x i1>)