[VE] Split unsupported v512.32 ops

[simoll]

Summary

Split v512.32 binary ops into two v256.32 ops using packing support opcodes (vec_unpack_lo|hi, vec_pack).

Depends on D120053 for packing opcodes.

[kaz7]

llvm/lib/Target/VE/VEISelLowering.cpp

1684–1685

Is it possible to add some comments why returning Legal to isPackingSupportOpcode is correct. This function is basically returning legal in general and returning custom for special opcodes. So, adding one more special treatment to return legal looks like strange for me.

[simoll]

Added comment and assertion to document that vec_pack and vec_unpack_lo|hi are always pack legalized.

llvm/lib/Target/VE/VECustomDAG.h

 #include "llvm/CodeGen/TargetLowering.h"
 
 namespace llvm {
 
 Optional<unsigned> getVVPOpcode(unsigned Opcode);
 
 bool isVVPBinaryOp(unsigned Opcode);
 
+MVT splitVectorType(MVT VT);
+
 bool isPackedVectorType(EVT SomeVT);
 
 bool isMaskType(EVT SomeVT);
 
 bool isMaskArithmetic(SDValue Op);
 
 bool isVVPOrVEC(unsigned);
 
+bool supportsPackedMode(unsigned Opcode, EVT IdiomVT);
+
+bool isPackingSupportOpcode(unsigned Opc);
+
 bool maySafelyIgnoreMask(SDValue Op);
 
 /// The VE backend uses a two-staged process to lower and legalize vector
 /// instructions:
 //
 /// 1. VP and standard vector SDNodes are lowered to SDNodes of the VVP_* layer.
 //
 //     All VVP nodes have a mask and an Active Vector Length (AVL) parameter.

 Optional<int> getAVLPos(unsigned);
 
 // Whether this is a LEGALAVL node.
 bool isLegalAVL(SDValue AVL);
 
 // The AVL operand of this node.
 SDValue getNodeAVL(SDValue);
 
+// Mask position of this node.
+Optional<int> getMaskPos(unsigned);
+
+SDValue getNodeMask(SDValue);
+
 // Return the AVL operand of this node. If it is a LEGALAVL node, unwrap it.
 // Return with the boolean whether unwrapping happened.
 std::pair<SDValue, bool> getAnnotatedNodeAVL(SDValue);
 
 /// } AVL Functions
 
 enum class Packing {
   Normal = 0, // 256 element standard mode.
   Dense = 1   // 512 element packed mode.
 };
 
 // Get the vector or mask register type for this packing and element type.
 MVT getLegalVectorType(Packing P, MVT ElemVT);
 
 // Whether this type belongs to a packed mask or vector register.
 Packing getTypePacking(EVT);
 
 enum class PackElem : int8_t {
   Lo = 0, // Integer (63, 32]
   Hi = 1  // Float   (32,  0]
 };
 
+struct VETargetMasks {
+  SDValue Mask;
+  SDValue AVL;
+  VETargetMasks(SDValue Mask = SDValue(), SDValue AVL = SDValue())
+      : Mask(Mask), AVL(AVL) {}
+};
+
 class VECustomDAG {
   SelectionDAG &DAG;
   SDLoc DL;
 
 public:
   SelectionDAG *getDAG() const { return &DAG; }
 
   VECustomDAG(SelectionDAG &DAG, SDLoc DL) : DAG(DAG), DL(DL) {}
if (Flags)
       N->setFlags(*Flags);
     return N;
   }
 
   SDValue getUNDEF(EVT VT) const { return DAG.getUNDEF(VT); }
   /// } getNode
 
   /// Packing {
-  SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL);
-  SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL);
+  SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL) const;
+  SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL) const;
   /// } Packing
 
   SDValue getConstant(uint64_t Val, EVT VT, bool IsTarget = false,
                       bool IsOpaque = false) const;
 
   SDValue getConstantMask(Packing Packing, bool AllTrue) const;
   SDValue getMaskBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const;
   SDValue getBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const;
 
   // Wrap AVL in a LEGALAVL node (unless it is one already).
   SDValue annotateLegalAVL(SDValue AVL) const;
+  VETargetMasks getTargetSplitMask(SDValue RawMask, SDValue RawAVL,
+                                   PackElem Part) const;
 };
 
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_VE_VECUSTOMDAG_H

llvm/lib/Target/VE/VECustomDAG.cpp

 namespace llvm {
 
 bool isPackedVectorType(EVT SomeVT) {
   if (!SomeVT.isVector())
     return false;
   return SomeVT.getVectorNumElements() > StandardVectorWidth;
 }
 
+MVT splitVectorType(MVT VT) {
+  if (!VT.isVector())
+    return VT;
+  return MVT::getVectorVT(VT.getVectorElementType(), StandardVectorWidth);
+}
+
 MVT getLegalVectorType(Packing P, MVT ElemVT) {
   return MVT::getVectorVT(ElemVT, P == Packing::Normal ? StandardVectorWidth
                                                        : PackedVectorWidth);
 }
 
 Packing getTypePacking(EVT VT) {
   assert(VT.isVector());
   return isPackedVectorType(VT) ? Packing::Dense : Packing::Normal;
bool maySafelyIgnoreMask(SDValue Op) {
   case VEISD::VVP_SELECT:
     return false;
 
   default:
     return true;
   }
 }
 
+bool supportsPackedMode(unsigned Opcode, EVT IdiomVT) {
+  bool IsPackedOp = isPackedVectorType(IdiomVT);
+  bool IsMaskOp = isMaskType(IdiomVT);
+  switch (Opcode) {
+  default:
+    return false;
+
+  case VEISD::VEC_BROADCAST:
+    return true;
+#define REGISTER_PACKED(VVP_NAME) case VEISD::VVP_NAME:
+#include "VVPNodes.def"
+    return IsPackedOp && !IsMaskOp;
+  }
+}
+
+bool isPackingSupportOpcode(unsigned Opc) {
+  switch (Opc) {
+  case VEISD::VEC_PACK:
+  case VEISD::VEC_UNPACK_LO:
+  case VEISD::VEC_UNPACK_HI:
+    return true;
+  }
+  return false;
+}
+
 bool isVVPOrVEC(unsigned Opcode) {
   switch (Opcode) {
   case VEISD::VEC_BROADCAST:
 #define ADD_VVP_OP(VVPNAME, ...) case VEISD::VVPNAME:
 #include "VVPNodes.def"
     return true;
   }
   return false;
case VEISD::VEC_BROADCAST:
     return 1;
   case VEISD::VVP_SELECT:
     return 3;
   }
 
   return None;
 }
 
+Optional<int> getMaskPos(unsigned Opc) {
+  // This is only available for VP SDNodes
+  auto PosOpt = ISD::getVPMaskIdx(Opc);
+  if (PosOpt)
+    return *PosOpt;
+
+  // VVP Opcodes.
+  if (isVVPBinaryOp(Opc))
+    return 2;
+
+  // VM Opcodes.
+  switch (Opc) {
+  case VEISD::VVP_SELECT:
+    return 2;
+  }
+
+  return None;
+}
+
 bool isLegalAVL(SDValue AVL) { return AVL->getOpcode() == VEISD::LEGALAVL; }
 
 SDValue getNodeAVL(SDValue Op) {
   auto PosOpt = getAVLPos(Op->getOpcode());
   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
 }
 
+SDValue getNodeMask(SDValue Op) {
+  auto PosOpt = getMaskPos(Op->getOpcode());
+  return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
+}
+
 std::pair<SDValue, bool> getAnnotatedNodeAVL(SDValue Op) {
   SDValue AVL = getNodeAVL(Op);
   if (!AVL)
     return {SDValue(), true};
   if (isLegalAVL(AVL))
     return {AVL->getOperand(0), true};
   return {AVL, false};
 }

 
 SDValue VECustomDAG::annotateLegalAVL(SDValue AVL) const {
   if (isLegalAVL(AVL))
     return AVL;
   return getNode(VEISD::LEGALAVL, AVL.getValueType(), AVL);
 }
 
 SDValue VECustomDAG::getUnpack(EVT DestVT, SDValue Vec, PackElem Part,
-                               SDValue AVL) {
+                               SDValue AVL) const {
+  assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
+
   // TODO: Peek through VEC_PACK and VEC_BROADCAST(REPL_<sth> ..) operands.
   unsigned OC =
       (Part == PackElem::Lo) ? VEISD::VEC_UNPACK_LO : VEISD::VEC_UNPACK_HI;
   return DAG.getNode(OC, DL, DestVT, Vec, AVL);
 }
 
 SDValue VECustomDAG::getPack(EVT DestVT, SDValue LoVec, SDValue HiVec,
-                             SDValue AVL) {
+                             SDValue AVL) const {
+  assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
+
   // TODO: Peek through VEC_UNPACK_LO|HI operands.
   return DAG.getNode(VEISD::VEC_PACK, DL, DestVT, LoVec, HiVec, AVL);
 }
 
+VETargetMasks VECustomDAG::getTargetSplitMask(SDValue RawMask, SDValue RawAVL,
+                                              PackElem Part) const {
+  // Adjust AVL for this part
+  SDValue NewAVL;
+  SDValue OneV = getConstant(1, MVT::i32);
+  if (Part == PackElem::Hi)
+    NewAVL = getNode(ISD::ADD, MVT::i32, {RawAVL, OneV});
+  else
+    NewAVL = RawAVL;
+  NewAVL = getNode(ISD::SRL, MVT::i32, {NewAVL, OneV});
+
+  NewAVL = annotateLegalAVL(NewAVL);
+
+  // Legalize Mask (unpack or all-true)
+  SDValue NewMask;
+  if (!RawMask)
+    NewMask = getConstantMask(Packing::Normal, true);
+  else
+    NewMask = getUnpack(MVT::v256i1, RawMask, Part, NewAVL);
+
+  return VETargetMasks(NewMask, NewAVL);
+}
+
 } // namespace llvm

llvm/lib/Target/VE/VEISelLowering.h

public:
   Register prepareSymbol(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                          StringRef Symbol, const DebugLoc &DL, bool IsLocal,
                          bool IsCall) const;
   /// } Custom Inserter
 
   /// VVP Lowering {
   SDValue lowerToVVP(SDValue Op, SelectionDAG &DAG) const;
   SDValue legalizeInternalVectorOp(SDValue Op, SelectionDAG &DAG) const;
+  SDValue splitVectorOp(SDValue Op, VECustomDAG &CDAG) const;
   SDValue legalizePackedAVL(SDValue Op, VECustomDAG &CDAG) const;
   SDValue splitMaskArithmetic(SDValue Op, SelectionDAG &DAG) const;
   /// } VVPLowering
 
   /// Custom DAGCombine {
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
   SDValue combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const;

llvm/lib/Target/VE/VEISelLowering.cpp

SDValue VETargetLowering::lowerBUILD_VECTOR(SDValue Op,
   }
 
   // Expand
   return SDValue();
 }
 
 TargetLowering::LegalizeAction
 VETargetLowering::getCustomOperationAction(SDNode &Op) const {
+  // Custom legalization on VVP_* and VEC_* opcodes is required to pack-legalize
+  // these operations (transform nodes such that their AVL parameter refers to
+  // packs of 64bit, instead of number of elements.
+
+  // Packing opcodes are created with a pack-legal AVL (LEGALAVL). No need to
+  // re-visit them.
+  if (isPackingSupportOpcode(Op.getOpcode()))
+    return Legal;
+
   // Custom lower to legalize AVL for packed mode.
   if (isVVPOrVEC(Op.getOpcode()))
     return Custom;
   return Legal;
 }
 
 SDValue VETargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   LLVM_DEBUG(dbgs() << "::LowerOperation"; Op->print(dbgs()););

llvm/lib/Target/VE/VEInstrPatternsVec.td

def : Pat<(v256i1 (vec_unpack_lo v512i1:$vm, (i32 srcvalue))),
           (EXTRACT_SUBREG $vm, sub_vm_odd)>;
 def : Pat<(v256i1 (vec_unpack_hi v512i1:$vm, (i32 srcvalue))),
           (EXTRACT_SUBREG $vm, sub_vm_even)>;
 def : Pat<(v512i1 (vec_pack v256i1:$vlo, v256i1:$vhi, (i32 srcvalue))),
           (INSERT_SUBREG (INSERT_SUBREG
                          (v512i1 (IMPLICIT_DEF)),
                          $vlo, sub_vm_odd),
                          $vhi, sub_vm_even)>;
+
+// v256.32 <> v512.32
+multiclass Packing<ValueType PackVT> {
+  // no-op unpacks
+  def : Pat<(v256i32 (vec_unpack_lo PackVT:$vp, (i32 srcvalue))),
+            (COPY_TO_REGCLASS $vp, V64)>;
+  def : Pat<(v256f32 (vec_unpack_hi PackVT:$vp, (i32 srcvalue))),
+            (COPY_TO_REGCLASS $vp, V64)>;
+
+  // shuffle unpacks
+  def : Pat<(v256f32 (vec_unpack_lo PackVT:$vp, i32:$avl)),
+            (VSHFvvil $vp, $vp, 4, $avl)>; // always pick lo
+  def : Pat<(v256i32 (vec_unpack_hi PackVT:$vp, i32:$avl)),
+            (VSHFvvil $vp, $vp, 0, $avl)>; // always pick hi
+}
+
+defm : Packing<v512i32>;
+defm : Packing<v512f32>;
+
+def : Pat<(v512i32 (vec_pack v256i32:$vlo, v256i32:$vhi, i32:$avl)),
+          (VSHFvvil $vlo, $vhi, 13, $avl)>;
+def : Pat<(v512f32 (vec_pack v256f32:$vlo, v256f32:$vhi, i32:$avl)),
+          (VSHFvvil $vlo, $vhi, 8, $avl)>;

llvm/lib/Target/VE/VVPISelLowering.cpp

 
 using namespace llvm;
 
 #define DEBUG_TYPE "ve-lower"
 
 SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,
                                                    SelectionDAG &DAG) const {
   VECustomDAG CDAG(DAG, Op);
+
+  EVT IdiomVT = Op.getValueType();
+  if (isPackedVectorType(IdiomVT) &&
+      !supportsPackedMode(Op.getOpcode(), IdiomVT))
+    return splitVectorOp(Op, CDAG);
+
   // TODO: Implement odd/even splitting.
   return legalizePackedAVL(Op, CDAG);
 }
 
+SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const {
+  MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());
+
+  auto AVLPos = getAVLPos(Op->getOpcode());
+  auto MaskPos = getMaskPos(Op->getOpcode());
+
+  SDValue PackedMask = getNodeMask(Op);
+  auto AVLPair = getAnnotatedNodeAVL(Op);
+  SDValue PackedAVL = AVLPair.first;
+  assert(!AVLPair.second && "Expecting non pack-legalized oepration");
+
+  // request the parts
+  SDValue PartOps[2];
+
+  SDValue UpperPartAVL; // we will use this for packing things back together
+  for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
+    // VP ops already have an explicit mask and AVL. When expanding from non-VP
+    // attach those additional inputs here.
+    auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
+
+    if (Part == PackElem::Hi)
+      UpperPartAVL = SplitTM.AVL;
+
+    // Attach non-predicating value operands
+    SmallVector<SDValue, 4> OpVec;
+    for (unsigned i = 0; i < Op.getNumOperands(); ++i) {
+      if (AVLPos && ((int)i) == *AVLPos)
+        continue;
+      if (MaskPos && ((int)i) == *MaskPos)
+        continue;
+
+      // Value operand
+      auto PackedOperand = Op.getOperand(i);
+      auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());
+      SDValue PartV =
+          CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
+      OpVec.push_back(PartV);
+    }
+
+    // Add predicating args and generate part node.
+    OpVec.push_back(SplitTM.Mask);
+    OpVec.push_back(SplitTM.AVL);
+    // Emit legal VVP nodes.
+    PartOps[(int)Part] =
+        CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
+  }
+
+  // Re-package vectors.
+  return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],
+                      PartOps[(int)PackElem::Hi], UpperPartAVL);
+}
+
 SDValue VETargetLowering::legalizePackedAVL(SDValue Op,
                                             VECustomDAG &CDAG) const {
   LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";);
   // Only required for VEC and VVP ops.
   if (!isVVPOrVEC(Op->getOpcode()))
     return Op;
 
   // Operation already has a legal AVL.

llvm/lib/Target/VE/VVPNodes.def

#define ADD_BINARY_VVP_OP(VVPNAME,VPNAME,SDNAME) \
             HANDLE_VP_TO_VVP(VPNAME, VVPNAME)
 #endif
 
 #ifndef ADD_BINARY_VVP_OP_COMPACT
 #define ADD_BINARY_VVP_OP_COMPACT(NAME) \
     ADD_BINARY_VVP_OP(VVP_##NAME,VP_##NAME,NAME)
 #endif
 
+/// REGISTER_PACKED(OPC)
+/// \p OPC The VVP opcode of the operation.
+#ifndef REGISTER_PACKED
+#define REGISTER_PACKED(OPC)
+#endif
+
 // Integer arithmetic.
-ADD_BINARY_VVP_OP_COMPACT(ADD)
-ADD_BINARY_VVP_OP_COMPACT(SUB)
+ADD_BINARY_VVP_OP_COMPACT(ADD) REGISTER_PACKED(VVP_ADD)
+ADD_BINARY_VVP_OP_COMPACT(SUB) REGISTER_PACKED(VVP_SUB)
 ADD_BINARY_VVP_OP_COMPACT(MUL)
 ADD_BINARY_VVP_OP_COMPACT(UDIV)
 ADD_BINARY_VVP_OP_COMPACT(SDIV)
 
-ADD_BINARY_VVP_OP(VVP_SRA,VP_ASHR,SRA)
-ADD_BINARY_VVP_OP(VVP_SRL,VP_LSHR,SRL)
-ADD_BINARY_VVP_OP_COMPACT(SHL)
-
-ADD_BINARY_VVP_OP_COMPACT(AND)
-ADD_BINARY_VVP_OP_COMPACT(OR)
-ADD_BINARY_VVP_OP_COMPACT(XOR)
+ADD_BINARY_VVP_OP(VVP_SRA,VP_ASHR,SRA) REGISTER_PACKED(VVP_SRA)
+ADD_BINARY_VVP_OP(VVP_SRL,VP_LSHR,SRL) REGISTER_PACKED(VVP_SRL)
+ADD_BINARY_VVP_OP_COMPACT(SHL) REGISTER_PACKED(VVP_SHL)
+
+ADD_BINARY_VVP_OP_COMPACT(AND) REGISTER_PACKED(VVP_AND)
+ADD_BINARY_VVP_OP_COMPACT(OR)  REGISTER_PACKED(VVP_OR)
+ADD_BINARY_VVP_OP_COMPACT(XOR) REGISTER_PACKED(VVP_XOR)
 
 // FP arithmetic.
-ADD_BINARY_VVP_OP_COMPACT(FADD)
-ADD_BINARY_VVP_OP_COMPACT(FSUB)
-ADD_BINARY_VVP_OP_COMPACT(FMUL)
+ADD_BINARY_VVP_OP_COMPACT(FADD) REGISTER_PACKED(VVP_FADD)
+ADD_BINARY_VVP_OP_COMPACT(FSUB) REGISTER_PACKED(VVP_FSUB)
+ADD_BINARY_VVP_OP_COMPACT(FMUL) REGISTER_PACKED(VVP_FMUL)
 ADD_BINARY_VVP_OP_COMPACT(FDIV)
 
 ADD_VVP_OP(VVP_SETCC, SETCC)
 
 // Shuffles.
-ADD_VVP_OP(VVP_SELECT,VSELECT)
+ADD_VVP_OP(VVP_SELECT,VSELECT) REGISTER_PACKED(VVP_SELECT)
 HANDLE_VP_TO_VVP(VP_SELECT, VVP_SELECT)
 HANDLE_VP_TO_VVP(VP_MERGE, VVP_SELECT)
 
 #undef ADD_BINARY_VVP_OP
 #undef ADD_BINARY_VVP_OP_COMPACT
 #undef ADD_VVP_OP
 #undef HANDLE_VP_TO_VVP
+#undef REGISTER_PACKED

llvm/test/CodeGen/VE/Packed/vp_fdiv.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=ve -mattr=+vpu | FileCheck %s
+
+declare <512 x float> @llvm.vp.fdiv.v512f32(<512 x float>, <512 x float>, <512 x i1>, i32)
+
+define fastcc <512 x float> @test_vp_fdiv_v512f32_vv(<512 x float> %i0, <512 x float> %i1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_fdiv_v512f32_vv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s1, %s0, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v2, %v1, %v1, 4
+; CHECK-NEXT:    vshf %v3, %v0, %v0, 4
+; CHECK-NEXT:    vfdiv.s %v2, %v3, %v2, %vm3
+; CHECK-NEXT:    adds.w.sx %s0, 1, %s0
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.s %v0, %v0, %v1, %vm2
+; CHECK-NEXT:    vshf %v0, %v2, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %r0 = call <512 x float> @llvm.vp.fdiv.v512f32(<512 x float> %i0, <512 x float> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x float> %r0
+}
+
+define fastcc <512 x float> @test_vp_fdiv_v512f32_rv(float %s0, <512 x float> %i1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_fdiv_v512f32_rv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s2, %s0, (32)1
+; CHECK-NEXT:    srl %s0, %s0, 32
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vbrd %v1, %s0
+; CHECK-NEXT:    adds.w.sx %s0, 1, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.s %v2, %v1, %v0, %vm2
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v1, %v1, %v1, 4
+; CHECK-NEXT:    vshf %v0, %v0, %v0, 4
+; CHECK-NEXT:    vfdiv.s %v0, %v1, %v0, %vm3
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %xins = insertelement <512 x float> undef, float %s0, i32 0
+  %i0 = shufflevector <512 x float> %xins, <512 x float> undef, <512 x i32> zeroinitializer
+  %r0 = call <512 x float> @llvm.vp.fdiv.v512f32(<512 x float> %i0, <512 x float> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x float> %r0
+}
+
+define fastcc <512 x float> @test_vp_fdiv_v512f32_vr(<512 x float> %i0, float %s1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_fdiv_v512f32_vr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s2, %s0, (32)1
+; CHECK-NEXT:    srl %s0, %s0, 32
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vbrd %v1, %s0
+; CHECK-NEXT:    adds.w.sx %s0, 1, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.s %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v1, %v1, %v1, 4
+; CHECK-NEXT:    vshf %v0, %v0, %v0, 4
+; CHECK-NEXT:    vfdiv.s %v0, %v0, %v1, %vm3
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %yins = insertelement <512 x float> undef, float %s1, i32 0
+  %i1 = shufflevector <512 x float> %yins, <512 x float> undef, <512 x i32> zeroinitializer
+  %r0 = call <512 x float> @llvm.vp.fdiv.v512f32(<512 x float> %i0, <512 x float> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x float> %r0
+}

llvm/test/CodeGen/VE/Packed/vp_mul.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=ve -mattr=+vpu | FileCheck %s
+
+declare <512 x i32> @llvm.vp.mul.v512i32(<512 x i32>, <512 x i32>, <512 x i1>, i32)
+
+define fastcc <512 x i32> @test_vp_v512i32(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_v512i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    adds.w.sx %s1, 1, %s0
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v2, %v1, %v1, 0
+; CHECK-NEXT:    vshf %v3, %v0, %v0, 0
+; CHECK-NEXT:    vmuls.w.sx %v2, %v3, %v2, %vm2
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.sx %v0, %v0, %v1, %vm3
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 13
+; CHECK-NEXT:    b.l.t (, %s10)
+  %r0 = call <512 x i32> @llvm.vp.mul.v512i32(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x i32> %r0
+}

llvm/test/CodeGen/VE/Packed/vp_sdiv.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=ve -mattr=+vpu | FileCheck %s
+
+declare <512 x i32> @llvm.vp.sdiv.v512i32(<512 x i32>, <512 x i32>, <512 x i1>, i32)
+
+define fastcc <512 x i32> @test_vp_sdiv_v512i32_vv(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_sdiv_v512i32_vv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    adds.w.sx %s1, 1, %s0
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v2, %v1, %v1, 0
+; CHECK-NEXT:    vshf %v3, %v0, %v0, 0
+; CHECK-NEXT:    vdivs.w.sx %v2, %v3, %v2, %vm2
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v0, %v0, %v1, %vm3
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 13
+; CHECK-NEXT:    b.l.t (, %s10)
+  %r0 = call <512 x i32> @llvm.vp.sdiv.v512i32(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x i32> %r0
+}
+
+define fastcc <512 x i32> @test_vp_sdiv_v512i32_rv(i32 %s0, <512 x i32> %i1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_sdiv_v512i32_rv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    sll %s2, %s0, 32
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vbrd %v1, %s0
+; CHECK-NEXT:    adds.w.sx %s0, 1, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v2, %v1, %v1, 0
+; CHECK-NEXT:    vshf %v3, %v0, %v0, 0
+; CHECK-NEXT:    vdivs.w.sx %v2, %v2, %v3, %vm2
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.sx %v0, %v1, %v0, %vm3
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 13
+; CHECK-NEXT:    b.l.t (, %s10)
+  %xins = insertelement <512 x i32> undef, i32 %s0, i32 0
+  %i0 = shufflevector <512 x i32> %xins, <512 x i32> undef, <512 x i32> zeroinitializer
+  %r0 = call <512 x i32> @llvm.vp.sdiv.v512i32(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x i32> %r0
+}
+
+define fastcc <512 x i32> @test_vp_sdiv_v512i32_vr(<512 x i32> %i0, i32 %s1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_sdiv_v512i32_vr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    sll %s2, %s0, 32
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vbrd %v1, %s0
+; CHECK-NEXT:    adds.w.sx %s0, 1, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v2, %v1, %v1, 0
+; CHECK-NEXT:    vshf %v3, %v0, %v0, 0
+; CHECK-NEXT:    vdivs.w.sx %v2, %v3, %v2, %vm2
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.sx %v0, %v0, %v1, %vm3
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 13
+; CHECK-NEXT:    b.l.t (, %s10)
+  %yins = insertelement <512 x i32> undef, i32 %s1, i32 0
+  %i1 = shufflevector <512 x i32> %yins, <512 x i32> undef, <512 x i32> zeroinitializer
+  %r0 = call <512 x i32> @llvm.vp.sdiv.v512i32(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x i32> %r0
+}

llvm/test/CodeGen/VE/Packed/vp_udiv.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=ve -mattr=+vpu | FileCheck %s
+
+declare <512 x i32> @llvm.vp.udiv.v512i32(<512 x i32>, <512 x i32>, <512 x i1>, i32)
+
+define fastcc <512 x i32> @test_vp_udiv_v512i32_vv(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_udiv_v512i32_vv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    adds.w.sx %s1, 1, %s0
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v2, %v1, %v1, 0
+; CHECK-NEXT:    vshf %v3, %v0, %v0, 0
+; CHECK-NEXT:    vdivu.w %v2, %v3, %v2, %vm2
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v0, %v0, %v1, %vm3
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 13
+; CHECK-NEXT:    b.l.t (, %s10)
+  %r0 = call <512 x i32> @llvm.vp.udiv.v512i32(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x i32> %r0
+}
+
+define fastcc <512 x i32> @test_vp_udiv_v512i32_rv(i32 %s0, <512 x i32> %i1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_udiv_v512i32_rv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    sll %s2, %s0, 32
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vbrd %v1, %s0
+; CHECK-NEXT:    adds.w.sx %s0, 1, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v2, %v1, %v1, 0
+; CHECK-NEXT:    vshf %v3, %v0, %v0, 0
+; CHECK-NEXT:    vdivu.w %v2, %v2, %v3, %vm2
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.w %v0, %v1, %v0, %vm3
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 13
+; CHECK-NEXT:    b.l.t (, %s10)
+  %xins = insertelement <512 x i32> undef, i32 %s0, i32 0
+  %i0 = shufflevector <512 x i32> %xins, <512 x i32> undef, <512 x i32> zeroinitializer
+  %r0 = call <512 x i32> @llvm.vp.udiv.v512i32(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x i32> %r0
+}
+
+define fastcc <512 x i32> @test_vp_udiv_v512i32_vr(<512 x i32> %i0, i32 %s1, <512 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_udiv_v512i32_vr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    sll %s2, %s0, 32
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vbrd %v1, %s0
+; CHECK-NEXT:    adds.w.sx %s0, 1, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 1
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v2, %v1, %v1, 0
+; CHECK-NEXT:    vshf %v3, %v0, %v0, 0
+; CHECK-NEXT:    vdivu.w %v2, %v3, %v2, %vm2
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    srl %s1, %s1, 1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.w %v0, %v0, %v1, %vm3
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v0, %v0, %v2, 13
+; CHECK-NEXT:    b.l.t (, %s10)
+  %yins = insertelement <512 x i32> undef, i32 %s1, i32 0
+  %i1 = shufflevector <512 x i32> %yins, <512 x i32> undef, <512 x i32> zeroinitializer
+  %r0 = call <512 x i32> @llvm.vp.udiv.v512i32(<512 x i32> %i0, <512 x i32> %i1, <512 x i1> %m, i32 %n)
+  ret <512 x i32> %r0
+}