[VE] v512i1 mask arithmetic isel

[simoll]

Summary

Packed vector and mask registers (v512) are composed of two v256 subregisters that occupy the even and odd element positions. We add packing support SDNodes (vec_unpack_lo|hi and vec_pack) and splitting of v512i1 mask arithmetic ops with those.

[kaz7]

Please add summary. Otherwise, LGTM.

[kaz7]

LGTM.

llvm/lib/Target/VE/VECustomDAG.h

 Optional<unsigned> getVVPOpcode(unsigned Opcode);
 
 bool isVVPBinaryOp(unsigned Opcode);
 
 bool isPackedVectorType(EVT SomeVT);
 
 bool isMaskType(EVT SomeVT);
 
+bool isMaskArithmetic(SDValue Op);
+
 bool isVVPOrVEC(unsigned);
 
 bool maySafelyIgnoreMask(SDValue Op);
 
 /// The VE backend uses a two-staged process to lower and legalize vector
 /// instructions:
 //
 /// 1. VP and standard vector SDNodes are lowered to SDNodes of the VVP_* layer.

 };
 
 // Get the vector or mask register type for this packing and element type.
 MVT getLegalVectorType(Packing P, MVT ElemVT);
 
 // Whether this type belongs to a packed mask or vector register.
 Packing getTypePacking(EVT);
 
+enum class PackElem : int8_t {
+  Lo = 0, // Integer (63, 32]
+  Hi = 1  // Float   (32,  0]
+};
+
 class VECustomDAG {
   SelectionDAG &DAG;
   SDLoc DL;
 
 public:
   SelectionDAG *getDAG() const { return &DAG; }
 
   VECustomDAG(SelectionDAG &DAG, SDLoc DL) : DAG(DAG), DL(DL) {}
SDValue getNode(unsigned OC, EVT ResVT, ArrayRef<SDValue> OpV,
     if (Flags)
       N->setFlags(*Flags);
     return N;
   }
 
   SDValue getUNDEF(EVT VT) const { return DAG.getUNDEF(VT); }
   /// } getNode
 
+  /// Packing {
+  SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL);
+  SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL);
+  /// } Packing
+
   SDValue getConstant(uint64_t Val, EVT VT, bool IsTarget = false,
                       bool IsOpaque = false) const;
 
   SDValue getConstantMask(Packing Packing, bool AllTrue) const;
   SDValue getMaskBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const;
   SDValue getBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const;
 
   // Wrap AVL in a LEGALAVL node (unless it is one already).
   SDValue annotateLegalAVL(SDValue AVL) const;
 };
 
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_VE_VECUSTOMDAG_H

llvm/lib/Target/VE/VECustomDAG.cpp

 }
 
 bool isMaskType(EVT SomeVT) {
   if (!SomeVT.isVector())
     return false;
   return SomeVT.getVectorElementType() == MVT::i1;
 }
 
+bool isMaskArithmetic(SDValue Op) {
+  switch (Op.getOpcode()) {
+  default:
+    return false;
+  case ISD::AND:
+  case ISD::XOR:
+  case ISD::OR:
+    return isMaskType(Op.getValueType());
+  }
+}
+
 /// \returns the VVP_* SDNode opcode corresponsing to \p OC.
 Optional<unsigned> getVVPOpcode(unsigned Opcode) {
   switch (Opcode) {
 #define HANDLE_VP_TO_VVP(VPOPC, VVPNAME)                                       \
   case ISD::VPOPC:                                                             \
     return VEISD::VVPNAME;
 #define ADD_VVP_OP(VVPNAME, SDNAME)                                            \
   case VEISD::VVPNAME:                                                         \

 }
 
 SDValue VECustomDAG::annotateLegalAVL(SDValue AVL) const {
   if (isLegalAVL(AVL))
     return AVL;
   return getNode(VEISD::LEGALAVL, AVL.getValueType(), AVL);
 }
 
+SDValue VECustomDAG::getUnpack(EVT DestVT, SDValue Vec, PackElem Part,
+                               SDValue AVL) {
+  // TODO: Peek through VEC_PACK and VEC_BROADCAST(REPL_<sth> ..) operands.
+  unsigned OC =
+      (Part == PackElem::Lo) ? VEISD::VEC_UNPACK_LO : VEISD::VEC_UNPACK_HI;
+  return DAG.getNode(OC, DL, DestVT, Vec, AVL);
+}
+
+SDValue VECustomDAG::getPack(EVT DestVT, SDValue LoVec, SDValue HiVec,
+                             SDValue AVL) {
+  // TODO: Peek through VEC_UNPACK_LO|HI operands.
+  return DAG.getNode(VEISD::VEC_PACK, DL, DestVT, LoVec, HiVec, AVL);
+}
+
 } // namespace llvm

llvm/lib/Target/VE/VEISelLowering.h

enum NodeType : unsigned {
   GETSTACKTOP,            // Retrieve address of stack top (first address of
                           // locals and temporaries).
   GLOBAL_BASE_REG,        // Global base reg for PIC.
   Hi,                     // Hi/Lo operations, typically on a global address.
   Lo,                     // Hi/Lo operations, typically on a global address.
   MEMBARRIER,             // Compiler barrier only; generate a no-op.
   RET_FLAG,               // Return with a flag operand.
   TS1AM,                  // A TS1AM instruction used for 1/2 bytes swap.
-  VEC_BROADCAST,          // A vector broadcast instruction.
-                          //   0: scalar value, 1: VL
+  VEC_UNPACK_LO,          // unpack the lo v256 slice of a packed v512 vector.
+  VEC_UNPACK_HI,          // unpack the hi v256 slice of a packed v512 vector.
+                          //    0: v512 vector, 1: AVL
+  VEC_PACK,               // pack a lo and a hi vector into one v512 vector
+                          //    0: v256 lo vector, 1: v256 hi vector, 2: AVL
+
+  VEC_BROADCAST, // A vector broadcast instruction.
   REPL_I32,
   REPL_F32, // Replicate subregister to other half.
 
   // Annotation as a wrapper. LEGALAVL(VL) means that VL refers to 64bit of
   // data, whereas the raw EVL coming in from VP nodes always refers to number
   // of elements, regardless of their size.
   LEGALAVL,
 
Register prepareSymbol(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                          StringRef Symbol, const DebugLoc &DL, bool IsLocal,
                          bool IsCall) const;
   /// } Custom Inserter
 
   /// VVP Lowering {
   SDValue lowerToVVP(SDValue Op, SelectionDAG &DAG) const;
   SDValue legalizeInternalVectorOp(SDValue Op, SelectionDAG &DAG) const;
   SDValue legalizePackedAVL(SDValue Op, VECustomDAG &CDAG) const;
+  SDValue splitMaskArithmetic(SDValue Op, SelectionDAG &DAG) const;
   /// } VVPLowering
 
   /// Custom DAGCombine {
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
   SDValue combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const;
   /// } Custom DAGCombine
 

llvm/lib/Target/VE/VEISelLowering.cpp

void VETargetLowering::initSPUActions() {
   // Intrinsic instructions
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 }
 
 void VETargetLowering::initVPUActions() {
   for (MVT LegalMaskVT : AllMaskVTs)
     setOperationAction(ISD::BUILD_VECTOR, LegalMaskVT, Custom);
 
+  for (unsigned Opc : {ISD::AND, ISD::OR, ISD::XOR})
+    setOperationAction(Opc, MVT::v512i1, Custom);
+
   for (MVT LegalVecVT : AllVectorVTs) {
     setOperationAction(ISD::BUILD_VECTOR, LegalVecVT, Custom);
     setOperationAction(ISD::INSERT_VECTOR_ELT, LegalVecVT, Legal);
     setOperationAction(ISD::EXTRACT_VECTOR_ELT, LegalVecVT, Legal);
     // Translate all vector instructions with legal element types to VVP_*
     // nodes.
     // TODO We will custom-widen into VVP_* nodes in the future. While we are
     // buildling the infrastructure for this, we only do this for legal vector
case VEISD::FIRST_NUMBER:
     TARGET_NODE_CASE(GETSTACKTOP)
     TARGET_NODE_CASE(GETTLSADDR)
     TARGET_NODE_CASE(GLOBAL_BASE_REG)
     TARGET_NODE_CASE(Hi)
     TARGET_NODE_CASE(Lo)
     TARGET_NODE_CASE(MEMBARRIER)
     TARGET_NODE_CASE(RET_FLAG)
     TARGET_NODE_CASE(TS1AM)
+    TARGET_NODE_CASE(VEC_UNPACK_LO)
+    TARGET_NODE_CASE(VEC_UNPACK_HI)
+    TARGET_NODE_CASE(VEC_PACK)
     TARGET_NODE_CASE(VEC_BROADCAST)
     TARGET_NODE_CASE(REPL_I32)
     TARGET_NODE_CASE(REPL_F32)
 
     TARGET_NODE_CASE(LEGALAVL)
 
     // Register the VVP_* SDNodes.
 #define ADD_VVP_OP(VVP_NAME, ...) TARGET_NODE_CASE(VVP_NAME)
#include "VVPNodes.def"
     // AVL already legalized.
     if (getAnnotatedNodeAVL(Op).second)
       return Op;
     return legalizeInternalVectorOp(Op, DAG);
 
     // Translate into a VEC_*/VVP_* layer operation.
 #define ADD_VVP_OP(VVP_NAME, ISD_NAME) case ISD::ISD_NAME:
 #include "VVPNodes.def"
+    if (isMaskArithmetic(Op) && isPackedVectorType(Op.getValueType()))
+      return splitMaskArithmetic(Op, DAG);
     return lowerToVVP(Op, DAG);
   }
 }
 /// } Custom Lower
 
 void VETargetLowering::ReplaceNodeResults(SDNode *N,
                                           SmallVectorImpl<SDValue> &Results,
                                           SelectionDAG &DAG) const {
bool VETargetLowering::hasAndNot(SDValue Y) const {
   //        correctly with Aurora VE.
   if (isa<ConstantSDNode>(Y))
     return false;
 
   // It's ok for generic registers.
   return true;
 }
 
+SDValue VETargetLowering::splitMaskArithmetic(SDValue Op,
+                                              SelectionDAG &DAG) const {
+  VECustomDAG CDAG(DAG, Op);
+  SDValue AVL =
+      CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);
+  SDValue A = Op->getOperand(0);
+  SDValue B = Op->getOperand(1);
+  SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL);
+  SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL);
+  SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL);
+  SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL);
+  unsigned Opc = Op.getOpcode();
+  auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});
+  auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});
+  return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);
+}
+
 SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {
   // Can we represent this as a VVP node.
   const unsigned Opcode = Op->getOpcode();
   auto VVPOpcodeOpt = getVVPOpcode(Opcode);
   if (!VVPOpcodeOpt.hasValue())
     return SDValue();
   unsigned VVPOpcode = VVPOpcodeOpt.getValue();
   const bool FromVP = ISD::isVPOpcode(Opcode);

llvm/lib/Target/VE/VEInstrInfo.td

 // Vector Instruction Pattern Stuff
 //===----------------------------------------------------------------------===//
 
 // Custom intermediate ISDs.
 class IsVLVT<int OpIdx> : SDTCisVT<OpIdx,i32>;
 def vec_broadcast       : SDNode<"VEISD::VEC_BROADCAST", SDTypeProfile<1, 2,
                                  [SDTCisVec<0>, IsVLVT<2>]>>;
 
+///// Packed mode Support /////
+// unpack the lo part of this vector
+def vec_unpack_lo   : SDNode<"VEISD::VEC_UNPACK_LO", SDTypeProfile<1, 2,
+                             [SDTCisVec<0>, SDTCisVec<1>, IsVLVT<2>]>>;
+// unpack the hipart of this vector
+def vec_unpack_hi   : SDNode<"VEISD::VEC_UNPACK_HI", SDTypeProfile<1, 2,
+                             [SDTCisVec<0>, SDTCisVec<1>, IsVLVT<2>]>>;
+// re-pack v256i32, v256f32 back into tone v512.32
+def vec_pack        : SDNode<"VEISD::VEC_PACK", SDTypeProfile<1, 3,
+                             [SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>,
+                              SDTCisSameNumEltsAs<1,2>, IsVLVT<3>]>>;
+
 // replicate lower 32bit to upper 32bit (f32 scalar replication).
 def repl_f32            : SDNode<"VEISD::REPL_F32",
                             SDTypeProfile<1, 1,
                               [SDTCisInt<0>, SDTCisFP<1>]>>;
 // replicate upper 32bit to lower 32 bit (i32 scalar replication).
 def repl_i32            : SDNode<"VEISD::REPL_I32",
                             SDTypeProfile<1, 1,
                               [SDTCisInt<0>, SDTCisInt<1>]>>;

llvm/lib/Target/VE/VEInstrPatternsVec.td

 defm : vbrd_elem64<v512f32, f64, simm7fp, LO7FP>;
 
 class Mask_Binary<ValueType MaskVT, SDPatternOperator MaskOp, string InstName> :
   Pat<(MaskVT (MaskOp MaskVT:$ma, MaskVT:$mb)), (!cast<Instruction>(InstName#"mm") $ma, $mb)>;
 
 def: Mask_Binary<v256i1, and, "ANDM">;
 def: Mask_Binary<v256i1, or,  "ORM">;
 def: Mask_Binary<v256i1, xor, "XORM">;
+
+///// Packing support /////
+
+// v256i1 <> v512i1
+def : Pat<(v256i1 (vec_unpack_lo v512i1:$vm, (i32 srcvalue))),
+          (EXTRACT_SUBREG $vm, sub_vm_odd)>;
+def : Pat<(v256i1 (vec_unpack_hi v512i1:$vm, (i32 srcvalue))),
+          (EXTRACT_SUBREG $vm, sub_vm_even)>;
+def : Pat<(v512i1 (vec_pack v256i1:$vlo, v256i1:$vhi, (i32 srcvalue))),
+          (INSERT_SUBREG (INSERT_SUBREG
+                         (v512i1 (IMPLICIT_DEF)),
+                         $vlo, sub_vm_odd),
+                         $vhi, sub_vm_even)>;

llvm/test/CodeGen/VE/Packed/mask_binary.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+; Function Attrs: nounwind
+define fastcc <512 x i1> @and_mm_v512i1(<512 x i1> %x, <512 x i1> %y) {
+; CHECK-LABEL: and_mm_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andm %vm6, %vm2, %vm4
+; CHECK-NEXT:    andm %vm7, %vm3, %vm5
+; CHECK-NEXT:    andm %vm2, %vm0, %vm6
+; CHECK-NEXT:    andm %vm3, %vm0, %vm7
+; CHECK-NEXT:    b.l.t (, %s10)
+  %z = and <512 x i1> %x, %y
+  ret <512 x i1> %z
+}
+
+; Function Attrs: nounwind
+define fastcc <512 x i1> @or_mm_v512i1(<512 x i1> %x, <512 x i1> %y) {
+; CHECK-LABEL: or_mm_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orm %vm6, %vm2, %vm4
+; CHECK-NEXT:    orm %vm7, %vm3, %vm5
+; CHECK-NEXT:    andm %vm2, %vm0, %vm6
+; CHECK-NEXT:    andm %vm3, %vm0, %vm7
+; CHECK-NEXT:    b.l.t (, %s10)
+  %z = or <512 x i1> %x, %y
+  ret <512 x i1> %z
+}
+
+; Function Attrs: nounwind
+define fastcc <512 x i1> @xor_mm_v512i1(<512 x i1> %x, <512 x i1> %y) {
+; CHECK-LABEL: xor_mm_v512i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorm %vm6, %vm2, %vm4
+; CHECK-NEXT:    xorm %vm7, %vm3, %vm5
+; CHECK-NEXT:    andm %vm2, %vm0, %vm6
+; CHECK-NEXT:    andm %vm3, %vm0, %vm7
+; CHECK-NEXT:    b.l.t (, %s10)
+  %z = xor <512 x i1> %x, %y
+  ret <512 x i1> %z
+}
+