[AArch64][SVE] Implement ptrue intrinsic

[sdesmalen]

llvm/include/llvm/IR/IntrinsicsAArch64.td

928

I'm going to anticipate @efriedma's suggestion to use ImmArg here :)

That will require changing sve_pred_enum to be a TImmLeaf instead of ImmLeaf though.

[c-rhodes]

Use ImmArg in intrinsic definition.

[sdesmalen]

LGTM!

llvm/include/llvm/IR/IntrinsicsAArch64.td

: Intrinsic<[llvm_anyvector_ty],
                 [IntrNoMem]>;
 
   class AdvSIMD_SVE_INSR_Intrinsic
     : Intrinsic<[llvm_anyvector_ty],
                 [LLVMMatchType<0>,
                  LLVMVectorElementType<0>],
                 [IntrNoMem]>;
 
+  class AdvSIMD_SVE_PTRUE_Intrinsic
+    : Intrinsic<[llvm_anyvector_ty],
+                [llvm_i32_ty],
+                [IntrNoMem, ImmArg<0>]>;
+
   class AdvSIMD_SVE_PUNPKHI_Intrinsic
     : Intrinsic<[LLVMHalfElementsVectorType<0>],
                 [llvm_anyvector_ty],
                 [IntrNoMem]>;
 
   class AdvSIMD_SVE_SCALE_Intrinsic
     : Intrinsic<[llvm_anyvector_ty],
                 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
class AdvSIMD_SVE_TSMUL_Intrinsic
     : Intrinsic<[llvm_anyvector_ty],
                 [LLVMMatchType<0>,
                  LLVMVectorOfBitcastsToInt<0>],
                 [IntrNoMem]>;
 
   class AdvSIMD_SVE_CNTB_Intrinsic
     : Intrinsic<[llvm_i64_ty],
                 [llvm_i32_ty],
-                [IntrNoMem]>;
+                [IntrNoMem, ImmArg<0>]>;
 
   class AdvSIMD_SVE_CNTP_Intrinsic
     : Intrinsic<[llvm_i64_ty],
                 [llvm_anyvector_ty, LLVMMatchType<0>],
                 [IntrNoMem]>;
 
   class AdvSIMD_SVE_DOT_Intrinsic
     : Intrinsic<[llvm_anyvector_ty],

 def int_aarch64_sve_scvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv4i32_ty>;
 
 def int_aarch64_sve_ucvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i32_ty>;
 def int_aarch64_sve_ucvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i64_ty>;
 def int_aarch64_sve_ucvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i64_ty>;
 def int_aarch64_sve_ucvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv4i32_ty>;
 
 //
+// Predicate creation
+//
+
+def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic;
+
+//
 // Predicate operations
 //
 
 def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
 def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
 
 //
 // Gather loads:

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   FRSQRTE, FRSQRTS,
 
   SUNPKHI,
   SUNPKLO,
   UUNPKHI,
   UUNPKLO,
 
   INSR,
+  PTRUE,
 
   // Unsigned gather loads.
   GLD1,
   GLD1_SCALED,
   GLD1_UXTW,
   GLD1_SXTW,
   GLD1_UXTW_SCALED,
   GLD1_SXTW_SCALED,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case AArch64ISD::STZG:              return "AArch64ISD::STZG";
   case AArch64ISD::ST2G:              return "AArch64ISD::ST2G";
   case AArch64ISD::STZ2G:             return "AArch64ISD::STZ2G";
   case AArch64ISD::SUNPKHI:           return "AArch64ISD::SUNPKHI";
   case AArch64ISD::SUNPKLO:           return "AArch64ISD::SUNPKLO";
   case AArch64ISD::UUNPKHI:           return "AArch64ISD::UUNPKHI";
   case AArch64ISD::UUNPKLO:           return "AArch64ISD::UUNPKLO";
   case AArch64ISD::INSR:              return "AArch64ISD::INSR";
+  case AArch64ISD::PTRUE:             return "AArch64ISD::PTRUE";
   case AArch64ISD::GLD1:              return "AArch64ISD::GLD1";
   case AArch64ISD::GLD1_SCALED:       return "AArch64ISD::GLD1_SCALED";
   case AArch64ISD::GLD1_SXTW:         return "AArch64ISD::GLD1_SXTW";
   case AArch64ISD::GLD1_UXTW:         return "AArch64ISD::GLD1_UXTW";
   case AArch64ISD::GLD1_SXTW_SCALED:  return "AArch64ISD::GLD1_SXTW_SCALED";
   case AArch64ISD::GLD1_UXTW_SCALED:  return "AArch64ISD::GLD1_UXTW_SCALED";
   case AArch64ISD::GLD1_IMM:          return "AArch64ISD::GLD1_IMM";
   case AArch64ISD::GLD1S:             return "AArch64ISD::GLD1S";
case Intrinsic::aarch64_sve_sunpklo:
     return DAG.getNode(AArch64ISD::SUNPKLO, dl, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_uunpkhi:
     return DAG.getNode(AArch64ISD::UUNPKHI, dl, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_uunpklo:
     return DAG.getNode(AArch64ISD::UUNPKLO, dl, Op.getValueType(),
                        Op.getOperand(1));
+  case Intrinsic::aarch64_sve_ptrue:
+    return DAG.getNode(AArch64ISD::PTRUE, dl, Op.getValueType(),
+                       Op.getOperand(1));
 
   case Intrinsic::aarch64_sve_insr: {
     SDValue Scalar = Op.getOperand(2);
     EVT ScalarTy = Scalar.getValueType();
     if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
       Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
 
     return DAG.getNode(AArch64ISD::INSR, dl, Op.getValueType(),

llvm/lib/Target/AArch64/SVEInstrFormats.td

 def SVEPatternOperand : AsmOperandClass {
   let Name = "SVEPattern";
   let ParserMethod = "tryParseSVEPattern";
   let PredicateMethod = "isSVEPattern";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = "InvalidSVEPattern";
 }
 
-def sve_pred_enum : Operand<i32>, ImmLeaf<i32, [{
+def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{
   return (((uint32_t)Imm) < 32);
   }]> {
 
   let PrintMethod = "printSVEPattern";
   let ParserMatchClass = SVEPatternOperand;
 }
 
 def SVEPrefetchOperand : AsmOperandClass {
def sve_incdec_imm : Operand<i32>, ImmLeaf<i32, [{
   let DecoderMethod = "DecodeSVEIncDecImm";
 }
 
 //===----------------------------------------------------------------------===//
 // SVE PTrue - These are used extensively throughout the pattern matching so
 //             it's important we define them first.
 //===----------------------------------------------------------------------===//
 
-class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty>
+class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty,
+                    ValueType vt, SDPatternOperator op>
 : I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern),
   asm, "\t$Pd, $pattern",
   "",
-  []>, Sched<[]> {
+  [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> {
   bits<4> Pd;
   bits<5> pattern;
   let Inst{31-24} = 0b00100101;
   let Inst{23-22} = sz8_64;
   let Inst{21-19} = 0b011;
   let Inst{18-17} = opc{2-1};
   let Inst{16}    = opc{0};
   let Inst{15-10} = 0b111000;
   let Inst{9-5}   = pattern;
   let Inst{4}     = 0b0;
   let Inst{3-0}   = Pd;
 
   let Defs = !if(!eq (opc{0}, 1), [NZCV], []);
 }
 
-multiclass sve_int_ptrue<bits<3> opc, string asm> {
-  def _B : sve_int_ptrue<0b00, opc, asm, PPR8>;
-  def _H : sve_int_ptrue<0b01, opc, asm, PPR16>;
-  def _S : sve_int_ptrue<0b10, opc, asm, PPR32>;
-  def _D : sve_int_ptrue<0b11, opc, asm, PPR64>;
+multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> {
+  def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>;
+  def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>;
+  def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>;
+  def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>;
 
   def : InstAlias<asm # "\t$Pd",
                   (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>;
   def : InstAlias<asm # "\t$Pd",
                   (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>;
   def : InstAlias<asm # "\t$Pd",
                   (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>;
   def : InstAlias<asm # "\t$Pd",
                   (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>;
 }
 
+def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>;
+def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>;
+
 let Predicates = [HasSVE] in {
-  defm PTRUE  : sve_int_ptrue<0b000, "ptrue">;
-  defm PTRUES : sve_int_ptrue<0b001, "ptrues">;
+  defm PTRUE  : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>;
+  defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>;
 }
 
 //===----------------------------------------------------------------------===//
 // SVE pattern match helpers.
 //===----------------------------------------------------------------------===//
 
 class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                    Instruction inst>

llvm/test/CodeGen/AArch64/sve-intrinsics-pred-creation.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; PTRUE
+;
+
+define <vscale x 16 x i1> @ptrue_b8() {
+; CHECK-LABEL: ptrue_b8:
+; CHECK: ptrue p0.b, pow2
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 0)
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @ptrue_b16() {
+; CHECK-LABEL: ptrue_b16:
+; CHECK: ptrue p0.h, vl1
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 1)
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @ptrue_b32() {
+; CHECK-LABEL: ptrue_b32:
+; CHECK: ptrue p0.s, mul3
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 30)
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @ptrue_b64() {
+; CHECK-LABEL: ptrue_b64:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  ret <vscale x 2 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 %pattern)
+declare <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 %pattern)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 %pattern)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 %pattern)