[SveEmitter] Add IsAppendSVALL and builtins for svptrue and svcnt[bhwd]

[sdesmalen]

Summary

Some ACLE builtins leave out the argument to specify the predicate
pattern, which is expected to be expanded to an SV_ALL pattern.

This patch adds the flag IsAppendSVALL to append SV_ALL as the final
operand.

[sdesmalen]

rebased patch.

[efriedma]

clang/lib/CodeGen/CGBuiltin.cpp

7913

The Ops.size() <= 1 seems to return the same result for all the intrinsics you're implementing now. Does it become relevant later?

[sdesmalen]

clang/lib/CodeGen/CGBuiltin.cpp

7913

Yes that's right, this is there for the saturating inc/dec intrinsics, e.g.

There is the one that takes a predicate pattern:

int32_t svqdecb_pat[_n_s32](int32_t op, svpattern pattern, uint64_t imm_factor)

and one where the pattern SV_ALL is implicit (and inserted as the second operand):

int32_t svqdecb[_n_s32](int32_t op, uint64_t imm_factor)

[efriedma]

clang/lib/CodeGen/CGBuiltin.cpp

7913

I think it would be more clear to use a separate flag for that, so it's obvious what codepaths in EmitAArch64SVEBuiltinExpr actually correspond to particular intrinsics.

[sdesmalen]

• Renamed flag to IsAppendSVALL to distinguish it from the case of IsInsertOp1SVALL that will follow in a separate patch.

[sdesmalen]

• Added comment explaining need for expanding SV_ALL

[sdesmalen]

Gentle ping :)

[efriedma]

LGTM

clang/include/clang/Basic/TargetBuiltins.h

#undef LLVM_GET_SVE_IMMCHECKTYPES
     bool isByteIndexed() const { return Flags & IsByteIndexed; }
     bool isOverloadNone() const { return Flags & IsOverloadNone; }
     bool isOverloadWhile() const { return Flags & IsOverloadWhile; }
     bool isOverloadDefault() const { return !(Flags & OverloadKindMask); }
     bool isOverloadWhileRW() const { return Flags & IsOverloadWhileRW; }
     bool isOverloadCvt() const { return Flags & IsOverloadCvt; }
     bool isPrefetch() const { return Flags & IsPrefetch; }
     bool isReverseCompare() const { return Flags & ReverseCompare; }
+    bool isAppendSVALL() const { return Flags & IsAppendSVALL; }
 
     uint64_t getBits() const { return Flags; }
     bool isFlagSet(uint64_t Flag) const { return Flags & Flag; }
   };
 
   /// Hexagon builtins
   namespace Hexagon {
     enum {

clang/include/clang/Basic/arm_sve.td

 def IsStructStore             : FlagType<0x00040000>;
 def IsZExtReturn              : FlagType<0x00080000>; // Return value is sign-extend by default
 def IsOverloadNone            : FlagType<0x00100000>; // Intrinsic does not take any overloaded types.
 def IsOverloadWhile           : FlagType<0x00200000>; // Use {default type, typeof(operand1)} as overloaded types.
 def IsOverloadWhileRW         : FlagType<0x00400000>; // Use {pred(default type), typeof(operand0)} as overloaded types.
 def IsOverloadCvt             : FlagType<0x00800000>; // Use {typeof(operand0), typeof(last operand)} as overloaded types.
 def OverloadKindMask          : FlagType<0x00E00000>; // When the masked values are all '0', the default type is used as overload type.
 def IsByteIndexed             : FlagType<0x01000000>;
+def IsAppendSVALL             : FlagType<0x02000000>; // Appends SV_ALL as the last operand.
 def IsPrefetch                : FlagType<0x08000000>; // Contiguous prefetches.
 def ReverseCompare            : FlagType<0x20000000>; // Compare operands must be swapped.
 
 // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h
 class ImmCheckType<int val> {
   int Value = val;
 }
 def ImmCheck0_31                : ImmCheckType<0>;  // 0..31 (used for e.g. predicate patterns)

 def SVQDECH_U : SInst<"svqdech_pat[_{d}]",   "ddIi", "Us", MergeNone, "aarch64_sve_uqdech", [], [ImmCheck<2, ImmCheck1_16>]>;
 
 
 ////////////////////////////////////////////////////////////////////////////////
 // Predicate creation
 
 def SVPFALSE : SInst<"svpfalse[_b]", "P", "", MergeNone, "", [IsOverloadNone]>;
 
+def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue">;
+def SVPTRUE     : SInst<"svptrue_{d}",     "P",  "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL]>;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Counting elements
 
 def SVCNTB_PAT : SInst<"svcntb_pat", "nI", "", MergeNone, "aarch64_sve_cntb", [IsOverloadNone]>;
 def SVCNTH_PAT : SInst<"svcnth_pat", "nI", "", MergeNone, "aarch64_sve_cnth", [IsOverloadNone]>;
 def SVCNTW_PAT : SInst<"svcntw_pat", "nI", "", MergeNone, "aarch64_sve_cntw", [IsOverloadNone]>;
 def SVCNTD_PAT : SInst<"svcntd_pat", "nI", "", MergeNone, "aarch64_sve_cntd", [IsOverloadNone]>;
+
+def SVCNTB : SInst<"svcntb", "n", "", MergeNone, "aarch64_sve_cntb", [IsAppendSVALL, IsOverloadNone]>;
+def SVCNTH : SInst<"svcnth", "n", "", MergeNone, "aarch64_sve_cnth", [IsAppendSVALL, IsOverloadNone]>;
+def SVCNTW : SInst<"svcntw", "n", "", MergeNone, "aarch64_sve_cntw", [IsAppendSVALL, IsOverloadNone]>;
+def SVCNTD : SInst<"svcntd", "n", "", MergeNone, "aarch64_sve_cntd", [IsAppendSVALL, IsOverloadNone]>;
 ////////////////////////////////////////////////////////////////////////////////
 // Integer arithmetic
 def SVDOT_LANE_S : SInst<"svdot_lane[_{d}]",  "ddqqi",  "il",   MergeNone, "aarch64_sve_sdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
 def SVDOT_LANE_U : SInst<"svdot_lane[_{d}]",  "ddqqi",  "UiUl", MergeNone, "aarch64_sve_udot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
 
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 WhileGE/GT
 let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {

clang/lib/CodeGen/CGBuiltin.cpp

else if (TypeFlags.isPrefetch())
     return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
   else if (Builtin->LLVMIntrinsic != 0) {
     if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
       InsertExplicitZeroOperand(Builder, Ty, Ops);
 
     if (TypeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
       InsertExplicitUndefOperand(Builder, Ty, Ops);
 
+    // Some ACLE builtins leave out the argument to specify the predicate
+    // pattern, which is expected to be expanded to an SV_ALL pattern.
+    if (TypeFlags.isAppendSVALL())
+      Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31));
+
     // Predicates must match the main datatype.
     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
       if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
         if (PredTy->getElementType()->isIntegerTy(1))
           Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags));
 
     // Splat scalar operand to vector (intrinsics with _n infix)
     if (TypeFlags.hasSplatOperand()) {

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntb.c

 // RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
+uint64_t test_svcntb()
+{
+  // CHECK-LABEL: test_svcntb
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 31)
+  // CHECK: ret i64 %[[INTRINSIC]]
+  return svcntb();
+}
+
 uint64_t test_svcntb_pat()
 {
   // CHECK-LABEL: test_svcntb_pat
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 0)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntb_pat(SV_POW2);
 }
 

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntd.c

 // RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
+uint64_t test_svcntd()
+{
+  // CHECK-LABEL: test_svcntd
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 31)
+  // CHECK: ret i64 %[[INTRINSIC]]
+  return svcntd();
+}
+
 uint64_t test_svcntd_pat()
 {
   // CHECK-LABEL: test_svcntd_pat
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 0)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntd_pat(SV_POW2);
 }
 

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnth.c

 // RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
+uint64_t test_svcnth()
+{
+  // CHECK-LABEL: test_svcnth
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 31)
+  // CHECK: ret i64 %[[INTRINSIC]]
+  return svcnth();
+}
+
 uint64_t test_svcnth_pat()
 {
   // CHECK-LABEL: test_svcnth_pat
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 0)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcnth_pat(SV_POW2);
 }
 

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntw.c

 // RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 
 #include <arm_sve.h>
 
+uint64_t test_svcntw()
+{
+  // CHECK-LABEL: test_svcntw
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 31)
+  // CHECK: ret i64 %[[INTRINSIC]]
+  return svcntw();
+}
+
 uint64_t test_svcntw_pat()
 {
   // CHECK-LABEL: test_svcntw_pat
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 0)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntw_pat(SV_POW2);
 }
 

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ptrue.c

+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+svbool_t test_svptrue_b8()
+{
+  // CHECK-LABEL: test_svptrue_b8
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_b8();
+}
+
+svbool_t test_svptrue_b16()
+{
+  // CHECK-LABEL: test_svptrue_b16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+  // CHECK: %[[CAST:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %[[INTRINSIC]])
+  // CHECK: ret <vscale x 16 x i1> %[[CAST]]
+  return svptrue_b16();
+}
+
+svbool_t test_svptrue_b32()
+{
+  // CHECK-LABEL: test_svptrue_b32
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  // CHECK: %[[CAST:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %[[INTRINSIC]])
+  // CHECK: ret <vscale x 16 x i1> %[[CAST]]
+  return svptrue_b32();
+}
+
+svbool_t test_svptrue_b64()
+{
+  // CHECK-LABEL: test_svptrue_b64
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  // CHECK: %[[CAST:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %[[INTRINSIC]])
+  // CHECK: ret <vscale x 16 x i1> %[[CAST]]
+  return svptrue_b64();
+}
+
+svbool_t test_svptrue_pat_b8()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 0)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_POW2);
+}
+
+svbool_t test_svptrue_pat_b8_1()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_1
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 1)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL1);
+}
+
+svbool_t test_svptrue_pat_b8_2()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_2
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 2)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL2);
+}
+
+svbool_t test_svptrue_pat_b8_3()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_3
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 3)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL3);
+}
+
+svbool_t test_svptrue_pat_b8_4()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_4
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 4)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL4);
+}
+
+svbool_t test_svptrue_pat_b8_5()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_5
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 5)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL5);
+}
+
+svbool_t test_svptrue_pat_b8_6()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_6
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 6)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL6);
+}
+
+svbool_t test_svptrue_pat_b8_7()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_7
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 7)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL7);
+}
+
+svbool_t test_svptrue_pat_b8_8()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_8
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 8)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL8);
+}
+
+svbool_t test_svptrue_pat_b8_9()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_9
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 9)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL16);
+}
+
+svbool_t test_svptrue_pat_b8_10()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_10
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 10)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL32);
+}
+
+svbool_t test_svptrue_pat_b8_11()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_11
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 11)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL64);
+}
+
+svbool_t test_svptrue_pat_b8_12()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_12
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 12)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL128);
+}
+
+svbool_t test_svptrue_pat_b8_13()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_13
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 13)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_VL256);
+}
+
+svbool_t test_svptrue_pat_b8_14()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_14
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 29)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_MUL4);
+}
+
+svbool_t test_svptrue_pat_b8_15()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_15
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 30)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_MUL3);
+}
+
+svbool_t test_svptrue_pat_b8_16()
+{
+  // CHECK-LABEL: test_svptrue_pat_b8_16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
+  return svptrue_pat_b8(SV_ALL);
+}
+
+svbool_t test_svptrue_pat_b16()
+{
+  // CHECK-LABEL: test_svptrue_pat_b16
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 0)
+  // CHECK: %[[CAST:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %[[INTRINSIC]])
+  // CHECK: ret <vscale x 16 x i1> %[[CAST]]
+  return svptrue_pat_b16(SV_POW2);
+}
+
+svbool_t test_svptrue_pat_b32()
+{
+  // CHECK-LABEL: test_svptrue_pat_b32
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 1)
+  // CHECK: %[[CAST:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %[[INTRINSIC]])
+  // CHECK: ret <vscale x 16 x i1> %[[CAST]]
+  return svptrue_pat_b32(SV_VL1);
+}
+
+svbool_t test_svptrue_pat_b64()
+{
+  // CHECK-LABEL: test_svptrue_pat_b64
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 2)
+  // CHECK: %[[CAST:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %[[INTRINSIC]])
+  // CHECK: ret <vscale x 16 x i1> %[[CAST]]
+  return svptrue_pat_b64(SV_VL2);
+}