[clang][SveEmitter] SVE builtins for `svusdot` and `svsudot` ACLE.

[fpetrogalli]

Summary

Intrinsics, guarded by __ARM_FEATURE_SVE_MATMUL_INT8:

• svusdot[_s32]
• svusdot[_n_s32]
• svusdot_lane[_s32]
• svsudot[_s32]
• svsudot[_n_s32]
• svsudot_lane[_s32]

[efriedma]

LGTM with one small comment.

clang/include/clang/Basic/arm_sve.td

1249

These intrinsics aren't overloaded; maybe consider writing the actual type int8_t/uint8_t, instead of introducing "unsigned scalar of 1/4 width element type"? If there's some reason the current form is better, that's fine, though.

[fpetrogalli]

clang/include/clang/Basic/arm_sve.td

1249

I am probably missing something, but I can't see a type specifier that is not overloaded and that renders to 8-bit elements?

I have opted for adding "unsigned scalar of 1/4 width element type" because it keeps things symmetric with the vector equivalent, signed and unsigned.

// r: scalar of 1/4 width element type (splat to vector type)
// @: unsigned scalar of 1/4 width element type (splat to vector type)
[..]
// b: 1/4 width unsigned elements, 4x element count
// q: 1/4 width elements, 4x element count

[fpetrogalli]

Fix formatting of code in test. NFC

[fpetrogalli]

clang/include/clang/Basic/arm_sve.td

1249

@efriedma , are you satisfied with this answer? are you happy for me to submit this as it is?

[efriedma]

clang/include/clang/Basic/arm_sve.td

1249

Yes, it's fine.

clang/include/clang/Basic/TargetBuiltins.h

#undef LLVM_GET_SVE_IMMCHECKTYPES
     bool isOverloadDefault() const { return !(Flags & OverloadKindMask); }
     bool isOverloadWhileRW() const { return Flags & IsOverloadWhileRW; }
     bool isOverloadCvt() const { return Flags & IsOverloadCvt; }
     bool isPrefetch() const { return Flags & IsPrefetch; }
     bool isReverseCompare() const { return Flags & ReverseCompare; }
     bool isAppendSVALL() const { return Flags & IsAppendSVALL; }
     bool isInsertOp1SVALL() const { return Flags & IsInsertOp1SVALL; }
     bool isGatherPrefetch() const { return Flags & IsGatherPrefetch; }
+    bool isReverseUSDOT() const { return Flags & ReverseUSDOT; }
 
     uint64_t getBits() const { return Flags; }
     bool isFlagSet(uint64_t Flag) const { return Flags & Flag; }
   };
 
   /// Hexagon builtins
   namespace Hexagon {
     enum {

clang/include/clang/Basic/arm_sve.td

 // u: vector of unsigned integers
 // d: default
 // c: const pointer type
 // P: predicate type
 // s: scalar of element type
 // a: scalar of element type (splat to vector type)
 // R: scalar of 1/2 width element type (splat to vector type)
 // r: scalar of 1/4 width element type (splat to vector type)
+// @: unsigned scalar of 1/4 width element type (splat to vector type)
 // e: 1/2 width unsigned elements, 2x element count
 // b: 1/4 width unsigned elements, 4x element count
 // h: 1/2 width elements, 2x element count
 // q: 1/4 width elements, 4x element count
 // o: 4x width elements, 1/4 element count
 //
 // w: vector of element type promoted to 64bits, vector maintains
 //    signedness of its element type.

 def IsOverloadCvt             : FlagType<0x00800000>; // Use {typeof(operand0), typeof(last operand)} as overloaded types.
 def OverloadKindMask          : FlagType<0x00E00000>; // When the masked values are all '0', the default type is used as overload type.
 def IsByteIndexed             : FlagType<0x01000000>;
 def IsAppendSVALL             : FlagType<0x02000000>; // Appends SV_ALL as the last operand.
 def IsInsertOp1SVALL          : FlagType<0x04000000>; // Inserts SV_ALL as the second operand.
 def IsPrefetch                : FlagType<0x08000000>; // Contiguous prefetches.
 def IsGatherPrefetch          : FlagType<0x10000000>;
 def ReverseCompare            : FlagType<0x20000000>; // Compare operands must be swapped.
+def ReverseUSDOT              : FlagType<0x40000000>; // Unsigned/signed operands must be swapped.
 
 // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h
 class ImmCheckType<int val> {
   int Value = val;
 }
 def ImmCheck0_31                : ImmCheckType<0>;  // 0..31 (used for e.g. predicate patterns)
 def ImmCheck1_16                : ImmCheckType<1>;  // 1..16
 def ImmCheckExtract             : ImmCheckType<2>;  // 0..(2048/sizeinbits(elt) - 1)

 def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n64">;
 def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32">;
 def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64">;
 
 let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_INT8)" in {
 def SVMLLA_S32   : SInst<"svmmla[_s32]",   "ddqq","i",  MergeNone, "aarch64_sve_smmla">;
 def SVMLLA_U32   : SInst<"svmmla[_u32]",   "ddqq","Ui", MergeNone, "aarch64_sve_ummla">;
 def SVUSMLLA_S32 : SInst<"svusmmla[_s32]", "ddbq","i",  MergeNone, "aarch64_sve_usmmla">;
+
+def SVUSDOT_S    : SInst<"svusdot[_s32]",    "ddbq", "i",       MergeNone, "aarch64_sve_usdot">;
+def SVUSDOT_N_S  : SInst<"svusdot[_n_s32]",  "ddbr", "i",       MergeNone, "aarch64_sve_usdot">;
+def SVSUDOT_S    : SInst<"svsudot[_s32]",    "ddqb", "i",       MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>;
+def SVSUDOT_N_S  : SInst<"svsudot[_n_s32]",  "ddq@", "i",       MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>;
+
+def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]",  "ddbqi",  "i",   MergeNone, "aarch64_sve_usdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
+def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]",  "ddqbi",  "i",   MergeNone, "aarch64_sve_sudot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
 }
 
 let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP32)" in {
 def SVMLLA_F32 : SInst<"svmmla[_f32]", "dddd","f", MergeNone, "aarch64_sve_fmmla">;
 }
 
 let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64)" in {
 def SVMLLA_F64 : SInst<"svmmla[_f64]", "dddd","d", MergeNone, "aarch64_sve_fmmla">;

clang/lib/CodeGen/CGBuiltin.cpp

else if (Builtin->LLVMIntrinsic != 0) {
     if (TypeFlags.hasSplatOperand()) {
       unsigned OpNo = TypeFlags.getSplatOperand();
       Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
     }
 
     if (TypeFlags.isReverseCompare())
       std::swap(Ops[1], Ops[2]);
 
+    if (TypeFlags.isReverseUSDOT())
+      std::swap(Ops[1], Ops[2]);
+
     // Predicated intrinsics with _z suffix need a select w/ zeroinitializer.
     if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
       llvm::Type *OpndTy = Ops[1]->getType();
       auto *SplatZero = Constant::getNullValue(OpndTy);
       Function *Sel = CGM.getIntrinsic(Intrinsic::aarch64_sve_sel, OpndTy);
       Ops[1] = Builder.CreateCall(Sel, {Ops[0], Ops[1], SplatZero});
     }
 

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sudot.c

+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svint32_t test_svsudot_s32(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_s32
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %z, <vscale x 16 x i8> %y)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot, _s32, , )(x, y, z);
+}
+
+svint32_t test_svsudot_n_s32(svint32_t x, svint8_t y, uint8_t z) {
+  // CHECK-LABEL: test_svsudot_n_s32
+  // CHECK: %[[SPLAT:.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.dup.x.nxv16i8(i8 %z)
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %[[SPLAT]], <vscale x 16 x i8> %y)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot, _n_s32, , )(x, y, z);
+}
+
+svint32_t test_svsudot_lane_s32_0(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_lane_s32_0
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 0)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot_lane, _s32, , )(x, y, z, 0);
+}
+
+svint32_t test_svsudot_lane_s32_1(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_lane_s32_1
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 1)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot_lane, _s32, , )(x, y, z, 1);
+}
+
+svint32_t test_svsudot_lane_s32_2(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_lane_s32_2
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 2)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot_lane, _s32, , )(x, y, z, 2);
+}
+
+svint32_t test_svsudot_lane_s32_3(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_lane_s32_3
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 3)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot_lane, _s32, , )(x, y, z, 3);
+}

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_usdot.c

+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svint32_t test_svusdot_s32(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_s32
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot, _s32, , )(x, y, z);
+}
+
+svint32_t test_svusdot_n_s32(svint32_t x, svuint8_t y, int8_t z) {
+  // CHECK-LABEL: test_svusdot_n_s32
+  // CHECK: %[[SPLAT:.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.dup.x.nxv16i8(i8 %z)
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %[[SPLAT]])
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot, _n_s32, , )(x, y, z);
+}
+
+svint32_t test_svusdot_lane_s32_0(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_lane_s32_0
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 0)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot_lane, _s32, , )(x, y, z, 0);
+}
+
+svint32_t test_svusdot_lane_s32_1(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_lane_s32_1
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 1)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot_lane, _s32, , )(x, y, z, 1);
+}
+
+svint32_t test_svusdot_lane_s32_2(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_lane_s32_2
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 2)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot_lane, _s32, , )(x, y, z, 2);
+}
+
+svint32_t test_svusdot_lane_s32_3(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_lane_s32_3
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 3)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot_lane, _s32, , )(x, y, z, 3);
+}

clang/utils/TableGen/SveEmitter.cpp

static bool isOverloadedIntrinsic(StringRef Name) {
     auto BrOpen = Name.find("[");
     auto BrClose = Name.find(']');
     return BrOpen != std::string::npos && BrClose != std::string::npos;
   }
 
   /// Return true if the intrinsic takes a splat operand.
   bool hasSplat() const {
     // These prototype modifiers are described in arm_sve.td.
-    return Proto.find_first_of("ajfrKLR") != std::string::npos;
+    return Proto.find_first_of("ajfrKLR@") != std::string::npos;
   }
 
   /// Return the parameter index of the splat operand.
   unsigned getSplatIdx() const {
     // These prototype modifiers are described in arm_sve.td.
-    auto Idx = Proto.find_first_of("ajfrKLR");
+    auto Idx = Proto.find_first_of("ajfrKLR@");
     assert(Idx != std::string::npos && Idx > 0 &&
            "Prototype has no splat operand");
     return Idx - 1;
   }
 
   /// Emits the intrinsic declaration to the ostream.
   void emitIntrinsic(raw_ostream &OS) const;
 
void SVEType::applyModifier(char Mod) {
   case 'R':
     ElementBitwidth /= 2;
     NumVectors = 0;
     break;
   case 'r':
     ElementBitwidth /= 4;
     NumVectors = 0;
     break;
+  case '@':
+    Signed = false;
+    Float = false;
+    ElementBitwidth /= 4;
+    NumVectors = 0;
+    break;
   case 'K':
     Signed = true;
     Float = false;
     Bitwidth = ElementBitwidth;
     NumVectors = 0;
     break;
   case 'L':
     Signed = false;