[Clang][SVE2.1] Add clang support for prototypes using svcount_t

[CarolineConcatto]

Summary

In this patch it is used for the prototype:

• svptrue_c8 (and _c16/_c32/_c64)
  
  As described in: https://github.com/ARM-software/acle/pull/257

Patch by: Sander de Smalen <[e-mail]>

[david-arm]

LGTM!

[sdesmalen]

clang/include/clang/Basic/Builtins.def

42

Can we make 'Q' something to mean 'target type' and then use a second letter to clarify which exact target type it is, e.g.

// Q -> target builtin type, followed by a character to distinguish the builtin type
//   Qa -> AArch64 svcount_t builtin type.

That should make it more extensible for other target types in the future.

[CarolineConcatto]

-Address review's comments about Q target type.

[hassnaa-arm]

clang/include/clang/Basic/Builtins.def

42

Is that suggestion because maybe other targets don't support svcount ?

[sdesmalen]

clang/include/clang/Basic/Builtins.def

42

Correct, svcount_t is an AArch64 specific type.

[hassnaa-arm]

clang/include/clang/Basic/Builtins.def

42

Thanks.

[sdesmalen]

clang/include/clang/Basic/arm_sve.td

16

The typespec modifier in this file can remain Q rather than Qa. Can you change it back?

[hassnaa-arm]

clang/include/clang/Basic/arm_sve.td

16

what if there is a new type that is target-specific ?
At that case we will have to find a new character.
But if we now use Qa, we can use Qb for the new type.
Is that correct ?

[CarolineConcatto]

• Remove Qa from arm_sve.td and its dependencies

[CarolineConcatto]

clang/include/clang/Basic/arm_sve.td

16

This is now removed. I thought that should reflect what is in Builtins.def, but it does not.
Because Q is being defined in sve target we do not the 'a' in 'Qa'

clang/include/clang/Basic/Builtins.def

 //  F -> constant CFString
 //  G -> id
 //  H -> SEL
 //  M -> struct objc_super
 //  a -> __builtin_va_list
 //  A -> "reference" to __builtin_va_list
 //  V -> Vector, followed by the number of elements and the base type.
 //  q -> Scalable vector, followed by the number of elements and the base type.
+//  Q -> target builtin type, followed by a character to distinguish the builtin type
+//    Qa -> AArch64 svcount_t builtin type.
 //  E -> ext_vector, followed by the number of elements and the base type.
 //  X -> _Complex, followed by the base type.
 //  Y -> ptrdiff_t
 //  P -> FILE
 //  J -> jmp_buf
 //  SJ -> sigjmp_buf
 //  K -> ucontext_t
 //  p -> pid_t

clang/include/clang/Basic/arm_sve.td

 //===--- arm_sve.td - ARM SVE compiler interface ------------------------===//
 //
 //  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 //  See https://llvm.org/LICENSE.txt for license information.
 //  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file defines the TableGen definitions from which the ARM SVE header
 //  file will be generated.  See:
 //
 //      https://developer.arm.com/architectures/system-architectures/software-standards/acle
 //
 //===----------------------------------------------------------------------===//
 
 include "arm_sve_sme_incl.td"
 
 ////////////////////////////////////////////////////////////////////////////////
 // Loads
 
 // Load one vector (scalar base)
 def SVLD1   : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ld1">;
 def SVLD1SB : MInst<"svld1sb_{d}", "dPS", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ld1">;
 def SVLD1UB : MInst<"svld1ub_{d}", "dPW", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ld1">;

 def SVBEXT   : SInst<"svbext[_{d}]",   "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x">;
 def SVBEXT_N : SInst<"svbext[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x">;
 def SVBGRP   : SInst<"svbgrp[_{d}]",   "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">;
 def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">;
 }
 
 let TargetGuard = "sve2p1" in {
 def SVFCLAMP   : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [], []>;
+def SVPTRUE_COUNT  : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone], []>;
 }
 
 let TargetGuard = "sve2p1" in {
 def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil",     MergeNone, "aarch64_sve_sclamp", [], []>;
 def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>;
 }

clang/include/clang/Basic/arm_sve_sme_incl.td

 // h: half-float
 // d: double
 // b: bfloat
 
 // Typespec modifiers
 // ------------------
 // P: boolean
 // U: unsigned
+// Q: svcount
 
 // Prototype modifiers
 // -------------------
 // prototype: return (arg, arg, ...)
 //
 // 2,3,4: array of default vectors
 // v: void
 // x: vector of signed integers

 // U: const pointer to int32_t
 // V: const pointer to int64_t
 //
 // W: const pointer to uint8_t
 // X: const pointer to uint16_t
 // Y: const pointer to uint32_t
 // Z: const pointer to uint64_t
 
+// Prototype modifiers added for SVE2p1
+// }: svcount_t
+
 class MergeType<int val, string suffix=""> {
   int Value = val;
   string Suffix = suffix;
 }
 def MergeNone    : MergeType<0>;
 def MergeAny     : MergeType<1, "_x">;
 def MergeOp1     : MergeType<2, "_m">;
 def MergeZero    : MergeType<3, "_z">;

clang/lib/AST/ASTContext.cpp

case 'q': {
 
     QualType ElementType = DecodeTypeFromStr(Str, Context, Error,
                                              RequiresICE, false);
     assert(!RequiresICE && "Can't require vector ICE");
 
     Type = Context.getScalableVectorType(ElementType, NumElements);
     break;
   }
+  case 'Q': {
+    switch (*Str++) {
+    case 'a': {
+      Type = Context.SveCountTy;
+      break;
+    }
+    default:
+      llvm_unreachable("Unexpected target builtin type");
+    }
+    break;
+  }
   case 'V': {
     char *End;
     unsigned NumElements = strtoul(Str, &End, 10);
     assert(End != Str && "Missing vector size");
     Str = End;
 
     QualType ElementType = DecodeTypeFromStr(Str, Context, Error,
                                              RequiresICE, false);

clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ptrue.c

+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+
+#include <arm_sve.h>
+
+// CHECK-LABEL: @test_svptrue_c8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svptrue_c8v(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svptrue_c8(void) {
+  return svptrue_c8();
+}
+
+// CHECK-LABEL: @test_svptrue_c16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c16()
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svptrue_c16v(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c16()
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svptrue_c16(void) {
+  return svptrue_c16();
+}
+
+// CHECK-LABEL: @test_svptrue_c32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c32()
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svptrue_c32v(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c32()
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svptrue_c32(void) {
+  return svptrue_c32();
+}
+
+// CHECK-LABEL: @test_svptrue_c64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c64()
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svptrue_c64v(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c64()
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svptrue_c64(void) {
+  return svptrue_c64();
+}

clang/utils/TableGen/SveEmitter.cpp

public:
   unsigned getArg() const { return Arg; }
   unsigned getKind() const { return Kind; }
   unsigned getElementSizeInBits() const { return ElementSizeInBits; }
 };
 
 class SVEType {
   TypeSpec TS;
   bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
-  bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp;
+  bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
+      Svcount;
   unsigned Bitwidth, ElementBitwidth, NumVectors;
 
 public:
   SVEType() : SVEType(TypeSpec(), 'v') {}
 
   SVEType(TypeSpec TS, char CharMod)
       : TS(TS), Float(false), Signed(true), Immediate(false), Void(false),
         Constant(false), Pointer(false), BFloat(false), DefaultType(false),
         IsScalable(true), Predicate(false), PredicatePattern(false),
-        PrefetchOp(false), Bitwidth(128), ElementBitwidth(~0U), NumVectors(1) {
+        PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
+        NumVectors(1) {
     if (!TS.empty())
       applyTypespec();
     applyModifier(CharMod);
   }
 
   bool isPointer() const { return Pointer; }
   bool isVoidPointer() const { return Pointer && Void; }
   bool isSigned() const { return Signed; }
   bool isImmediate() const { return Immediate; }
   bool isScalar() const { return NumVectors == 0; }
   bool isVector() const { return NumVectors > 0; }
   bool isScalableVector() const { return isVector() && IsScalable; }
   bool isChar() const { return ElementBitwidth == 8; }
   bool isVoid() const { return Void & !Pointer; }
   bool isDefault() const { return DefaultType; }
   bool isFloat() const { return Float && !BFloat; }
   bool isBFloat() const { return BFloat && !Float; }
   bool isFloatingPoint() const { return Float || BFloat; }
-  bool isInteger() const { return !isFloatingPoint() && !Predicate; }
+  bool isInteger() const {
+    return !isFloatingPoint() && !Predicate && !Svcount;
+  }
   bool isScalarPredicate() const {
     return !isFloatingPoint() && Predicate && NumVectors == 0;
   }
   bool isPredicateVector() const { return Predicate; }
   bool isPredicatePattern() const { return PredicatePattern; }
   bool isPrefetchOp() const { return PrefetchOp; }
+  bool isSvcount() const { return Svcount; }
   bool isConstant() const { return Constant; }
   unsigned getElementSizeInBits() const { return ElementBitwidth; }
   unsigned getNumVectors() const { return NumVectors; }
 
   unsigned getNumElements() const {
     assert(ElementBitwidth != ~0U);
     return Bitwidth / ElementBitwidth;
   }
public:
 
   /// Return the type string for a BUILTIN() macro in Builtins.def.
   std::string getBuiltinTypeStr();
 
   /// Return the name, mangled with type information. The name is mangled for
   /// ClassS, so will add type suffixes such as _u32/_s32.
   std::string getMangledName() const { return mangleName(ClassS); }
 
+  /// As above, but mangles the LLVM name instead.
+  std::string getMangledLLVMName() const { return mangleLLVMName(); }
+
   /// Returns true if the intrinsic is overloaded, in that it should also generate
   /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
   /// 'svld1_u32(..)'.
   static bool isOverloadedIntrinsic(StringRef Name) {
     auto BrOpen = Name.find('[');
     auto BrClose = Name.find(']');
     return BrOpen != std::string::npos && BrClose != std::string::npos;
   }
public:
   }
 
   /// Emits the intrinsic declaration to the ostream.
   void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter) const;
 
 private:
   std::string getMergeSuffix() const { return MergeSuffix; }
   std::string mangleName(ClassKind LocalCK) const;
+  std::string mangleLLVMName() const;
   std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
                                    std::string Proto) const;
 };
 
 class SVEEmitter {
 private:
   // The reinterpret builtins are generated separately because they
   // need the cross product of all types (121 functions in total),

 std::string SVEType::builtin_str() const {
   std::string S;
   if (isVoid())
     return "v";
 
   if (isScalarPredicate())
     return "b";
 
+  if (isSvcount())
+    return "Qa";
+
   if (isVoidPointer())
     S += "v";
   else if (!isFloatingPoint())
     switch (ElementBitwidth) {
     case 1: S += "b"; break;
     case 8: S += "c"; break;
     case 16: S += "s"; break;
     case 32: S += "i"; break;
std::string SVEType::str() const {
 
   if (isPrefetchOp())
     return "enum svprfop";
 
   std::string S;
   if (Void)
     S += "void";
   else {
-    if (isScalableVector())
+    if (isScalableVector() || isSvcount())
       S += "sv";
     if (!Signed && !isFloatingPoint())
       S += "u";
 
     if (Float)
       S += "float";
+    else if (isSvcount())
+      S += "count";
     else if (isScalarPredicate() || isPredicateVector())
       S += "bool";
     else if (isBFloat())
       S += "bfloat";
     else
       S += "int";
 
-    if (!isScalarPredicate() && !isPredicateVector())
+    if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
       S += utostr(ElementBitwidth);
     if (!isScalableVector() && isVector())
       S += "x" + utostr(getNumElements());
     if (NumVectors > 1)
       S += "x" + utostr(NumVectors);
     if (!isScalarPredicate())
       S += "_t";
   }
 
   if (Constant)
     S += " const";
   if (Pointer)
     S += " *";
 
   return S;
 }
 void SVEType::applyTypespec() {
   for (char I : TS) {
     switch (I) {
+    case 'Q':
+      Svcount = true;
+      break;
     case 'P':
       Predicate = true;
       break;
     case 'U':
       Signed = false;
       break;
     case 'c':
       ElementBitwidth = 8;
void SVEType::applyModifier(char Mod) {
   case 'o':
     ElementBitwidth *= 4;
     break;
   case 'P':
     Signed = true;
     Float = false;
     BFloat = false;
     Predicate = true;
+    Svcount = false;
     Bitwidth = 16;
     ElementBitwidth = 1;
     break;
   case 's':
   case 'a':
     Bitwidth = ElementBitwidth;
     NumVectors = 0;
     break;
case 'L':
     Signed = false;
     Float = false;
     BFloat = false;
     Bitwidth = ElementBitwidth;
     NumVectors = 0;
     break;
   case 'u':
     Predicate = false;
+    Svcount = false;
     Signed = false;
     Float = false;
     BFloat = false;
     break;
   case 'x':
     Predicate = false;
+    Svcount = false;
     Signed = true;
     Float = false;
     BFloat = false;
     break;
   case 'i':
     Predicate = false;
+    Svcount = false;
     Float = false;
     BFloat = false;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     Signed = false;
     Immediate = true;
     break;
   case 'I':
     Predicate = false;
+    Svcount = false;
     Float = false;
     BFloat = false;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     Signed = true;
     Immediate = true;
     PredicatePattern = true;
     break;
   case 'J':
     Predicate = false;
+    Svcount = false;
     Float = false;
     BFloat = false;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     Signed = true;
     Immediate = true;
     PrefetchOp = true;
     break;
   case 'k':
     Predicate = false;
+    Svcount = false;
     Signed = true;
     Float = false;
     BFloat = false;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     break;
   case 'l':
     Predicate = false;
+    Svcount = false;
     Signed = true;
     Float = false;
     BFloat = false;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     break;
   case 'm':
     Predicate = false;
+    Svcount = false;
     Signed = false;
     Float = false;
     BFloat = false;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     break;
   case 'n':
     Predicate = false;
+    Svcount = false;
     Signed = false;
     Float = false;
     BFloat = false;
     ElementBitwidth = Bitwidth = 64;
     NumVectors = 0;
     break;
   case 'w':
     ElementBitwidth = 64;
void SVEType::applyModifier(char Mod) {
   case 'z':
     Signed = false;
     Float = false;
     BFloat = false;
     ElementBitwidth = 32;
     break;
   case 'O':
     Predicate = false;
+    Svcount = false;
     Float = true;
     ElementBitwidth = 16;
     break;
   case 'M':
     Predicate = false;
+    Svcount = false;
     Float = true;
     BFloat = false;
     ElementBitwidth = 32;
     break;
   case 'N':
     Predicate = false;
+    Svcount = false;
     Float = true;
     ElementBitwidth = 64;
     break;
   case 'Q':
     Constant = true;
     Pointer = true;
     Void = true;
     NumVectors = 0;
case 'F':
     Signed = false;
     break;
   case 'G':
     Pointer = true;
     ElementBitwidth = Bitwidth = 32;
     NumVectors = 0;
     Signed = false;
     break;
+  case '}':
+    Predicate = false;
+    Signed = true;
+    Svcount = true;
+    NumVectors = 0;
+    Float = false;
+    BFloat = false;
+    break;
   default:
     llvm_unreachable("Unhandled character!");
   }
 }
 
 
 //===----------------------------------------------------------------------===//
 // Intrinsic implementation
case '3':
       T = SVEType(TS, Proto[C - '0']);
       break;
     }
 
     // Replace templated arg with the right suffix (e.g. u32)
     std::string TypeCode;
     if (T.isInteger())
       TypeCode = T.isSigned() ? 's' : 'u';
+    else if (T.isSvcount())
+      TypeCode = 'c';
     else if (T.isPredicateVector())
       TypeCode = 'b';
     else if (T.isBFloat())
       TypeCode = "bf";
     else
       TypeCode = 'f';
     Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
   }
 
   return Ret;
 }
 
+std::string Intrinsic::mangleLLVMName() const {
+  std::string S = getLLVMName();
+
+  // Replace all {d} like expressions with e.g. 'u32'
+  return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
+}
+
 std::string Intrinsic::mangleName(ClassKind LocalCK) const {
   std::string S = getName();
 
   if (LocalCK == ClassG) {
     // Remove the square brackets and everything in between.
     while (S.find('[') != std::string::npos) {
       auto Start = S.find('[');
       auto End = S.find(']');
if (T.isFloat()) {
     }
   }
 
   if (T.isBFloat()) {
     assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
     return encodeEltType("EltTyBFloat16");
   }
 
-  if (T.isPredicateVector()) {
+  if (T.isPredicateVector() || T.isSvcount()) {
     switch (T.getElementSizeInBits()) {
     case 8:
       return encodeEltType("EltTyBool8");
     case 16:
       return encodeEltType("EltTyBool16");
     case 32:
       return encodeEltType("EltTyBool32");
     case 64:
void SVEEmitter::createHeader(raw_ostream &OS) {
   OS << "typedef __SVBool_t  svbool_t;\n";
   OS << "typedef __clang_svboolx2_t  svboolx2_t;\n";
   OS << "typedef __clang_svboolx4_t  svboolx4_t;\n\n";
 
   OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
   OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
   OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
 
+  OS << "typedef __SVCount_t svcount_t;\n\n";
+
   OS << "enum svpattern\n";
   OS << "{\n";
   OS << "  SV_POW2 = 0,\n";
   OS << "  SV_VL1 = 1,\n";
   OS << "  SV_VL2 = 2,\n";
   OS << "  SV_VL3 = 3,\n";
   OS << "  SV_VL4 = 4,\n";
   OS << "  SV_VL5 = 5,\n";
for (auto &Def : Defs) {
     // Builtins only exist for non-overloaded intrinsics, overloaded
     // declarations only live in the header file.
     if (Def->getClassKind() == ClassG)
       continue;
 
     uint64_t Flags = Def->getFlags();
     auto FlagString = std::to_string(Flags);
 
-    std::string LLVMName = Def->getLLVMName();
+    std::string LLVMName = Def->getMangledLLVMName();
     std::string Builtin = Def->getMangledName();
     if (!LLVMName.empty())
       OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
          << "),\n";
     else
       OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
   }
   OS << "#endif\n\n";