[Clang][AArch64][SME] Add ZA zeroing intrinsics

[bryanpkc]

Summary

This patch adds support for the following SME ACLE intrinsics (as defined
in https://arm-software.github.io/acle/main/acle.html):

• svzero_mask_za
• svzero_za

Co-authored-by: Sagar Kulkarni <[e-mail]>

[bryanpkc]

Update patch with more context.

[sdesmalen]

clang/lib/CodeGen/CGBuiltin.cpp

9937–9938

Given that the type flags are a little precious (we've already used 42 out of the 64 bits) and there only being a single intrinsic for svzero, can you do something similar to what's done in EmitAArch64SVEBuiltinExpr and create a switch on BuiltinID instead?

clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c

14

These tests are missing some kind of ARM_SHARED_ZA_ATTR macro (similar to what you've done for ARM_STREAMING_ATTR in D127910 and D128648), because svzero requires ZA state to be available.

clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp

208

svzero is streaming-compatible, but does require shared_za. So you should probably change this to ARM_SHARED_ZA_ATTR (and add the #define for it)

[bryanpkc]

Addressed @sdesmalen's comments.

[bryanpkc]

Rebased. NFC.

[sdesmalen]

Other than my comment on the test, the patch looks good to me.

clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c

21

The new keyword attributes are a bit stricter on the placement.

Because this is a type attribute (as opposed to a declaration attribute), it should be placed like this:

void test_svzero_mask_za() ARM_SHARED_ZA_ATTR {

As I mentioned on D128648, I think it's better to remove all these macros for now and only add the attributes when we add diagnostics in Sema for missing attributes.

[bryanpkc]

Removed the attribute macro as suggested.

[sdesmalen]

LGTM

clang/include/clang/Basic/arm_sme.td

multiclass ZAWrite<string n_suffix, string t, string i_prefix, list<ImmCheck> ch> {
   }
 }
 
 defm SVWRITE_ZA8 : ZAWrite<"za8", "cUc", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_15>]>;
 defm SVWRITE_ZA16 : ZAWrite<"za16", "sUshb", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_1>, ImmCheck<2, ImmCheck0_7>]>;
 defm SVWRITE_ZA32 : ZAWrite<"za32", "iUif", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_3>, ImmCheck<2, ImmCheck0_3>]>;
 defm SVWRITE_ZA64 : ZAWrite<"za64", "lUld", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_7>, ImmCheck<2, ImmCheck0_1>]>;
 defm SVWRITE_ZA128 : ZAWrite<"za128", "csilUcUsUiUlhbfd", "aarch64_sme_writeq", [ImmCheck<0, ImmCheck0_15>, ImmCheck<2, ImmCheck0_0>]>;
+
+////////////////////////////////////////////////////////////////////////////////
+// SME - Zero
+
+let TargetGuard = "sme" in {
+  def SVZERO_MASK_ZA : SInst<"svzero_mask_za", "vi", "", MergeNone, "aarch64_sme_zero",
+                             [IsOverloadNone, IsStreamingCompatible, IsSharedZA],
+                             [ImmCheck<0, ImmCheck0_255>]>;
+  def SVZERO_ZA      : SInst<"svzero_za", "v", "", MergeNone, "aarch64_sme_zero",
+                             [IsOverloadNone, IsStreamingCompatible, IsSharedZA]>;
+}

clang/include/clang/Basic/arm_sve_sme_incl.td

 def ImmCheckComplexRot90_270    : ImmCheckType<10>; // [90,270]
 def ImmCheckComplexRotAll90     : ImmCheckType<11>; // [0, 90, 180,270]
 def ImmCheck0_13                : ImmCheckType<12>; // 0..13
 def ImmCheck0_1                 : ImmCheckType<13>; // 0..1
 def ImmCheck0_2                 : ImmCheckType<14>; // 0..2
 def ImmCheck0_3                 : ImmCheckType<15>; // 0..3
 def ImmCheck0_0                 : ImmCheckType<16>; // 0..0
 def ImmCheck0_15                : ImmCheckType<17>; // 0..15
+def ImmCheck0_255               : ImmCheckType<18>; // 0..255
 
 class ImmCheck<int arg, ImmCheckType kind, int eltSizeArg = -1> {
   int Arg = arg;
   int EltSizeArg = eltSizeArg;
   ImmCheckType Kind = kind;
 }
 
 class Inst<string n, string p, string t, MergeType mt, string i,

clang/lib/CodeGen/CGBuiltin.cpp

Value *CodeGenFunction::EmitSMEReadWrite(SVETypeFlags TypeFlags,
   } else if (TypeFlags.isWriteZA()) {
     Ops[1] = EmitTileslice(Ops[2], Ops[1]);
     Ops[2] = EmitSVEPredicateCast(Ops[3], VecTy);
     Ops.erase(&Ops[3]);
   }
   return Builder.CreateCall(F, Ops);
 }
 
+Value *CodeGenFunction::EmitSMEZero(SVETypeFlags TypeFlags,
+                                    SmallVectorImpl<Value *> &Ops,
+                                    unsigned IntID) {
+  // svzero_za() intrinsic zeros the entire za tile and has no paramters.
+  if (Ops.size() == 0)
+    Ops.push_back(llvm::ConstantInt::get(Int32Ty, 255));
+  Function *F = CGM.getIntrinsic(IntID, {});
+  return Builder.CreateCall(F, Ops);
+}
+
 // Limit the usage of scalable llvm IR generated by the ACLE by using the
 // sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat.
 Value *CodeGenFunction::EmitSVEDupX(Value *Scalar, llvm::Type *Ty) {
   return Builder.CreateVectorSplat(
       cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
 }
 
 Value *CodeGenFunction::EmitSVEDupX(Value* Scalar) {
Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID,
 
   auto *Builtin = findARMVectorIntrinsicInMap(AArch64SMEIntrinsicMap, BuiltinID,
                                               AArch64SMEIntrinsicsProvenSorted);
   SVETypeFlags TypeFlags(Builtin->TypeModifier);
   if (TypeFlags.isLoad() || TypeFlags.isStore())
     return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
   else if (TypeFlags.isReadZA() || TypeFlags.isWriteZA())
     return EmitSMEReadWrite(TypeFlags, Ops, Builtin->LLVMIntrinsic);
+  else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
+           BuiltinID == SME::BI__builtin_sme_svzero_za)
+    return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
 
   /// Should not happen
   return nullptr;
 }
 
 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
                                                const CallExpr *E,
                                                llvm::Triple::ArchType Arch) {

clang/lib/CodeGen/CodeGenFunction.h

public:
   llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
 
   llvm::Value *EmitSMELd1St1(SVETypeFlags TypeFlags,
                              llvm::SmallVectorImpl<llvm::Value *> &Ops,
                              unsigned IntID);
   llvm::Value *EmitSMEReadWrite(SVETypeFlags TypeFlags,
                                 llvm::SmallVectorImpl<llvm::Value *> &Ops,
                                 unsigned IntID);
+  llvm::Value *EmitSMEZero(SVETypeFlags TypeFlags,
+                           llvm::SmallVectorImpl<llvm::Value *> &Ops,
+                           unsigned IntID);
   llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
 
   llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
                                       llvm::Triple::ArchType Arch);
   llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
 
   llvm::Value *BuildVector(ArrayRef<llvm::Value*> Ops);
   llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);

clang/lib/Sema/SemaChecking.cpp

for (auto &I : ImmChecks) {
     case SVETypeFlags::ImmCheck0_0:
       if (SemaBuiltinConstantArgRange(TheCall, ArgNum, 0, 0))
         HasError = true;
       break;
     case SVETypeFlags::ImmCheck0_15:
       if (SemaBuiltinConstantArgRange(TheCall, ArgNum, 0, 15))
         HasError = true;
       break;
+    case SVETypeFlags::ImmCheck0_255:
+      if (SemaBuiltinConstantArgRange(TheCall, ArgNum, 0, 255))
+        HasError = true;
+      break;
     }
   }
 
   return HasError;
 }
 
 bool Sema::CheckNeonBuiltinFunctionCall(const TargetInfo &TI,
                                         unsigned BuiltinID, CallExpr *TheCall) {

clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c

+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
+// RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
+// RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -o /dev/null %s
+
+#include <arm_sme_draft_spec_subject_to_change.h>
+
+#ifdef DISABLE_SME_ATTRIBUTES
+#define ARM_SHARED_ZA_ATTR
+#else
+#define ARM_SHARED_ZA_ATTR __attribute__((arm_shared_za))
+#endif
+
+// CHECK-C-LABEL: @test_svzero_mask_za(
+// CHECK-CXX-LABEL: @_Z19test_svzero_mask_zav(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 0)
+// CHECK-NEXT:    ret void
+//
+ARM_SHARED_ZA_ATTR void test_svzero_mask_za() {
+  svzero_mask_za(0);
+}
+
+// CHECK-C-LABEL: @test_svzero_mask_za_1(
+// CHECK-CXX-LABEL: @_Z21test_svzero_mask_za_1v(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 176)
+// CHECK-NEXT:    ret void
+//
+ARM_SHARED_ZA_ATTR void test_svzero_mask_za_1() {
+  svzero_mask_za(176);
+}
+
+// CHECK-C-LABEL: @test_svzero_mask_za_2(
+// CHECK-CXX-LABEL: @_Z21test_svzero_mask_za_2v(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 255)
+// CHECK-NEXT:    ret void
+//
+ARM_SHARED_ZA_ATTR void test_svzero_mask_za_2() {
+  svzero_mask_za(255);
+}
+
+// CHECK-C-LABEL: @test_svzero_za(
+// CHECK-CXX-LABEL: @_Z14test_svzero_zav(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 255)
+// CHECK-NEXT:    ret void
+//
+ARM_SHARED_ZA_ATTR void test_svzero_za() {
+  svzero_za();
+}

clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp

 // REQUIRES: aarch64-registered-target
 
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -fsyntax-only -verify -verify-ignore-unexpected=error %s
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -fsyntax-only -verify -verify-ignore-unexpected=error %s
 
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
 #else
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
 #include <arm_sme_draft_spec_subject_to_change.h>
 
 #ifdef DISABLE_SME_ATTRIBUTES
 #define ARM_STREAMING_ATTR
+#define ARM_SHARED_ZA_ATTR
 #else
 #define ARM_STREAMING_ATTR __attribute__((arm_streaming))
+#define ARM_SHARED_ZA_ATTR __attribute__((arm_shared_za))
 #endif
 
 ARM_STREAMING_ATTR
 void test_range_0_0(svbool_t pg, void *ptr) {
   // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 0]}}
   SVE_ACLE_FUNC(svld1_hor_za8,,,)(-1, -1, 0, pg, ptr);
   // expected-error@+1 {{argument value 1 is outside the valid range [0, 0]}}
   SVE_ACLE_FUNC(svst1_ver_za8,,,)(1, -1, 15, pg, ptr);
void test_range_0_15(svbool_t pg, void *ptr) {
   // expected-error@+1 {{argument value 16 is outside the valid range [0, 15]}}
   SVE_ACLE_FUNC(svwrite_ver_za128, _s8, _m,)(16, -1, 0, pg, svundef_s8());
   // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 15]}}
   SVE_ACLE_FUNC(svwrite_hor_za8, _s8, _m,)(0, -1, -1, pg, svundef_s8());
   // expected-error@+1 {{argument value 16 is outside the valid range [0, 15]}}
   SVE_ACLE_FUNC(svwrite_ver_za8, _s8, _m,)(0, -1, 16, pg, svundef_s8());
 }
 
+ARM_SHARED_ZA_ATTR
+void test_range_0_255(svbool_t pg, void *ptr) {
+  // expected-error@+1 {{argument value 256 is outside the valid range [0, 255]}}
+  SVE_ACLE_FUNC(svzero_mask_za,,,)(256);
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 255]}}
+  SVE_ACLE_FUNC(svzero_mask_za,,,)(-1);
+}
+
 ARM_STREAMING_ATTR
 void test_constant(uint64_t u64, svbool_t pg, void *ptr) {
   SVE_ACLE_FUNC(svld1_hor_za8,,,)(u64, u64, 0, pg, ptr);  // expected-error {{argument to 'svld1_hor_za8' must be a constant integer}}
   SVE_ACLE_FUNC(svld1_ver_za16,,,)(0, u64, u64, pg, ptr); // expected-error {{argument to 'svld1_ver_za16' must be a constant integer}}
   SVE_ACLE_FUNC(svst1_hor_za32,,,)(u64, u64, 0, pg, ptr); // expected-error {{argument to 'svst1_hor_za32' must be a constant integer}}
   SVE_ACLE_FUNC(svst1_ver_za64,,,)(0, u64, u64, pg, ptr); // expected-error {{argument to 'svst1_ver_za64' must be a constant integer}}
   SVE_ACLE_FUNC(svld1_hor_vnum_za8,,,)(u64, u64, 0, pg, ptr, u64);  // expected-error {{argument to 'svld1_hor_vnum_za8' must be a constant integer}}
   SVE_ACLE_FUNC(svld1_ver_vnum_za16,,,)(0, u64, u64, pg, ptr, u64); // expected-error {{argument to 'svld1_ver_vnum_za16' must be a constant integer}}
   SVE_ACLE_FUNC(svst1_hor_vnum_za32,,,)(u64, u64, 0, pg, ptr, u64); // expected-error {{argument to 'svst1_hor_vnum_za32' must be a constant integer}}
   SVE_ACLE_FUNC(svst1_ver_vnum_za64,,,)(0, u64, u64, pg, ptr, u64); // expected-error {{argument to 'svst1_ver_vnum_za64' must be a constant integer}}
 
   SVE_ACLE_FUNC(svread_hor_za8, _s8, _m,)(svundef_s8(), pg, 0, u64, u64);     // expected-error-re {{argument to 'svread_hor_za8{{.*}}_m' must be a constant integer}}
   SVE_ACLE_FUNC(svread_ver_za16, _s16, _m,)(svundef_s16(), pg, u64, u64, 0);  // expected-error-re {{argument to 'svread_ver_za16{{.*}}_m' must be a constant integer}}
   SVE_ACLE_FUNC(svwrite_hor_za32, _s32, _m,)(0, u64, u64, pg, svundef_s32()); // expected-error-re {{argument to 'svwrite_hor_za32{{.*}}_m' must be a constant integer}}
   SVE_ACLE_FUNC(svwrite_ver_za64, _s64, _m,)(u64, u64, 0, pg, svundef_s64()); // expected-error-re {{argument to 'svwrite_ver_za64{{.*}}_m' must be a constant integer}}
 }