[Clang][AArch64][SME] Add ZA zeroing intrinsics

[bryanpkc]

Summary

This patch adds support for the following SME ACLE intrinsics (as defined
in https://arm-software.github.io/acle/main/acle.html):

• svzero_mask_za
• svzero_za

Co-authored-by: Sagar Kulkarni <[e-mail]>

[bryanpkc]

Update patch with more context.

[sdesmalen]

clang/lib/CodeGen/CGBuiltin.cpp

9567–9568

Given that the type flags are a little precious (we've already used 42 out of the 64 bits) and there only being a single intrinsic for svzero, can you do something similar to what's done in EmitAArch64SVEBuiltinExpr and create a switch on BuiltinID instead?

clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c

14

These tests are missing some kind of ARM_SHARED_ZA_ATTR macro (similar to what you've done for ARM_STREAMING_ATTR in D127910 and D128648), because svzero requires ZA state to be available.

clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp

206 ↗

(On Diff #526351)

svzero is streaming-compatible, but does require shared_za. So you should probably change this to ARM_SHARED_ZA_ATTR (and add the #define for it)

[bryanpkc]

Addressed @sdesmalen's comments.

[bryanpkc]

Rebased. NFC.

[sdesmalen]

Other than my comment on the test, the patch looks good to me.

clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c

21

The new keyword attributes are a bit stricter on the placement.

Because this is a type attribute (as opposed to a declaration attribute), it should be placed like this:

void test_svzero_mask_za() ARM_SHARED_ZA_ATTR {

As I mentioned on D128648, I think it's better to remove all these macros for now and only add the attributes when we add diagnostics in Sema for missing attributes.

[bryanpkc]

Removed the attribute macro as suggested.

[sdesmalen]

LGTM

clang/include/clang/Basic/TargetBuiltins.h

#undef LLVM_GET_SVE_IMMCHECKTYPES
     bool isTupleCreate() const { return Flags & IsTupleCreate; }
     bool isTupleGet() const { return Flags & IsTupleGet; }
     bool isTupleSet() const { return Flags & IsTupleSet; }
     bool isSME() const { return Flags & IsSME; }
     bool isSMELd1() const { return Flags & IsSMELd1; }
     bool isSMESt1() const { return Flags & IsSMESt1; }
     bool isSMERead() const { return Flags & IsSMERead; }
     bool isSMEWrite() const { return Flags & IsSMEWrite; }
+    bool isSMEZero() const { return Flags & IsSMEZero; }
 
     uint64_t getBits() const { return Flags; }
     bool isFlagSet(uint64_t Flag) const { return Flags & Flag; }
   };
 
   /// Hexagon builtins
   namespace Hexagon {
     enum {

clang/include/clang/Basic/arm_sve.td

 def IsTupleCreate             : FlagType<0x100000000>;
 def IsTupleGet                : FlagType<0x200000000>;
 def IsTupleSet                : FlagType<0x400000000>;
 def IsSME                     : FlagType<0x800000000>;
 def IsSMELd1                  : FlagType<0x1000000000>;
 def IsSMESt1                  : FlagType<0x2000000000>;
 def IsSMERead                 : FlagType<0x4000000000>;
 def IsSMEWrite                : FlagType<0x8000000000>;
+def IsSMEZero                 : FlagType<0x10000000000>;
 
 // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h
 class ImmCheckType<int val> {
   int Value = val;
 }
 def ImmCheck0_31                : ImmCheckType<0>;  // 0..31 (used for e.g. predicate patterns)
 def ImmCheck1_16                : ImmCheckType<1>;  // 1..16
 def ImmCheckExtract             : ImmCheckType<2>;  // 0..(2048/sizeinbits(elt) - 1)

 def SVWRITE_HOR_ZA32 : SInst<"svwrite_hor_za32[_{d}]", "vimiPd", "iUif", MergeOp1, "aarch64_sme_write_horiz", [IsSME, IsSMEWrite]>;
 def SVWRITE_HOR_ZA64 : SInst<"svwrite_hor_za64[_{d}]", "vimiPd", "lUld", MergeOp1, "aarch64_sme_write_horiz", [IsSME, IsSMEWrite]>;
 def SVWRITE_HOR_ZA128 : SInst<"svwrite_hor_za128[_{d}]", "vimiPd", "csilUcUsUiUlhbfd", MergeOp1, "aarch64_sme_writeq_horiz", [IsSME, IsSMEWrite]>;
 def SVWRITE_VER_ZA8 : SInst<"svwrite_ver_za8[_{d}]", "vimiPd", "cUc", MergeOp1, "aarch64_sme_write_vert", [IsSME, IsSMEWrite]>;
 def SVWRITE_VER_ZA16 : SInst<"svwrite_ver_za16[_{d}]", "vimiPd", "sUshb", MergeOp1, "aarch64_sme_write_vert", [IsSME, IsSMEWrite]>;
 def SVWRITE_VER_ZA32 : SInst<"svwrite_ver_za32[_{d}]", "vimiPd", "iUif", MergeOp1, "aarch64_sme_write_vert", [IsSME, IsSMEWrite]>;
 def SVWRITE_VER_ZA64 : SInst<"svwrite_ver_za64[_{d}]", "vimiPd", "lUld", MergeOp1, "aarch64_sme_write_vert", [IsSME, IsSMEWrite]>;
 def SVWRITE_VER_ZA128 : SInst<"svwrite_ver_za128[_{d}]", "vimiPd", "csilUcUsUiUlhbfd", MergeOp1, "aarch64_sme_writeq_vert", [IsSME, IsSMEWrite]>;
+
+
+////////////////////////////////////////////////////////////////////////////////
+// SME - Zero
+
+def SVZERO_MASK_ZA : SInst<"svzero_mask_za", "vi", "", MergeNone, "aarch64_sme_zero", [IsOverloadNone, IsSME, IsSMEZero]>;
+def SVZERO_ZA      : SInst<"svzero_za", "v", "", MergeNone, "aarch64_sme_zero", [IsOverloadNone, IsSME, IsSMEZero]>;

clang/lib/CodeGen/CGBuiltin.cpp

Value *CodeGenFunction::EmitSMEWrite(SVETypeFlags TypeFlags,
   Ops[1] = EmitTileslice(Ops[2], Ops[1]);
   Ops[2] = EmitSVEPredicateCast(Ops[3], getSVEType(TypeFlags));
   // Erase the slice_offset operand.
   Ops.erase(&Ops[3]);
   Function *F = CGM.getIntrinsic(IntID, {getSVEType(TypeFlags)});
   return Builder.CreateCall(F, Ops);
 }
 
+Value *CodeGenFunction::EmitSMEZero(SVETypeFlags TypeFlags,
+                                    SmallVectorImpl<Value *> &Ops,
+                                    unsigned IntID) {
+  // svzero_za() intrinsic zeros the entire za tile and has no paramters.
+  if (Ops.size() == 0)
+    Ops.push_back(llvm::ConstantInt::get(Int64Ty, 255));
+  else
+    Ops[0] = Builder.CreateIntCast(Ops[0], Int64Ty, false);
+  Function *F = CGM.getIntrinsic(IntID, {});
+  return Builder.CreateCall(F, Ops);
+}
+
 // Limit the usage of scalable llvm IR generated by the ACLE by using the
 // sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat.
 Value *CodeGenFunction::EmitSVEDupX(Value *Scalar, llvm::Type *Ty) {
   auto F = CGM.getIntrinsic(Intrinsic::aarch64_sve_dup_x, Ty);
   return Builder.CreateCall(F, Scalar);
 }
 
 Value *CodeGenFunction::EmitSVEDupX(Value* Scalar) {
Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
   else if (TypeFlags.isStore())
     return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic);
   else if (TypeFlags.isSMELd1() || TypeFlags.isSMESt1())
     return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
   else if (TypeFlags.isSMERead())
     return EmitSMERead(TypeFlags, Ops, Builtin->LLVMIntrinsic);
   else if (TypeFlags.isSMEWrite())
     return EmitSMEWrite(TypeFlags, Ops, Builtin->LLVMIntrinsic);
+  else if (TypeFlags.isSMEZero())
+    return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
   else if (TypeFlags.isGatherLoad())
     return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
   else if (TypeFlags.isScatterStore())
     return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
   else if (TypeFlags.isPrefetch())
     return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
   else if (TypeFlags.isGatherPrefetch())
     return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic);
return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
                                    {Insert, Builder.getInt64(0)});
   }
   }
 
   /// Should not happen
   return nullptr;
 }
 
 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
                                                const CallExpr *E,
                                                llvm::Triple::ArchType Arch) {
   if (BuiltinID >= clang::AArch64::FirstSVEBuiltin &&
       BuiltinID <= clang::AArch64::LastSVEBuiltin)
     return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
 
   unsigned HintID = static_cast<unsigned>(-1);
   switch (BuiltinID) {
   default: break;

clang/lib/CodeGen/CodeGenFunction.h

llvm::Value *EmitSMELd1St1(SVETypeFlags TypeFlags,
                              llvm::SmallVectorImpl<llvm::Value *> &Ops,
                              unsigned IntID);
   llvm::Value *EmitSMERead(SVETypeFlags TypeFlags,
                            llvm::SmallVectorImpl<llvm::Value *> &Ops,
                            unsigned IntID);
   llvm::Value *EmitSMEWrite(SVETypeFlags TypeFlags,
                             llvm::SmallVectorImpl<llvm::Value *> &Ops,
                             unsigned IntID);
+  llvm::Value *EmitSMEZero(SVETypeFlags TypeFlags,
+                           llvm::SmallVectorImpl<llvm::Value *> &Ops,
+                           unsigned IntID);
   llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
                                    SmallVectorImpl<llvm::Value *> &Ops,
                                    unsigned BuiltinID);
   llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
                                      SmallVectorImpl<llvm::Value *> &Ops,
                                      unsigned IntID);
   llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
                                  SmallVectorImpl<llvm::Value *> &Ops,

clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c

+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -D__ARM_FEATURE_SME -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SME -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -D__ARM_FEATURE_SME -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -o /dev/null %s
+
+#include <arm_sme.h>
+
+// CHECK-LABEL: @test_svzero_mask_za(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i64 0)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svzero_mask_zav(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i64 0)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svzero_mask_za() {
+  svzero_mask_za(0);
+}
+
+// CHECK-LABEL: @test_svzero_mask_za_1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i64 176)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z21test_svzero_mask_za_1v(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i64 176)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svzero_mask_za_1() {
+  svzero_mask_za(176);
+}
+
+// CHECK-LABEL: @test_svzero_mask_za_2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i64 255)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z21test_svzero_mask_za_2v(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i64 255)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svzero_mask_za_2() {
+  svzero_mask_za(255);
+}
+
+// CHECK-LABEL: @test_svzero_za(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i64 255)
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z14test_svzero_zav(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i64 255)
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svzero_za() {
+  svzero_za();
+}

clang/utils/TableGen/SveEmitter.cpp

if (this->Flags & Emitter.getEnumValueForFlag("IsSME")) {
     if (this->Flags & Emitter.getEnumValueForFlag("IsSMELd1"))
       this->SMEAttributes = "arm_streaming, arm_shared_za";
     else if (this->Flags & Emitter.getEnumValueForFlag("IsSMESt1"))
       this->SMEAttributes = "arm_streaming, arm_shared_za, arm_preserves_za";
     else if (this->Flags & Emitter.getEnumValueForFlag("IsSMERead"))
       this->SMEAttributes = "arm_streaming, arm_shared_za, arm_preserves_za";
     else if (this->Flags & Emitter.getEnumValueForFlag("IsSMEWrite"))
       this->SMEAttributes = "arm_streaming, arm_shared_za";
+    else if (this->Flags & Emitter.getEnumValueForFlag("IsSMEZero"))
+      this->SMEAttributes = "arm_streaming_compatible, arm_shared_za";
     else
       llvm_unreachable("Unknown SME instruction");
   } else
     this->IsSMEIntrinsic = false;
 }
 
 std::string Intrinsic::getBuiltinTypeStr() {
   std::string S = getReturnType().builtin_str();