[AArch64] Support for memset tagged intrinsic

[tyb0807]

Summary

This introduces a new ACLE intrinsic for memset tagged
(https://github.com/ARM-software/acle/blob/next-release/main/acle.md#memcpy-family-of-operations-intrinsics---mops).

void *__builtin_arm_mops_memset_tag(void *, int, size_t)

A corresponding LLVM intrinsic is introduced:

i8* llvm.aarch64.mops.memset.tag(i8*, i8, i64)

The types match llvm.memset but the return type is not void.

This is part 1/4 of a series of patches split from
https://reviews.llvm.org/D117405 to facilitate reviewing.

Patch by Tomas Matheson

[tyb0807]

Update commit message to clarify context.

[SjoerdMeijer]

clang/lib/Headers/arm_acle.h

734

Why does this also need MTE? I think the ACLE specifies this intrinsic to be available when __ARM_FEATURE_MOPS is defined?

[dmgreen]

clang/lib/Headers/arm_acle.h

736

The arguments are better named value and size, etc. That way they only use reserved names and uses won't run into trouble if they #define value 42 before the include.

[dmgreen]

clang/lib/Headers/arm_acle.h

736

Oh, I mean __value and __size, etc :)

[tyb0807]

clang/lib/Headers/arm_acle.h

734

Yes you are right, thanks for spotting this.

736

Yes, thanks for the review

[tyb0807]

__ARM_FEATURE_MEMORY_TAGGING not needed to enable __builtin_arm_mops_memset_tag.

Follow variable naming convention.

[tyb0807]

Update reference to ACLE specification

[SjoerdMeijer]

clang/test/CodeGen/aarch64-mops.c

4

I forgot if we add negative tests for these things, i.e. check if we error if we don't have +mops or __ARM_FEATURE_MOPS set? I guess so?

[tyb0807]

clang/test/CodeGen/aarch64-mops.c

4

Currently this only tests if the support for memset tagged intrinsic is in clang, how to enable the memset tagged intrinsic is implemented (and tested) in a separate patch (https://reviews.llvm.org/D116160). Note how we set __ARM_FEATURE_MOPS manually in line 5.

This test will thus be updated with negative tests in https://reviews.llvm.org/D116160

[SjoerdMeijer]

LGTM

[dmgreen]

clang/lib/Headers/arm_acle.h

734

Hmm. These map to SETMG, and those instructions require MTE. It wouldn't make sense to have an intrinsic that cannot be emitted to a valid instruction. I think the spec might be wrong, to be honest.

[SjoerdMeijer]

clang/lib/Headers/arm_acle.h

734

Ah yeah, thanks Dave. These are the tag setting variants, I didn't look careful enough, then probably got confused about the ACLE which I agree must be wrong. Would be good to double check that first.

[dmgreen]

clang/lib/Headers/arm_acle.h

734

Yep - Apparently it is getting fixed in https://github.com/ARM-software/acle/pull/161 thanks to Lucas

[tyb0807]

Update accordingly to change from ACLE specification: __builtin_arm_mops_memset_tag requires _both_ MOPS and MTE features

[tyb0807]

Update link to the relevant section of ACLE specification

[dmgreen]

Thanks. LGTM

clang/include/clang/Basic/BuiltinsAArch64.def

 // Memory Tagging Extensions (MTE)
 BUILTIN(__builtin_arm_irg, "v*v*Ui", "t")
 BUILTIN(__builtin_arm_addg, "v*v*Ui", "t")
 BUILTIN(__builtin_arm_gmi, "Uiv*Ui", "t")
 BUILTIN(__builtin_arm_ldg, "v*v*", "t")
 BUILTIN(__builtin_arm_stg, "vv*", "t")
 BUILTIN(__builtin_arm_subp, "Uiv*v*", "t")
 
+// Memory Operations
+BUILTIN(__builtin_arm_mops_memset_tag, "v*v*iz", "")
+
 // Memory barrier
 BUILTIN(__builtin_arm_dmb, "vUi", "nc")
 BUILTIN(__builtin_arm_dsb, "vUi", "nc")
 BUILTIN(__builtin_arm_isb, "vUi", "nc")
 
 BUILTIN(__builtin_arm_jcvt, "Zid", "nc")
 
 // Prefetch

clang/lib/CodeGen/CGBuiltin.cpp

if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
     Function *F = CGM.getIntrinsic(CRCIntrinsicID);
 
     llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
     Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
 
     return Builder.CreateCall(F, {Arg0, Arg1});
   }
 
+  // Memory Operations (MOPS)
+  if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
+    Value *Dst = EmitScalarExpr(E->getArg(0));
+    Value *Val = EmitScalarExpr(E->getArg(1));
+    Value *Size = EmitScalarExpr(E->getArg(2));
+    Dst = Builder.CreatePointerCast(Dst, Int8PtrTy);
+    Val = Builder.CreateTrunc(Val, Int8Ty);
+    Size = Builder.CreateIntCast(Size, Int64Ty, false);
+    return Builder.CreateCall(
+        CGM.getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
+  }
+
   // Memory Tagging Extensions (MTE) Intrinsics
   Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
   switch (BuiltinID) {
   case AArch64::BI__builtin_arm_irg:
     MTEIntrinsicID = Intrinsic::aarch64_irg; break;
   case  AArch64::BI__builtin_arm_addg:
     MTEIntrinsicID = Intrinsic::aarch64_addg; break;
   case  AArch64::BI__builtin_arm_gmi:

clang/lib/Headers/arm_acle.h

 #define __arm_mte_create_random_tag(__ptr, __mask)  __builtin_arm_irg(__ptr, __mask)
 #define __arm_mte_increment_tag(__ptr, __tag_offset)  __builtin_arm_addg(__ptr, __tag_offset)
 #define __arm_mte_exclude_tag(__ptr, __excluded)  __builtin_arm_gmi(__ptr, __excluded)
 #define __arm_mte_get_tag(__ptr) __builtin_arm_ldg(__ptr)
 #define __arm_mte_set_tag(__ptr) __builtin_arm_stg(__ptr)
 #define __arm_mte_ptrdiff(__ptra, __ptrb) __builtin_arm_subp(__ptra, __ptrb)
 #endif
 
+/* Memory Operations Intrinsics */
+#if __ARM_FEATURE_MOPS && __ARM_FEATURE_MEMORY_TAGGING
+#define __arm_mops_memset_tag(__tagged_address, __value, __size)    \
+  __builtin_arm_mops_memset_tag(__tagged_address, __value, __size)
+#endif
+
 /* Transactional Memory Extension (TME) Intrinsics */
 #if __ARM_FEATURE_TME
 
 #define _TMFAILURE_REASON  0x00007fffu
 #define _TMFAILURE_RTRY    0x00008000u
 #define _TMFAILURE_CNCL    0x00010000u
 #define _TMFAILURE_MEM     0x00020000u
 #define _TMFAILURE_IMP     0x00040000u

clang/test/CodeGen/aarch64-mops.c

+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+
+// RUN: %clang_cc1 -triple aarch64-arm-unknown-eabi -target-feature +mops -target-feature +mte -S -emit-llvm -o - %s  | FileCheck %s
+
+#define __ARM_FEATURE_MOPS 1
+#include <arm_acle.h>
+#include <stddef.h>
+
+// CHECK-LABEL: @bzero_0(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 0, i64 0)
+// CHECK-NEXT:    ret i8* [[TMP1]]
+//
+void *bzero_0(void *dst) {
+  return __arm_mops_memset_tag(dst, 0, 0);
+}
+
+// CHECK-LABEL: @bzero_1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 0, i64 1)
+// CHECK-NEXT:    ret i8* [[TMP1]]
+//
+void *bzero_1(void *dst) {
+  return __arm_mops_memset_tag(dst, 0, 1);
+}
+
+// CHECK-LABEL: @bzero_10(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 0, i64 10)
+// CHECK-NEXT:    ret i8* [[TMP1]]
+//
+void *bzero_10(void *dst) {
+  return __arm_mops_memset_tag(dst, 0, 10);
+}
+
+// CHECK-LABEL: @bzero_10000(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 0, i64 10000)
+// CHECK-NEXT:    ret i8* [[TMP1]]
+//
+void *bzero_10000(void *dst) {
+  return __arm_mops_memset_tag(dst, 0, 10000);
+}
+
+// CHECK-LABEL: @bzero_n(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    [[SIZE_ADDR:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    store i64 [[SIZE:%.*]], i64* [[SIZE_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[SIZE_ADDR]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 0, i64 [[TMP1]])
+// CHECK-NEXT:    ret i8* [[TMP2]]
+//
+void *bzero_n(void *dst, size_t size) {
+  return __arm_mops_memset_tag(dst, 0, size);
+}
+
+// CHECK-LABEL: @memset_0(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    [[VALUE_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[VALUE:%.*]], i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
+// CHECK-NEXT:    [[TMP3:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 [[TMP2]], i64 0)
+// CHECK-NEXT:    ret i8* [[TMP3]]
+//
+void *memset_0(void *dst, int value) {
+  return __arm_mops_memset_tag(dst, value, 0);
+}
+
+// CHECK-LABEL: @memset_1(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    [[VALUE_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[VALUE:%.*]], i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
+// CHECK-NEXT:    [[TMP3:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 [[TMP2]], i64 1)
+// CHECK-NEXT:    ret i8* [[TMP3]]
+//
+void *memset_1(void *dst, int value) {
+  return __arm_mops_memset_tag(dst, value, 1);
+}
+
+// CHECK-LABEL: @memset_10(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    [[VALUE_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[VALUE:%.*]], i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
+// CHECK-NEXT:    [[TMP3:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 [[TMP2]], i64 10)
+// CHECK-NEXT:    ret i8* [[TMP3]]
+//
+void *memset_10(void *dst, int value) {
+  return __arm_mops_memset_tag(dst, value, 10);
+}
+
+// CHECK-LABEL: @memset_10000(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    [[VALUE_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[VALUE:%.*]], i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
+// CHECK-NEXT:    [[TMP3:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 [[TMP2]], i64 10000)
+// CHECK-NEXT:    ret i8* [[TMP3]]
+//
+void *memset_10000(void *dst, int value) {
+  return __arm_mops_memset_tag(dst, value, 10000);
+}
+
+// CHECK-LABEL: @memset_n(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DST_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    [[VALUE_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[SIZE_ADDR:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    store i8* [[DST:%.*]], i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[VALUE:%.*]], i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    store i64 [[SIZE:%.*]], i64* [[SIZE_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[DST_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[VALUE_ADDR]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* [[SIZE_ADDR]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP1]] to i8
+// CHECK-NEXT:    [[TMP4:%.*]] = call i8* @llvm.aarch64.mops.memset.tag(i8* [[TMP0]], i8 [[TMP3]], i64 [[TMP2]])
+// CHECK-NEXT:    ret i8* [[TMP4]]
+//
+void *memset_n(void *dst, int value, size_t size) {
+  return __arm_mops_memset_tag(dst, value, size);
+}

llvm/include/llvm/IR/IntrinsicsAArch64.td

 def int_aarch64_settag_zero  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 
 // Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
 def int_aarch64_stgp  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
     [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
 }
 
+//===----------------------------------------------------------------------===//
+// Memory Operations (MOPS) Intrinsics
+let TargetPrefix = "aarch64" in {
+  // Sizes are chosen to correspond to the llvm.memset intrinsic: ptr, i8, i64
+  def int_aarch64_mops_memset_tag : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i8_ty, llvm_i64_ty],
+      [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
+}
+
 // Transactional Memory Extension (TME) Intrinsics
 let TargetPrefix = "aarch64" in {
 def int_aarch64_tstart  : GCCBuiltin<"__builtin_arm_tstart">,
                          Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>;
 
 def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>;
 
 def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">,