Index: clang/include/clang/Basic/BuiltinsAArch64.def =================================================================== --- clang/include/clang/Basic/BuiltinsAArch64.def +++ clang/include/clang/Basic/BuiltinsAArch64.def @@ -91,6 +91,12 @@ // Misc BUILTIN(__builtin_sponentry, "v*", "c") +// Transactional Memory Extension +BUILTIN(__builtin_arm_tstart, "WUi", "nj") +BUILTIN(__builtin_arm_tcommit, "v", "n") +BUILTIN(__builtin_arm_tcancel, "vWUIi", "n") +BUILTIN(__builtin_arm_ttest, "WUi", "nc") + TARGET_HEADER_BUILTIN(_BitScanForward, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "") TARGET_HEADER_BUILTIN(_BitScanReverse, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "") TARGET_HEADER_BUILTIN(_BitScanForward64, "UcUNi*ULLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "") Index: clang/lib/Basic/Targets/AArch64.h =================================================================== --- clang/lib/Basic/Targets/AArch64.h +++ clang/lib/Basic/Targets/AArch64.h @@ -35,6 +35,7 @@ bool HasDotProd; bool HasFP16FML; bool HasMTE; + bool HasTME; llvm::AArch64::ArchKind ArchKind; Index: clang/lib/Basic/Targets/AArch64.cpp =================================================================== --- clang/lib/Basic/Targets/AArch64.cpp +++ clang/lib/Basic/Targets/AArch64.cpp @@ -216,6 +216,9 @@ if (HasMTE) Builder.defineMacro("__ARM_FEATURE_MEMORY_TAGGING", "1"); + if (HasTME) + Builder.defineMacro("__ARM_FEATURE_TME", "1"); + if ((FPU & NeonMode) && HasFP16FML) Builder.defineMacro("__ARM_FEATURE_FP16FML", "1"); @@ -267,6 +270,7 @@ HasDotProd = false; HasFP16FML = false; HasMTE = false; + HasTME = false; ArchKind = llvm::AArch64::ArchKind::ARMV8A; for (const auto &Feature : Features) { @@ -298,6 +302,8 @@ HasFP16FML = true; if (Feature == "+mte") HasMTE = true; + if (Feature == "+tme") + HasTME = true; } setDataLayout(); Index: clang/lib/Headers/arm_acle.h =================================================================== --- clang/lib/Headers/arm_acle.h +++ clang/lib/Headers/arm_acle.h @@ -613,7 +613,7 @@ #define __arm_wsr64(sysreg, v) __builtin_arm_wsr64(sysreg, v) #define __arm_wsrp(sysreg, v) __builtin_arm_wsrp(sysreg, v) -// Memory Tagging Extensions (MTE) Intrinsics +/* Memory Tagging Extensions (MTE) Intrinsics */ #if __ARM_FEATURE_MEMORY_TAGGING #define __arm_mte_create_random_tag(__ptr, __mask) __builtin_arm_irg(__ptr, __mask) #define __arm_mte_increment_tag(__ptr, __tag_offset) __builtin_arm_addg(__ptr, __tag_offset) @@ -623,6 +623,28 @@ #define __arm_mte_ptrdiff(__ptra, __ptrb) __builtin_arm_subp(__ptra, __ptrb) #endif +/* Transactional Memory Extension (TME) Intrinsics */ +#if __ARM_FEATURE_TME + +#define _TMFAILURE_REASON 0x00007fffu +#define _TMFAILURE_RTRY 0x00008000u +#define _TMFAILURE_CNCL 0x00010000u +#define _TMFAILURE_MEM 0x00020000u +#define _TMFAILURE_IMP 0x00040000u +#define _TMFAILURE_ERR 0x00080000u +#define _TMFAILURE_SIZE 0x00100000u +#define _TMFAILURE_NEST 0x00200000u +#define _TMFAILURE_DBG 0x00400000u +#define _TMFAILURE_INT 0x00800000u +#define _TMFAILURE_TRIVIAL 0x01000000u + +#define __tstart() __builtin_arm_tstart() +#define __tcommit() __builtin_arm_tcommit() +#define __tcancel(__arg) __builtin_arm_tcancel(__arg) +#define __ttest() __builtin_arm_ttest() + +#endif /* __ARM_FEATURE_TME */ + #if defined(__cplusplus) } #endif Index: clang/lib/Sema/SemaChecking.cpp =================================================================== --- clang/lib/Sema/SemaChecking.cpp +++ clang/lib/Sema/SemaChecking.cpp @@ -1932,6 +1932,7 @@ case AArch64::BI__builtin_arm_dmb: case AArch64::BI__builtin_arm_dsb: case AArch64::BI__builtin_arm_isb: l = 0; u = 15; break; + case AArch64::BI__builtin_arm_tcancel: l = 0; u = 65535; break; } return SemaBuiltinConstantArgRange(TheCall, i, l, u + l); Index: clang/test/CodeGen/aarch64-tme.cpp =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-tme.cpp @@ -0,0 +1,42 @@ +// RUN: %clang_cc1 -triple aarch64-eabi -target-feature +tme -S -emit-llvm %s -o - | FileCheck %s +// RUN: %clang_cc1 -DUSE_ACLE -triple aarch64-eabi -target-feature +tme -S -emit-llvm %s -o - | FileCheck %s + +#define A -1 +constexpr int f() { return 65536; } + +#ifdef USE_ACLE +#include "arm_acle.h" +void test_tme_funcs() { + __tstart(); + (void)__ttest(); + __tcommit(); + __tcancel(0x789a); + __tcancel(f() + A); +} +#else +void test_tme_funcs() { + __builtin_arm_tstart(); + (void)__builtin_arm_ttest(); + __builtin_arm_tcommit(); + __builtin_arm_tcancel(0x789a); + __builtin_arm_tcancel(f() + A); +} +#endif +// CHECK: call i64 @llvm.aarch64.tstart() +// CHECK: call i64 @llvm.aarch64.ttest() +// CHECK: call void @llvm.aarch64.tcommit() +// CHECK: call void @llvm.aarch64.tcancel(i64 30874) +// CHECK: call void @llvm.aarch64.tcancel(i64 65535) + +// CHECK: declare i64 @llvm.aarch64.tstart() #1 +// CHECK: declare i64 @llvm.aarch64.ttest() #1 +// CHECK: declare void @llvm.aarch64.tcommit() #1 +// CHECK: declare void @llvm.aarch64.tcancel(i64 immarg) #1 + +#ifdef __ARM_FEATURE_TME +extern "C" void arm_feature_tme_defined() {} +#endif +// CHECK: define void @arm_feature_tme_defined() + +// CHECK: attributes #1 = { nounwind } + Index: clang/test/Sema/aarch64-tme-errors.c =================================================================== --- /dev/null +++ clang/test/Sema/aarch64-tme-errors.c @@ -0,0 +1,8 @@ +// RUN: %clang_cc1 -triple aarch64-eabi -verify %s + +#include "arm_acle.h" + +void test_no_tme_funcs() { + __tstart(); // expected-warning{{implicit declaration of function '__tstart'}} + __builtin_tstart(); // expected-error{{use of unknown builtin '__builtin_tstart'}} +} Index: clang/test/Sema/aarch64-tme-tcancel-errors.c =================================================================== --- /dev/null +++ clang/test/Sema/aarch64-tme-tcancel-errors.c @@ -0,0 +1,9 @@ +// RUN: %clang_cc1 -triple aarch64-eabi -target-feature +tme -verify %s +void t_cancel_const(unsigned short u) { + __builtin_arm_tcancel(u); // expected-error{{argument to '__builtin_arm_tcancel' must be a constant integer}} +} + +// RUN: %clang_cc1 -triple aarch64-eabi -target-feature +tme -verify %s +void t_cancel_range() { + __builtin_arm_tcancel(0x12345u); // expected-error{{argument value 74565 is outside the valid range [0, 65535]}} +} Index: llvm/include/llvm/IR/IntrinsicsAArch64.td =================================================================== --- llvm/include/llvm/IR/IntrinsicsAArch64.td +++ llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -733,3 +733,18 @@ def int_aarch64_stgp : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], [IntrWriteMem, IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>; } + +// Transactional Memory Extension (TME) Intrinsics +let TargetPrefix = "aarch64" in { +def int_aarch64_tstart : GCCBuiltin<"__builtin_arm_tstart">, + Intrinsic<[llvm_i64_ty]>; + +def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[]>; + +def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">, + Intrinsic<[], [llvm_i64_ty], [ImmArg<0>]>; + +def int_aarch64_ttest : GCCBuiltin<"__builtin_arm_ttest">, + Intrinsic<[llvm_i64_ty], [], + [IntrNoMem, IntrHasSideEffects]>; +} Index: llvm/include/llvm/Support/AArch64TargetParser.h =================================================================== --- llvm/include/llvm/Support/AArch64TargetParser.h +++ llvm/include/llvm/Support/AArch64TargetParser.h @@ -54,6 +54,7 @@ AEK_SVE2SM4 = 1 << 25, AEK_SVE2SHA3 = 1 << 26, AEK_SVE2BITPERM = 1 << 27, + AEK_TME = 1 << 28, }; enum class ArchKind { Index: llvm/include/llvm/Support/AArch64TargetParser.def =================================================================== --- llvm/include/llvm/Support/AArch64TargetParser.def +++ llvm/include/llvm/Support/AArch64TargetParser.def @@ -79,6 +79,7 @@ AARCH64_ARCH_EXT_NAME("ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs") AARCH64_ARCH_EXT_NAME("sb", AArch64::AEK_SB, "+sb", "-sb") AARCH64_ARCH_EXT_NAME("predres", AArch64::AEK_PREDRES, "+predres", "-predres") +AARCH64_ARCH_EXT_NAME("tme", AArch64::AEK_TME, "+tme", "-tme") #undef AARCH64_ARCH_EXT_NAME #ifndef AARCH64_CPU_NAME Index: llvm/lib/Target/AArch64/AArch64.td =================================================================== --- llvm/lib/Target/AArch64/AArch64.td +++ llvm/lib/Target/AArch64/AArch64.td @@ -352,6 +352,9 @@ "true", "Enable Embedded Trace Extension", [FeatureTRBE]>; +def FeatureTME : SubtargetFeature<"tme", "HasTME", + "true", "Enable Transactional Memory Extension" >; + //===----------------------------------------------------------------------===// // Architectures. // Index: llvm/lib/Target/AArch64/AArch64InstrFormats.td =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -678,12 +678,15 @@ let ParserMatchClass = LogicalImm64NotOperand; } -// imm0_65535 predicate - True if the immediate is in the range [0,65535]. -def imm0_65535 : Operand, ImmLeaf, PrintMethod = "printImmHex" in { +def i32_imm0_65535 : Operand, ImmLeaf { - let ParserMatchClass = AsmImmRange<0, 65535>; - let PrintMethod = "printImmHex"; +}]>; + +def i64_imm0_65535 : Operand, ImmLeaf; } // imm0_255 predicate - True if the immediate is in the range [0,255]. @@ -1046,6 +1049,45 @@ let Inst{4-0} = Rt; } +// System instructions for transactional memory extension +class TMBaseSystemI CRm, bits<3> op2, dag oops, dag iops, + string asm, string operands, list pattern> + : BaseSystemI, + Sched<[WriteSys]> { + let Inst{20-12} = 0b000110011; + let Inst{11-8} = CRm; + let Inst{7-5} = op2; + let DecoderMethod = ""; + + let mayLoad = 1; + let mayStore = 1; +} + +// System instructions for transactional memory - single input operand +class TMSystemI CRm, string asm, list pattern> + : TMBaseSystemI<0b1, CRm, 0b011, + (outs GPR64:$Rt), (ins), asm, "\t$Rt", pattern> { + bits<5> Rt; + let Inst{4-0} = Rt; +} + +// System instructions for transactional memory - no operand +class TMSystemINoOperand CRm, string asm, list pattern> + : TMBaseSystemI<0b0, CRm, 0b011, (outs), (ins), asm, "", pattern> { + let Inst{4-0} = 0b11111; +} + +// System instructions for exit from transactions +class TMSystemException op1, string asm, list pattern> + : I<(outs), (ins i64_imm0_65535:$imm), asm, "\t$imm", "", pattern>, + Sched<[WriteSys]> { + bits<16> imm; + let Inst{31-24} = 0b11010100; + let Inst{23-21} = op1; + let Inst{20-5} = imm; + let Inst{4-0} = 0b00000; +} + // Hint instructions that take both a CRm and a 3-bit immediate. // NOTE: ideally, this would have mayStore = 0, mayLoad = 0, but we cannot // model patterns with sufficiently fine granularity @@ -4050,7 +4092,7 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 1 in class ExceptionGeneration op1, bits<2> ll, string asm> - : I<(outs), (ins imm0_65535:$imm), asm, "\t$imm", "", []>, + : I<(outs), (ins i32_imm0_65535:$imm), asm, "\t$imm", "", []>, Sched<[WriteSys]> { bits<16> imm; let Inst{31-24} = 0b11010100; Index: llvm/lib/Target/AArch64/AArch64InstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -133,6 +133,8 @@ AssemblerPredicate<"FeatureBranchTargetId", "bti">; def HasMTE : Predicate<"Subtarget->hasMTE()">, AssemblerPredicate<"FeatureMTE", "mte">; +def HasTME : Predicate<"Subtarget->hasTME()">, + AssemblerPredicate<"FeatureTME", "tme">; def HasETE : Predicate<"Subtarget->hasETE()">, AssemblerPredicate<"FeatureETE", "ete">; def HasTRBE : Predicate<"Subtarget->hasTRBE()">, @@ -808,6 +810,23 @@ (SYSxt imm0_7:$op1, sys_cr_op:$Cn, sys_cr_op:$Cm, imm0_7:$op2, XZR)>; + +let Predicates = [HasTME] in { + +def TSTART : TMSystemI<0b0000, "tstart", + [(set GPR64:$Rt, (int_aarch64_tstart))]>; + +def TCOMMIT : TMSystemINoOperand<0b0000, "tcommit", [(int_aarch64_tcommit)]>; + +def TCANCEL : TMSystemException<0b011, "tcancel", + [(int_aarch64_tcancel i64_imm0_65535:$imm)]>; + +def TTEST : TMSystemI<0b0001, "ttest", [(set GPR64:$Rt, (int_aarch64_ttest))]> { + let mayLoad = 0; + let mayStore = 0; +} +} // HasTME + //===----------------------------------------------------------------------===// // Move immediate instructions. //===----------------------------------------------------------------------===// @@ -819,12 +838,12 @@ defm MOVZ : MoveImmediate<0b10, "movz">; // First group of aliases covers an implicit "lsl #0". -def : InstAlias<"movk $dst, $imm", (MOVKWi GPR32:$dst, imm0_65535:$imm, 0), 0>; -def : InstAlias<"movk $dst, $imm", (MOVKXi GPR64:$dst, imm0_65535:$imm, 0), 0>; -def : InstAlias<"movn $dst, $imm", (MOVNWi GPR32:$dst, imm0_65535:$imm, 0)>; -def : InstAlias<"movn $dst, $imm", (MOVNXi GPR64:$dst, imm0_65535:$imm, 0)>; -def : InstAlias<"movz $dst, $imm", (MOVZWi GPR32:$dst, imm0_65535:$imm, 0)>; -def : InstAlias<"movz $dst, $imm", (MOVZXi GPR64:$dst, imm0_65535:$imm, 0)>; +def : InstAlias<"movk $dst, $imm", (MOVKWi GPR32:$dst, i32_imm0_65535:$imm, 0), 0>; +def : InstAlias<"movk $dst, $imm", (MOVKXi GPR64:$dst, i32_imm0_65535:$imm, 0), 0>; +def : InstAlias<"movn $dst, $imm", (MOVNWi GPR32:$dst, i32_imm0_65535:$imm, 0)>; +def : InstAlias<"movn $dst, $imm", (MOVNXi GPR64:$dst, i32_imm0_65535:$imm, 0)>; +def : InstAlias<"movz $dst, $imm", (MOVZWi GPR32:$dst, i32_imm0_65535:$imm, 0)>; +def : InstAlias<"movz $dst, $imm", (MOVZXi GPR64:$dst, i32_imm0_65535:$imm, 0)>; // Next, we have various ELF relocations with the ":XYZ_g0:sym" syntax. def : InstAlias<"movz $Rd, $sym", (MOVZXi GPR64:$Rd, movw_symbol_g3:$sym, 48)>; Index: llvm/lib/Target/AArch64/AArch64Subtarget.h =================================================================== --- llvm/lib/Target/AArch64/AArch64Subtarget.h +++ llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -137,6 +137,7 @@ bool HasBTI = false; bool HasRandGen = false; bool HasMTE = false; + bool HasTME = false; // Arm SVE2 extensions bool HasSVE2AES = false; @@ -387,6 +388,7 @@ bool hasBTI() const { return HasBTI; } bool hasRandGen() const { return HasRandGen; } bool hasMTE() const { return HasMTE; } + bool hasTME() const { return HasTME; } // Arm SVE2 extensions bool hasSVE2AES() const { return HasSVE2AES; } bool hasSVE2SM4() const { return HasSVE2SM4; } Index: llvm/test/CodeGen/AArch64/tme.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/tme.ll @@ -0,0 +1,44 @@ +; RUN: llc %s -verify-machineinstrs -o - | FileCheck %s + +target triple = "aarch64-unknown-unknown-eabi" + +define i64 @test_tstart() #0 { + %r = tail call i64 @llvm.aarch64.tstart() + ret i64 %r +} +declare i64 @llvm.aarch64.tstart() #1 +; CHECK-LABEL: test_tstart +; CHECK: tstart x + +define i64 @test_ttest() #0 { + %r = tail call i64 @llvm.aarch64.ttest() + ret i64 %r +} +declare i64 @llvm.aarch64.ttest() #1 +; CHECK-LABEL: test_ttest +; CHECK: ttest x + +define void @test_tcommit() #0 { + tail call void @llvm.aarch64.tcommit() + ret void +} +declare void @llvm.aarch64.tcommit() #1 +; CHECK-LABEL: test_tcommit +; CHECK: tcommit + +define void @test_tcancel() #0 { + tail call void @llvm.aarch64.tcancel(i64 0) #1 + tail call void @llvm.aarch64.tcancel(i64 1) #1 + tail call void @llvm.aarch64.tcancel(i64 65534) #1 + tail call void @llvm.aarch64.tcancel(i64 65535) #1 + ret void +} +declare void @llvm.aarch64.tcancel(i64 immarg) #1 +; CHECK-LABEL: test_tcancel +; CHECK: tcancel #0 +; CHECK: tcancel #0x1 +; CHECK: tcancel #0xfffe +; CHECK: tcancel #0xffff + +attributes #0 = { "target-features"="+tme" } +attributes #1 = { nounwind } Index: llvm/test/MC/AArch64/tme-error.s =================================================================== --- /dev/null +++ llvm/test/MC/AArch64/tme-error.s @@ -0,0 +1,47 @@ +// Tests for transactional memory extension instructions +// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+tme < %s 2>&1 | FileCheck %s + +tstart +// CHECK: error: too few operands for instruction +// CHECK-NEXT: tstart +tstart x4, x5 +// CHECK: error: invalid operand for instruction +// CHECK-NEXT: tstart x4, x5 +tstart x4, #1 +// CHECK: error: invalid operand for instruction +// CHECK-NEXT: tstart x4, #1 +tstart sp +// CHECK: error: invalid operand for instruction +// CHECK-NEXT: tstart sp + +ttest +// CHECK: error: too few operands for instruction +// CHECK-NEXT: ttest +ttest x4, x5 +// CHECK: error: invalid operand for instruction +// CHECK-NEXT: ttest x4, x5 +ttest x4, #1 +// CHECK: error: invalid operand for instruction +// CHECK-NEXT: ttest x4, #1 +ttest sp +// CHECK: error: invalid operand for instruction +// CHECK-NEXT: ttest sp + +tcommit x4 +// CHECK: error: invalid operand for instruction +// CHECK-NEXT: tcommit x4 +tcommit sp +// CHECK: error: invalid operand for instruction +// CHECK-NEXT: tcommit sp + + +tcancel +// CHECK: error: too few operands for instruction +// CHECK-NEXT tcancel +tcancel x0 +// CHECK: error: immediate must be an integer in range [0, 65535] +// CHECK-NEXT tcancel +tcancel #65536 +// CHECK: error: immediate must be an integer in range [0, 65535] +// CHECK-NEXT: tcancel #65536 + Index: llvm/test/MC/AArch64/tme.s =================================================================== --- /dev/null +++ llvm/test/MC/AArch64/tme.s @@ -0,0 +1,24 @@ +// Tests for transaction memory extension instructions +// +// RUN: llvm-mc -triple aarch64 -show-encoding -mattr=+tme < %s | FileCheck %s +// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=-tme < %s 2>&1 | FileCheck %s --check-prefix=NOTME + +tstart x3 +ttest x4 +tcommit +tcancel #0x1234 + +// CHECK: tstart x3 // encoding: [0x63,0x30,0x23,0xd5] +// CHECK: ttest x4 // encoding: [0x64,0x31,0x23,0xd5] +// CHECK: tcommit // encoding: [0x7f,0x30,0x03,0xd5] +// CHECK: tcancel #0x1234 // encoding: [0x80,0x46,0x62,0xd4] + + +// NOTME: instruction requires: tme +// NOTME-NEXT: tstart x3 +// NOTME: instruction requires: tme +// NOTME-NEXT: ttest x4 +// NOTME: instruction requires: tme +// NOTME-NEXT: tcommit +// NOTME: instruction requires: tme +// NOTME-NEXT: tcancel #0x1234 Index: llvm/test/MC/Disassembler/AArch64/tme.txt =================================================================== --- /dev/null +++ llvm/test/MC/Disassembler/AArch64/tme.txt @@ -0,0 +1,19 @@ +# Tests for transaction memory extension instructions +# RUN: llvm-mc -triple=aarch64 -mattr=+tme -disassemble < %s | FileCheck %s +# RUN: not llvm-mc -triple=aarch64 -mattr=-tme -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOTME + +[0x63,0x30,0x23,0xd5] +[0x64,0x31,0x23,0xd5] +[0x7f,0x30,0x03,0xd5] +[0x80,0x46,0x62,0xd4] + +# CHECK: tstart x3 +# CHECK: ttest x4 +# CHECK: tcommit +# CHECK: tcancel #0x1234 + +# NOTEME: mrs +# NOTEME-NEXT: mrs +# NOTEME-NEXT: msr +# NOTME: warning: invalid instruction encoding +# NOTME-NEXT: [0x80,0x46,0x62,0xd4] Index: llvm/unittests/Support/TargetParserTest.cpp =================================================================== --- llvm/unittests/Support/TargetParserTest.cpp +++ llvm/unittests/Support/TargetParserTest.cpp @@ -1153,6 +1153,7 @@ {"rcpc", "norcpc", "+rcpc", "-rcpc" }, {"rng", "norng", "+rand", "-rand"}, {"memtag", "nomemtag", "+mte", "-mte"}, + {"tme", "notme", "+tme", "-tme"}, {"ssbs", "nossbs", "+ssbs", "-ssbs"}, {"sb", "nosb", "+sb", "-sb"}, {"predres", "nopredres", "+predres", "-predres"}