[builtins][AArch64] Implement _sync out-of-line atomics

[jrtc27]

Summary

Whilst Clang does not use these, recent GCC does, and so on systems such
as FreeBSD that wish to use compiler-rt as the system runtime library
but also wish to support building programs with GCC these interfaces are
needed.

This is a light adaptation of the code committed to GCC by Sebastian Pop
<[e-mail]>, relicensed with permission for use in compiler-rt.

Fixes https://github.com/llvm/llvm-project/issues/63483

[sebpop]

Looks good to me.
Thank you.

[jrtc27]

Updated documentation comments at top of file

[jrtc27]

Ping for a compiler-rt-facing person's review?

compiler-rt/lib/builtins/CMakeLists.txt

 if(CMAKE_HOST_UNIX)
   set(COMPILER_RT_LINK_OR_COPY create_symlink)
 else()
   set(COMPILER_RT_LINK_OR_COPY copy)
 endif()
 
 foreach(pat cas swp ldadd ldclr ldeor ldset)
   foreach(size 1 2 4 8 16)
-    foreach(model 1 2 3 4)
+    foreach(model 1 2 3 4 5)
       if(pat STREQUAL "cas" OR NOT size STREQUAL "16")
         set(helper_asm "${OA_HELPERS_DIR}/outline_atomic_${pat}${size}_${model}.S")
         list(APPEND lse_builtins "${helper_asm}")
         list(APPEND arm64_lse_commands COMMAND ${CMAKE_COMMAND} -E ${COMPILER_RT_LINK_OR_COPY} "${CMAKE_CURRENT_SOURCE_DIR}/aarch64/lse.S" "${helper_asm}")
         set_source_files_properties("${helper_asm}"
           PROPERTIES
           COMPILE_DEFINITIONS "L_${pat};SIZE=${size};MODEL=${model}"
           INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}"

compiler-rt/lib/builtins/aarch64/lse.S

 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 #include "assembly.h"
 
 // Out-of-line LSE atomics helpers. Ported from libgcc library.
 // N = {1, 2, 4, 8}
 // M = {1, 2, 4, 8, 16}
-// ORDER = {'relax', 'acq', 'rel', 'acq_rel'}
+// ORDER = {'relax', 'acq', 'rel', 'acq_rel', 'sync'}
 // Routines implemented:
 //
 //  iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr)
 //  iN __aarch64_swpN_ORDER(iN val, iN *ptr)
 //  iN __aarch64_ldaddN_ORDER(iN val, iN *ptr)
 //  iN __aarch64_ldclrN_ORDER(iN val, iN *ptr)
 //  iN __aarch64_ldeorN_ORDER(iN val, iN *ptr)
 //  iN __aarch64_ldsetN_ORDER(iN val, iN *ptr)

 
 #if !defined(__APPLE__)
 HIDDEN(__aarch64_have_lse_atomics)
 #else
 HIDDEN(___aarch64_have_lse_atomics)
 #endif
 
 // Generate mnemonics for
-// L_cas:                                 SIZE: 1,2,4,8,16 MODEL: 1,2,3,4
-// L_swp L_ldadd L_ldclr L_ldeor L_ldset: SIZE: 1,2,4,8    MODEL: 1,2,3,4
+// L_cas:                                 SIZE: 1,2,4,8,16 MODEL: 1,2,3,4,5
+// L_swp L_ldadd L_ldclr L_ldeor L_ldset: SIZE: 1,2,4,8    MODEL: 1,2,3,4,5
 
 #if SIZE == 1
 #define S b
 #define UXT uxtb
 #define B 0x00000000
 #elif SIZE == 2
 #define S h
 #define UXT uxth

 #endif // SIZE
 
 #if MODEL == 1
 #define SUFF _relax
 #define A
 #define L
 #define M 0x000000
 #define N 0x000000
+#define BARRIER
 #elif MODEL == 2
 #define SUFF _acq
 #define A a
 #define L
 #define M 0x400000
 #define N 0x800000
+#define BARRIER
 #elif MODEL == 3
 #define SUFF _rel
 #define A
 #define L l
 #define M 0x008000
 #define N 0x400000
+#define BARRIER
 #elif MODEL == 4
 #define SUFF _acq_rel
 #define A a
 #define L l
 #define M 0x408000
 #define N 0xc00000
+#define BARRIER
+#elif MODEL == 5
+#define SUFF _sync
+#ifdef L_swp
+// swp has _acq semantics.
+#define A a
+#define L
+#define M 0x400000
+#define N 0x800000
+#else
+// All other _sync functions have _seq semantics.
+#define A a
+#define L l
+#define M 0x408000
+#define N 0xc00000
+#endif
+#define BARRIER dmb ish
 #else
 #error
 #endif // MODEL
 
 // Define register size.
 #define x(N) GLUE2(x, N)
 #define w(N) GLUE2(w, N)
 #if SIZE < 8
 #define s(N) w(N)
 #else
 #define s(N) x(N)
 #endif
 
 #define NAME(BASE) GLUE4(__aarch64_, BASE, SIZE, SUFF)
+#if MODEL == 5
+// Drop A for _sync functions.
+#define LDXR GLUE3(ld, xr, S)
+#else
 #define LDXR GLUE4(ld, A, xr, S)
+#endif
 #define STXR GLUE4(st, L, xr, S)
 
 // Define temporary registers.
 #define tmp0 16
 #define tmp1 17
 #define tmp2 15
 
 // Macro for branch to label if no LSE available
8:
         UXT    s(tmp0), s(0)
 0:
         LDXR   s(0), [x2]
         cmp    s(0), s(tmp0)
         bne    1f
         STXR   w(tmp1), s(1), [x2]
         cbnz   w(tmp1), 0b
 1:
+        BARRIER
         ret
 #else
+#if MODEL == 5
+// Drop A for _sync functions.
+#define LDXP GLUE2(ld, xp)
+#else
 #define LDXP GLUE3(ld, A, xp)
+#endif
 #define STXP GLUE3(st, L, xp)
 #ifdef HAS_ASM_LSE
 #define CASP GLUE3(casp, A, L)  x0, x1, x2, x3, [x4]
 #else
 #define CASP .inst 0x48207c82 + M
 #endif
 
         CASP   // x0, x1, x2, x3, [x4]
         ret
 8:
         mov    x(tmp0), x0
         mov    x(tmp1), x1
 0:
         LDXP   x0, x1, [x4]
         cmp    x0, x(tmp0)
         ccmp   x1, x(tmp1), #0, eq
         bne    1f
         STXP   w(tmp2), x2, x3, [x4]
         cbnz   w(tmp2), 0b
 1:
+        BARRIER
         ret
 #endif
 END_COMPILERRT_OUTLINE_FUNCTION(NAME(cas))
 #endif // L_cas
 
 #ifdef L_swp
 #ifdef HAS_ASM_LSE
 #define SWP GLUE4(swp, A, L, S)  s(0), s(0), [x1]
 #else
 #define SWP .inst 0x38208020 + B + N
 #endif
 DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(swp))
         JUMP_IF_NOT_LSE 8f
         SWP    // s(0), s(0), [x1]
         ret
 8:
         mov    s(tmp0), s(0)
 0:
         LDXR   s(0), [x1]
         STXR   w(tmp1), s(tmp0), [x1]
         cbnz   w(tmp1), 0b
+        BARRIER
         ret
 END_COMPILERRT_OUTLINE_FUNCTION(NAME(swp))
 #endif // L_swp
 
 #if defined(L_ldadd) || defined(L_ldclr) ||                                    \
     defined(L_ldeor) || defined(L_ldset)
 
 #ifdef L_ldadd
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(LDNM))
         ret
 8:
         mov    s(tmp0), s(0)
 0:
         LDXR   s(0), [x1]
         OP     s(tmp1), s(0), s(tmp0)
         STXR   w(tmp2), s(tmp1), [x1]
         cbnz   w(tmp2), 0b
+        BARRIER
         ret
 END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM))
 #endif // L_ldadd L_ldclr L_ldeor L_ldset
 
 NO_EXEC_STACK_DIRECTIVE
 
 // GNU property note for BTI and PAC
 GNU_PROPERTY_BTI_PAC
 
 #endif // __aarch64__