[OpenCL] Add OpenCL 3.0 atomics to -fdeclare-opencl-builtins

[svenvh]

Summary

Add the atomic overloads for the global and local address spaces,
which are new in OpenCL 3.0. Ensure the preexisting generic
overloads are guarded by the generic address space feature macro.

Ensure a subset of the atomic builtins are guarded by the
__opencl_c_atomic_order_seq_cst and __opencl_c_atomic_scope_device
feature macros, and enable those macros for SPIR/SPIR-V targets in
opencl-c-base.h.

Also guard the cl_ext_float_atomics builtins with the atomic order
and scope feature macros.

[Anastasia]

clang/lib/Sema/OpenCLBuiltins.td

1151–1159

maybe we can improve the name, so something like extension_fp16?

Same for below: extension_fp16, extension_32, extension_64?

clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl

84

Why do we need to set those for __OPENCL_CPP_VERSION__ == 100 since those atomics are for OpenCL 2.0 onwards?

[Anastasia]

LGTM! The rename can be made on commit.

clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl

84

Sorry C++ for OpenCL is aligned with OpenCL 2, so this check makes sense.

clang/lib/Headers/opencl-c-base.h

 #define __opencl_c_program_scope_global_variables 1
 #define __opencl_c_images 1
 #endif
 
 // Define header-only feature macros for OpenCL C 3.0.
 #if (__OPENCL_CPP_VERSION__ == 202100 || __OPENCL_C_VERSION__ == 300)
 // For the SPIR and SPIR-V target all features are supported.
 #if defined(__SPIR__) || defined(__SPIRV__)
+#define __opencl_c_atomic_order_seq_cst 1
+#define __opencl_c_atomic_scope_device 1
 #define __opencl_c_atomic_scope_all_devices 1
 #define __opencl_c_read_write_images 1
 #endif // defined(__SPIR__)
 #endif // (__OPENCL_CPP_VERSION__ == 202100 || __OPENCL_C_VERSION__ == 300)
 
 #if !defined(__opencl_c_generic_address_space)
 // Internal feature macro to provide named (global, local, private) address
 // space overloads for builtin functions that take a pointer argument.

clang/lib/Sema/OpenCLBuiltins.td

 
 // Extension associated to a type.  This enables implicit conditionalization of
 // builtin function overloads containing a type that depends on an extension.
 // During overload resolution, when a builtin function overload contains a type
 // with a TypeExtension, those overloads are skipped when the extension is
 // disabled.
 class TypeExtension<string _Ext> : AbstractExtension<_Ext>;
 
+// Concatenate zero or more space-separated extensions in NewExts to Base and
+// return the resulting FunctionExtension in ret.
+class concatExtension<FunctionExtension Base, string NewExts> {
+  FunctionExtension ret = FunctionExtension<
+    !cond(
+      // Return Base extension if NewExts is empty,
+      !empty(NewExts) : Base.ExtName,
+
+      // otherwise, return NewExts if Base extension is empty,
+      !empty(Base.ExtName) : NewExts,
+
+      // otherwise, concatenate NewExts to Base.
+      true : Base.ExtName # " " # NewExts
+    )
+  >;
+}
+
 // TypeExtension definitions.
 def NoTypeExt   : TypeExtension<"">;
 def Fp16TypeExt : TypeExtension<"cl_khr_fp16">;
 def Fp64TypeExt : TypeExtension<"cl_khr_fp64">;
 
 // FunctionExtension definitions.
 def FuncExtNone                          : FunctionExtension<"">;
 def FuncExtKhrSubgroups                  : FunctionExtension<"cl_khr_subgroups">;
foreach name = ["atomic_cmpxchg"] in {
       def : Builtin<name, [Type, PointerType<VolatileType<Type>, GenericAS>, Type, Type]>;
     }
   }
 }
 
 // OpenCL v2.0 s6.13.11 - Atomic Functions.
 
 // An atomic builtin with 2 additional _explicit variants.
-multiclass BuiltinAtomicExplicit<string Name, list<Type> Types> {
+multiclass BuiltinAtomicExplicit<string Name, list<Type> Types, FunctionExtension BaseExt> {
   // Without explicit MemoryOrder or MemoryScope.
+  let Extension = concatExtension<BaseExt, "__opencl_c_atomic_order_seq_cst __opencl_c_atomic_scope_device">.ret in {
     def : Builtin<Name, Types>;
+  }
 
   // With an explicit MemoryOrder argument.
+  let Extension = concatExtension<BaseExt, "__opencl_c_atomic_scope_device">.ret in {
     def : Builtin<Name # "_explicit", !listconcat(Types, [MemoryOrder])>;
+  }
 
   // With explicit MemoryOrder and MemoryScope arguments.
+  let Extension = BaseExt in {
     def : Builtin<Name # "_explicit", !listconcat(Types, [MemoryOrder, MemoryScope])>;
   }
+}
 
 // OpenCL 2.0 atomic functions that have a pointer argument in a given address space.
-multiclass OpenCL2Atomics<AddressSpace addrspace> {
+multiclass OpenCL2Atomics<AddressSpace addrspace, FunctionExtension BaseExt> {
   foreach TypePair = [[AtomicInt, Int], [AtomicUInt, UInt],
                       [AtomicLong, Long], [AtomicULong, ULong],
                       [AtomicFloat, Float], [AtomicDouble, Double]] in {
     def : Builtin<"atomic_init",
         [Void, PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[1]]>;
     defm : BuiltinAtomicExplicit<"atomic_store",
-        [Void, PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[1]]>;
+        [Void, PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[1]], BaseExt>;
     defm : BuiltinAtomicExplicit<"atomic_load",
-        [TypePair[1], PointerType<VolatileType<TypePair[0]>, addrspace>]>;
+        [TypePair[1], PointerType<VolatileType<TypePair[0]>, addrspace>], BaseExt>;
     defm : BuiltinAtomicExplicit<"atomic_exchange",
-        [TypePair[1], PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[1]]>;
+        [TypePair[1], PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[1]], BaseExt>;
     foreach Variant = ["weak", "strong"] in {
+      foreach exp_ptr_addrspace = !cond(
+            !eq(BaseExt, FuncExtOpenCLCGenericAddressSpace): [GenericAS],
+            !eq(BaseExt, FuncExtOpenCLCNamedAddressSpaceBuiltins): [GlobalAS, LocalAS, PrivateAS])
+          in {
+        let Extension = concatExtension<BaseExt, "__opencl_c_atomic_order_seq_cst __opencl_c_atomic_scope_device">.ret in {
           def : Builtin<"atomic_compare_exchange_" # Variant,
               [Bool, PointerType<VolatileType<TypePair[0]>, addrspace>,
-           PointerType<TypePair[1], addrspace>, TypePair[1]]>;
+               PointerType<TypePair[1], exp_ptr_addrspace>, TypePair[1]]>;
+        }
+        let Extension = concatExtension<BaseExt, "__opencl_c_atomic_scope_device">.ret in {
           def : Builtin<"atomic_compare_exchange_" # Variant # "_explicit",
               [Bool, PointerType<VolatileType<TypePair[0]>, addrspace>,
-           PointerType<TypePair[1], addrspace>, TypePair[1], MemoryOrder, MemoryOrder]>;
+               PointerType<TypePair[1], exp_ptr_addrspace>, TypePair[1], MemoryOrder, MemoryOrder]>;
+        }
+        let Extension = BaseExt in {
           def : Builtin<"atomic_compare_exchange_" # Variant # "_explicit",
               [Bool, PointerType<VolatileType<TypePair[0]>, addrspace>,
-           PointerType<TypePair[1], addrspace>, TypePair[1], MemoryOrder, MemoryOrder, MemoryScope]>;
+               PointerType<TypePair[1], exp_ptr_addrspace>, TypePair[1], MemoryOrder, MemoryOrder, MemoryScope]>;
+        }
+      }
     }
   }
 
   foreach TypePair = [[AtomicInt, Int, Int], [AtomicUInt, UInt, UInt],
                       [AtomicLong, Long, Long], [AtomicULong, ULong, ULong],
                       [AtomicUIntPtr, UIntPtr, PtrDiff]] in {
     foreach ModOp = ["add", "sub"] in {
       defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp,
-          [TypePair[1], PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[2]]>;
+          [TypePair[1], PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[2]], BaseExt>;
     }
   }
   foreach TypePair = [[AtomicInt, Int, Int], [AtomicUInt, UInt, UInt],
                       [AtomicLong, Long, Long], [AtomicULong, ULong, ULong]] in {
     foreach ModOp = ["or", "xor", "and", "min", "max"] in {
       defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp,
-          [TypePair[1], PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[2]]>;
+          [TypePair[1], PointerType<VolatileType<TypePair[0]>, addrspace>, TypePair[2]], BaseExt>;
     }
   }
 
   defm : BuiltinAtomicExplicit<"atomic_flag_clear",
-      [Void, PointerType<VolatileType<AtomicFlag>, addrspace>]>;
+      [Void, PointerType<VolatileType<AtomicFlag>, addrspace>], BaseExt>;
 
   defm : BuiltinAtomicExplicit<"atomic_flag_test_and_set",
-      [Bool, PointerType<VolatileType<AtomicFlag>, addrspace>]>;
+      [Bool, PointerType<VolatileType<AtomicFlag>, addrspace>], BaseExt>;
 }
 
 let MinVersion = CL20 in {
   def : Builtin<"atomic_work_item_fence", [Void, MemFenceFlags, MemoryOrder, MemoryScope]>;
 
-  defm : OpenCL2Atomics<GenericAS>;
+  defm : OpenCL2Atomics<GenericAS, FuncExtOpenCLCGenericAddressSpace>;
+  defm : OpenCL2Atomics<GlobalAS, FuncExtOpenCLCNamedAddressSpaceBuiltins>;
+  defm : OpenCL2Atomics<LocalAS, FuncExtOpenCLCNamedAddressSpaceBuiltins>;
 }
 
 // The functionality added by cl_ext_float_atomics extension
 let MinVersion = CL20 in {
   foreach addrspace = [GlobalAS, LocalAS, GenericAS] in {
-    let Extension = !cast<FunctionExtension>("FuncExtFloatAtomicsFp16" # addrspace # "LoadStore") in {
-      defm : BuiltinAtomicExplicit<"atomic_store",
-          [Void, PointerType<VolatileType<AtomicHalf>, addrspace>, AtomicHalf]>;
-      defm : BuiltinAtomicExplicit<"atomic_load",
-          [Half, PointerType<VolatileType<AtomicHalf>, addrspace>]>;
-      defm : BuiltinAtomicExplicit<"atomic_exchange",
-          [Half, PointerType<VolatileType<AtomicHalf>, addrspace>, Half]>;
-    }
+    defvar extension_fp16 = !cast<FunctionExtension>("FuncExtFloatAtomicsFp16" # addrspace # "LoadStore");
+
+        [Void, PointerType<VolatileType<AtomicHalf>, addrspace>, AtomicHalf], extension_fp16>;
+        [Half, PointerType<VolatileType<AtomicHalf>, addrspace>], extension_fp16>;
+        [Half, PointerType<VolatileType<AtomicHalf>, addrspace>, Half], extension_fp16>;
+
     foreach ModOp = ["add", "sub"] in {
-      let Extension = !cast<FunctionExtension>("FuncExtFloatAtomicsFp16" # addrspace # "Add") in {
-        defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp,
-            [Half, PointerType<VolatileType<AtomicHalf>, addrspace>, Half]>;
-      }
-      let Extension = !cast<FunctionExtension>("FuncExtFloatAtomicsFp32" # addrspace # "Add") in {
-        defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp,
-            [Float, PointerType<VolatileType<AtomicFloat>, addrspace>, Float]>;
-      }
-      let Extension = !cast<FunctionExtension>("FuncExtFloatAtomicsFp64" # addrspace # "Add") in {
-        defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp,
-            [Double, PointerType<VolatileType<AtomicDouble>, addrspace>, Double]>;
-      }
+      defvar extension_fp16 = !cast<FunctionExtension>("FuncExtFloatAtomicsFp16" # addrspace # "Add");
+      defvar extension_fp32 = !cast<FunctionExtension>("FuncExtFloatAtomicsFp32" # addrspace # "Add");
+      defvar extension_fp64 = !cast<FunctionExtension>("FuncExtFloatAtomicsFp64" # addrspace # "Add");
+
+          [Half, PointerType<VolatileType<AtomicFloat>, addrspace>, Half], extension_fp16>;
+          [Float, PointerType<VolatileType<AtomicFloat>, addrspace>, Float], extension_fp32>;
+          [Double, PointerType<VolatileType<AtomicDouble>, addrspace>, Double], extension_fp64>;
     }
+
     foreach ModOp = ["min", "max"] in {
-      let Extension = !cast<FunctionExtension>("FuncExtFloatAtomicsFp16" # addrspace # "MinMax") in {
-        defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp,
-            [Half, PointerType<VolatileType<AtomicHalf>, addrspace>, Half]>;
-      }
-      let Extension = !cast<FunctionExtension>("FuncExtFloatAtomicsFp32" # addrspace # "MinMax") in {
-        defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp,
-            [Float, PointerType<VolatileType<AtomicFloat>, addrspace>, Float]>;
-      }
-      let Extension = !cast<FunctionExtension>("FuncExtFloatAtomicsFp64" # addrspace # "MinMax") in {
-        defm : BuiltinAtomicExplicit<"atomic_fetch_" # ModOp,
-            [Double, PointerType<VolatileType<AtomicDouble>, addrspace>, Double]>;
-      }
+      defvar extension_fp16 = !cast<FunctionExtension>("FuncExtFloatAtomicsFp16" # addrspace # "MinMax");
+      defvar extension_fp32 = !cast<FunctionExtension>("FuncExtFloatAtomicsFp32" # addrspace # "MinMax");
+      defvar extension_fp64 = !cast<FunctionExtension>("FuncExtFloatAtomicsFp64" # addrspace # "MinMax");
+
+          [Half, PointerType<VolatileType<AtomicHalf>, addrspace>, Half], extension_fp16>;
+          [Float, PointerType<VolatileType<AtomicFloat>, addrspace>, Float], extension_fp32>;
+          [Double, PointerType<VolatileType<AtomicDouble>, addrspace>, Double], extension_fp64>;
     }
   }
 }
 
 //--------------------------------------------------------------------
 // OpenCL v1.1 s6.11.12, v1.2 s6.12.12, v2.0 s6.13.12 - Miscellaneous Vector Functions
 // --- Table 19 ---
 foreach VSize1 = [2, 4, 8, 16] in {

clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl

 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -fdeclare-opencl-builtins -DNO_HEADER
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -fdeclare-opencl-builtins -finclude-default-header
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL1.2 -fdeclare-opencl-builtins -DNO_HEADER
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL1.2 -fdeclare-opencl-builtins -finclude-default-header
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL2.0 -fdeclare-opencl-builtins -DNO_HEADER
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL2.0 -fdeclare-opencl-builtins -finclude-default-header
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL3.0 -fdeclare-opencl-builtins -finclude-default-header
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CLC++ -fdeclare-opencl-builtins -DNO_HEADER
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CLC++ -fdeclare-opencl-builtins -finclude-default-header
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CLC++2021 -fdeclare-opencl-builtins -finclude-default-header
 // RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL2.0 -fdeclare-opencl-builtins -finclude-default-header -cl-ext=-cl_khr_fp64 -DNO_FP64
+// RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL3.0 -fdeclare-opencl-builtins -finclude-default-header -DNO_ATOMSCOPE
 
 // Test the -fdeclare-opencl-builtins option.  This is not a completeness
 // test, so it should not test for all builtins defined by OpenCL.  Instead
 // this test should cover different functional aspects of the TableGen builtin
 // function machinery.
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 #if __OPENCL_C_VERSION__ < CL_VERSION_1_2

 #define __opencl_c_generic_address_space 1
 #define cl_khr_subgroup_extended_types 1
 #define cl_khr_subgroup_ballot 1
 #define cl_khr_subgroup_non_uniform_arithmetic 1
 #define cl_khr_subgroup_clustered_reduce 1
 #define __opencl_c_read_write_images 1
 #endif
 
+#if (__OPENCL_CPP_VERSION__ == 100 || __OPENCL_C_VERSION__ == 200)
+#define __opencl_c_atomic_order_seq_cst 1
+#define __opencl_c_atomic_scope_device 1
+#endif
+
 #define __opencl_c_named_address_space_builtins 1
 #endif
 
 kernel void test_pointers(volatile global void *global_p, global const int4 *a) {
   int i;
   unsigned int ui;
 
   prefetch(a, 2);
 
   atom_add((volatile __global int *)global_p, i);
   atom_cmpxchg((volatile __global unsigned int *)global_p, ui, ui);
 }
 
 // Only test enum arguments when the base header is included, because we need
 // the enum declarations.
 #if !defined(NO_HEADER) && (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
 kernel void test_enum_args(volatile global atomic_int *global_p, global int *expected) {
   int desired;
+  atomic_work_item_fence(CLK_GLOBAL_MEM_FENCE, memory_order_acq_rel, memory_scope_device);
   atomic_compare_exchange_strong_explicit(global_p, expected, desired,
                                           memory_order_acq_rel,
                                           memory_order_relaxed,
                                           memory_scope_work_group);
 }
 #endif
 
 #if defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200
void test_atomic_fetch_with_address_space(volatile __generic atomic_float *a_float,
   resf1 = atomic_fetch_add_explicit(a_float_global, f1, memory_order_seq_cst, memory_scope_work_group);
 
   resd1 = atomic_fetch_min(a_double, d1);
   resd1 = atomic_fetch_max_explicit(a_double_local, d1, memory_order_seq_cst);
   resd1 = atomic_fetch_add_explicit(a_double_global, d1, memory_order_seq_cst, memory_scope_work_group);
 }
 #endif // !defined(NO_HEADER) && __OPENCL_C_VERSION__ >= 200
 
+#if defined(NO_ATOMSCOPE) && __OPENCL_C_VERSION__ >= 300
+// Disable the feature by undefining the feature macro.
+#undef __opencl_c_atomic_scope_device
+
+// Test that only the overload with explicit order and scope arguments is
+// available when the __opencl_c_atomic_scope_device feature is disabled.
+void test_atomics_without_scope_device(volatile __generic atomic_int *a_int) {
+  int d;
+
+  atomic_exchange(a_int, d);
+  // expected-error@-1{{implicit declaration of function 'atomic_exchange' is invalid in OpenCL}}
+
+  atomic_exchange_explicit(a_int, d, memory_order_seq_cst);
+  // expected-error@-1{{no matching function for call to 'atomic_exchange_explicit'}}
+  // expected-note@-2 + {{candidate function not viable}}
+
+  atomic_exchange_explicit(a_int, d, memory_order_seq_cst, memory_scope_work_group);
+}
+
+#endif
+
 // Test old atomic overloaded with generic address space in C++ for OpenCL.
 #if __OPENCL_C_VERSION__ >= 200
 void test_legacy_atomics_cpp(__generic volatile unsigned int *a) {
   atomic_add(a, 1);
 #if !defined(__cplusplus)
   // expected-error@-2{{no matching function for call to 'atomic_add'}}
   // expected-note@-3 4 {{candidate function not viable}}
 #endif