[OpenMP] libomp: add OpenMP 5.1 memory allocation routines

[AndreyChurbanov]

Summary

Aligned allocation routines added.
Fortran interfaces added for all allocation routines.

[Nawrin]

LGTM

openmp/runtime/src/dllexports

kmp_set_disp_num_buffers                    890
     omp_free                                895
     omp_get_device_num                      896
     omp_init_allocator                      897
     omp_destroy_allocator                   898
     %ifndef stub
         __kmpc_set_default_allocator
         __kmpc_get_default_allocator
         __kmpc_alloc
+        __kmpc_aligned_alloc
         __kmpc_calloc
         __kmpc_realloc
         __kmpc_free
         __kmpc_init_allocator
         __kmpc_destroy_allocator
     %endif
     omp_set_affinity_format                 748
     omp_get_affinity_format                 749
kmp_set_disp_num_buffers                    890
     omp_fulfill_event                       759
     omp_set_num_teams                       802
     omp_get_max_teams                       803
     omp_set_teams_thread_limit              804
     omp_get_teams_thread_limit              805
     omp_display_env                         733
     omp_calloc                              776
     omp_realloc                             777
+    omp_aligned_alloc                       778
+    omp_aligned_calloc                      806
 
     omp_null_allocator                     DATA
     omp_default_mem_alloc                  DATA
     omp_large_cap_mem_alloc                DATA
     omp_const_mem_alloc                    DATA
     omp_high_bw_mem_alloc                  DATA
     omp_low_lat_mem_alloc                  DATA
     omp_cgroup_mem_alloc                   DATA

openmp/runtime/src/include/omp.h.var

#   endif
     extern omp_allocator_handle_t __KAI_KMPC_CONVENTION omp_init_allocator(omp_memspace_handle_t m,
                                                        int ntraits, omp_alloctrait_t traits[]);
     extern void __KAI_KMPC_CONVENTION omp_destroy_allocator(omp_allocator_handle_t allocator);
 
     extern void __KAI_KMPC_CONVENTION omp_set_default_allocator(omp_allocator_handle_t a);
     extern omp_allocator_handle_t __KAI_KMPC_CONVENTION omp_get_default_allocator(void);
 #   ifdef __cplusplus
     extern void *__KAI_KMPC_CONVENTION omp_alloc(size_t size, omp_allocator_handle_t a = omp_null_allocator);
-    extern void *__KAI_KMPC_CONVENTION omp_calloc(size_t nmemb, size_t size, omp_allocator_handle_t a = omp_null_allocator);
+    extern void *__KAI_KMPC_CONVENTION omp_aligned_alloc(size_t align, size_t size,
+                                                         omp_allocator_handle_t a = omp_null_allocator);
+    extern void *__KAI_KMPC_CONVENTION omp_calloc(size_t nmemb, size_t size,
+                                                  omp_allocator_handle_t a = omp_null_allocator);
+    extern void *__KAI_KMPC_CONVENTION omp_aligned_calloc(size_t align, size_t nmemb, size_t size,
+                                                          omp_allocator_handle_t a = omp_null_allocator);
     extern void *__KAI_KMPC_CONVENTION omp_realloc(void *ptr, size_t size,
                                                    omp_allocator_handle_t allocator = omp_null_allocator,
                                                    omp_allocator_handle_t free_allocator = omp_null_allocator);
     extern void __KAI_KMPC_CONVENTION omp_free(void * ptr, omp_allocator_handle_t a = omp_null_allocator);
 #   else
     extern void *__KAI_KMPC_CONVENTION omp_alloc(size_t size, omp_allocator_handle_t a);
+    extern void *__KAI_KMPC_CONVENTION omp_aligned_alloc(size_t align, size_t size,
+                                                         omp_allocator_handle_t a);
     extern void *__KAI_KMPC_CONVENTION omp_calloc(size_t nmemb, size_t size, omp_allocator_handle_t a);
+    extern void *__KAI_KMPC_CONVENTION omp_aligned_calloc(size_t align, size_t nmemb, size_t size,
+                                                          omp_allocator_handle_t a);
     extern void *__KAI_KMPC_CONVENTION omp_realloc(void *ptr, size_t size, omp_allocator_handle_t allocator,
                                                    omp_allocator_handle_t free_allocator);
     extern void __KAI_KMPC_CONVENTION omp_free(void *ptr, omp_allocator_handle_t a);
 #   endif
 
     /* OpenMP 5.0 Affinity Format */
     extern void __KAI_KMPC_CONVENTION omp_set_affinity_format(char const *);
     extern size_t __KAI_KMPC_CONVENTION omp_get_affinity_format(char *, size_t);

openmp/runtime/src/include/omp_lib.h.var

&      device_offset, device_num) bind(c)
         function omp_target_is_accessible(ptr, size, device_num) bind(c)
           use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t, c_int
           integer(c_int) omp_target_is_accessible
           type(c_ptr), value :: ptr
           integer(c_size_t), value :: size
           integer(c_int), value :: device_num
         end function omp_target_is_accessible
 
+        function omp_alloc(size, allocator) bind(c)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+          type(c_ptr) omp_alloc
+          integer(c_size_t), value :: size
+          integer(omp_allocator_handle_kind), value :: allocator
+        end function omp_alloc
+
+        function omp_aligned_alloc(alignment, size, allocator) bind(c)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+          type(c_ptr) omp_aligned_alloc
+          integer(c_size_t), value :: alignment, size
+          integer(omp_allocator_handle_kind), value :: allocator
+        end function omp_aligned_alloc
+
+        function omp_calloc(nmemb, size, allocator) bind(c)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+          type(c_ptr) omp_calloc
+          integer(c_size_t), value :: nmemb, size
+          integer(omp_allocator_handle_kind), value :: allocator
+        end function omp_calloc
+
+        function omp_aligned_calloc(alignment, nmemb, size, allocator) bind(c)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+          type(c_ptr) omp_aligned_calloc
+          integer(c_size_t), value :: alignment, nmemb, size
+          integer(omp_allocator_handle_kind), value :: allocator
+        end function omp_aligned_calloc
+
+        function omp_realloc(ptr, size, allocator, free_allocator) bind(c)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+          type(c_ptr) omp_realloc
+          type(c_ptr), value :: ptr
+          integer(c_size_t), value :: size
+          integer(omp_allocator_handle_kind), value :: allocator
+          integer(omp_allocator_handle_kind), value :: free_allocator
+        end function omp_realloc
+
+        subroutine omp_free(ptr, allocator) bind(c)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only : c_ptr
+          type(c_ptr), value :: ptr
+          integer(omp_allocator_handle_kind), value :: allocator
+        end subroutine omp_free
+
 !       ***
 !       *** kmp_* entry points
 !       ***
 
         subroutine kmp_set_stacksize(size) bind(c)
           import
           integer (kind=omp_integer_kind), value :: size
         end subroutine kmp_set_stacksize

openmp/runtime/src/include/omp_lib.f90.var

!DIR$ ENDIF
             use omp_lib_kinds
             use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t, c_int
             integer(c_int) omp_target_is_accessible
             type(c_ptr), value :: ptr
             integer(c_size_t), value :: size
             integer(c_int), value :: device_num
           end function omp_target_is_accessible
 
+          function omp_alloc(size, allocator) bind(c)
+            use omp_lib_kinds
+            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            type(c_ptr) omp_alloc
+            integer(c_size_t), value :: size
+            integer(omp_allocator_handle_kind), value :: allocator
+          end function omp_alloc
+
+          function omp_aligned_alloc(alignment, size, allocator) bind(c)
+            use omp_lib_kinds
+            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            type(c_ptr) omp_aligned_alloc
+            integer(c_size_t), value :: alignment, size
+            integer(omp_allocator_handle_kind), value :: allocator
+          end function omp_aligned_alloc
+
+          function omp_calloc(nmemb, size, allocator) bind(c)
+            use omp_lib_kinds
+            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            type(c_ptr) omp_calloc
+            integer(c_size_t), value :: nmemb, size
+            integer(omp_allocator_handle_kind), value :: allocator
+          end function omp_calloc
+
+          function omp_aligned_calloc(alignment, nmemb, size, allocator) bind(c)
+            use omp_lib_kinds
+            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            type(c_ptr) omp_aligned_calloc
+            integer(c_size_t), value :: alignment, nmemb, size
+            integer(omp_allocator_handle_kind), value :: allocator
+          end function omp_aligned_calloc
+
+          function omp_realloc(ptr, size, allocator, free_allocator) bind(c)
+            use omp_lib_kinds
+            use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
+            type(c_ptr) omp_realloc
+            type(c_ptr), value :: ptr
+            integer(c_size_t), value :: size
+            integer(omp_allocator_handle_kind), value :: allocator
+            integer(omp_allocator_handle_kind), value :: free_allocator
+          end function omp_realloc
+
+          subroutine omp_free(ptr, allocator) bind(c)
+            use omp_lib_kinds
+            use, intrinsic :: iso_c_binding, only : c_ptr
+            type(c_ptr), value :: ptr
+            integer(omp_allocator_handle_kind), value :: allocator
+          end subroutine omp_free
+
 !         ***
 !         *** kmp_* entry points
 !         ***
 
           subroutine kmp_set_stacksize(size) bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: size
           end subroutine kmp_set_stacksize

openmp/runtime/src/kmp.h

 
 extern int __kmp_memkind_available;
 
 typedef omp_memspace_handle_t kmp_memspace_t; // placeholder
 
 typedef struct kmp_allocator_t {
   omp_memspace_handle_t memspace;
   void **memkind; // pointer to memkind
-  int alignment;
+  size_t alignment;
   omp_alloctrait_value_t fb;
   kmp_allocator_t *fb_data;
   kmp_uint64 pool_size;
   kmp_uint64 pool_used;
 } kmp_allocator_t;
 
 extern omp_allocator_handle_t __kmpc_init_allocator(int gtid,
                                                     omp_memspace_handle_t,
                                                     int ntraits,
                                                     omp_alloctrait_t traits[]);
 extern void __kmpc_destroy_allocator(int gtid, omp_allocator_handle_t al);
 extern void __kmpc_set_default_allocator(int gtid, omp_allocator_handle_t al);
 extern omp_allocator_handle_t __kmpc_get_default_allocator(int gtid);
+// external interfaces, may be used by compiler
 extern void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
+extern void *__kmpc_aligned_alloc(int gtid, size_t align, size_t sz,
+                                  omp_allocator_handle_t al);
 extern void *__kmpc_calloc(int gtid, size_t nmemb, size_t sz,
                            omp_allocator_handle_t al);
 extern void *__kmpc_realloc(int gtid, void *ptr, size_t sz,
                             omp_allocator_handle_t al,
                             omp_allocator_handle_t free_al);
 extern void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
+// internal interfaces, contain real implementation
+extern void *__kmp_alloc(int gtid, size_t align, size_t sz,
+                         omp_allocator_handle_t al);
+extern void *__kmp_calloc(int gtid, size_t align, size_t nmemb, size_t sz,
+                          omp_allocator_handle_t al);
+extern void *__kmp_realloc(int gtid, void *ptr, size_t sz,
+                           omp_allocator_handle_t al,
+                           omp_allocator_handle_t free_al);
+extern void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
 
 extern void __kmp_init_memkind();
 extern void __kmp_fini_memkind();
 extern void __kmp_init_target_mem();
 
 /* ------------------------------------------------------------------------ */
 
 #define KMP_UINT64_MAX                                                         \

openmp/runtime/src/kmp_alloc.cpp

 
 typedef struct kmp_mem_desc { // Memory block descriptor
   void *ptr_alloc; // Pointer returned by allocator
   size_t size_a; // Size of allocated memory block (initial+descriptor+align)
   size_t size_orig; // Original size requested
   void *ptr_align; // Pointer to aligned memory, returned
   kmp_allocator_t *allocator; // allocator
 } kmp_mem_desc_t;
-static int alignment = sizeof(void *); // let's align to pointer size
+static int alignment = sizeof(void *); // align to pointer size by default
 
+// external interfaces are wrappers over internal implementation
 void *__kmpc_alloc(int gtid, size_t size, omp_allocator_handle_t allocator) {
+  KE_TRACE(25, ("__kmpc_alloc: T#%d (%d, %p)\n", gtid, (int)size, allocator));
+  void *ptr = __kmp_alloc(gtid, 0, size, allocator);
+  KE_TRACE(25, ("__kmpc_alloc returns %p, T#%d\n", ptr, gtid));
+  return ptr;
+}
+
+void *__kmpc_aligned_alloc(int gtid, size_t algn, size_t size,
+                           omp_allocator_handle_t allocator) {
+  KE_TRACE(25, ("__kmpc_aligned_alloc: T#%d (%d, %d, %p)\n", gtid, (int)algn,
+                (int)size, allocator));
+  void *ptr = __kmp_alloc(gtid, algn, size, allocator);
+  KE_TRACE(25, ("__kmpc_aligned_alloc returns %p, T#%d\n", ptr, gtid));
+  return ptr;
+}
+
+void *__kmpc_calloc(int gtid, size_t nmemb, size_t size,
+                    omp_allocator_handle_t allocator) {
+  KE_TRACE(25, ("__kmpc_calloc: T#%d (%d, %d, %p)\n", gtid, (int)nmemb,
+                (int)size, allocator));
+  void *ptr = __kmp_calloc(gtid, 0, nmemb, size, allocator);
+  KE_TRACE(25, ("__kmpc_calloc returns %p, T#%d\n", ptr, gtid));
+  return ptr;
+}
+
+void *__kmpc_realloc(int gtid, void *ptr, size_t size,
+                     omp_allocator_handle_t allocator,
+                     omp_allocator_handle_t free_allocator) {
+  KE_TRACE(25, ("__kmpc_realloc: T#%d (%p, %d, %p, %p)\n", gtid, ptr, (int)size,
+                allocator, free_allocator));
+  void *nptr = __kmp_realloc(gtid, ptr, size, allocator, free_allocator);
+  KE_TRACE(25, ("__kmpc_realloc returns %p, T#%d\n", nptr, gtid));
+  return nptr;
+}
+
+void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t allocator) {
+  KE_TRACE(25, ("__kmpc_free: T#%d free(%p,%p)\n", gtid, ptr, allocator));
+  ___kmpc_free(gtid, ptr, allocator);
+  KE_TRACE(10, ("__kmpc_free: T#%d freed %p (%p)\n", gtid, ptr, allocator));
+  return;
+}
+
+// internal implementation, called from inside the library
+void *__kmp_alloc(int gtid, size_t algn, size_t size,
+                  omp_allocator_handle_t allocator) {
   void *ptr = NULL;
   kmp_allocator_t *al;
   KMP_DEBUG_ASSERT(__kmp_init_serial);
-
   if (size == 0)
     return NULL;
-
   if (allocator == omp_null_allocator)
     allocator = __kmp_threads[gtid]->th.th_def_allocator;
 
-  KE_TRACE(25, ("__kmpc_alloc: T#%d (%d, %p)\n", gtid, (int)size, allocator));
-  al = RCAST(kmp_allocator_t *, CCAST(omp_allocator_handle_t, allocator));
+  al = RCAST(kmp_allocator_t *, allocator);
 
   int sz_desc = sizeof(kmp_mem_desc_t);
   kmp_mem_desc_t desc;
   kmp_uintptr_t addr; // address returned by allocator
   kmp_uintptr_t addr_align; // address to return to caller
   kmp_uintptr_t addr_descr; // address of memory block descriptor
-  int align = alignment; // default alignment
-  if (allocator > kmp_max_mem_alloc && al->alignment > 0) {
-    align = al->alignment; // alignment requested by user
-  }
+  size_t align = alignment; // default alignment
+  if (allocator > kmp_max_mem_alloc && al->alignment > align)
+    align = al->alignment; // alignment required by allocator trait
+  if (align < algn)
+    align = algn; // max of allocator trait, parameter and sizeof(void*)
   desc.size_orig = size;
   desc.size_a = size + sz_desc + align;
 
   if (__kmp_memkind_available) {
     if (allocator < kmp_max_mem_alloc) {
       // pre-defined allocator
       if (allocator == omp_high_bw_mem_alloc && mk_hbw_preferred) {
         ptr = kmp_mk_alloc(*mk_hbw_preferred, desc.size_a);
if (allocator < kmp_max_mem_alloc) {
         if (al->fb == omp_atv_default_mem_fb) {
           al = (kmp_allocator_t *)omp_default_mem_alloc;
           ptr = kmp_mk_alloc(*mk_default, desc.size_a);
         } else if (al->fb == omp_atv_abort_fb) {
           KMP_ASSERT(0); // abort fallback requested
         } else if (al->fb == omp_atv_allocator_fb) {
           KMP_ASSERT(al != al->fb_data);
           al = al->fb_data;
-          return __kmpc_alloc(gtid, size, (omp_allocator_handle_t)al);
+          return __kmp_alloc(gtid, algn, size, (omp_allocator_handle_t)al);
         } // else ptr == NULL;
       } else {
         // pool has enough space
         ptr = kmp_mk_alloc(*al->memkind, desc.size_a);
         if (ptr == NULL) {
           if (al->fb == omp_atv_default_mem_fb) {
             al = (kmp_allocator_t *)omp_default_mem_alloc;
             ptr = kmp_mk_alloc(*mk_default, desc.size_a);
           } else if (al->fb == omp_atv_abort_fb) {
             KMP_ASSERT(0); // abort fallback requested
           } else if (al->fb == omp_atv_allocator_fb) {
             KMP_ASSERT(al != al->fb_data);
             al = al->fb_data;
-            return __kmpc_alloc(gtid, size, (omp_allocator_handle_t)al);
+            return __kmp_alloc(gtid, algn, size, (omp_allocator_handle_t)al);
           }
         }
       }
     } else {
       // custom allocator, pool size not requested
       ptr = kmp_mk_alloc(*al->memkind, desc.size_a);
       if (ptr == NULL) {
         if (al->fb == omp_atv_default_mem_fb) {
           al = (kmp_allocator_t *)omp_default_mem_alloc;
           ptr = kmp_mk_alloc(*mk_default, desc.size_a);
         } else if (al->fb == omp_atv_abort_fb) {
           KMP_ASSERT(0); // abort fallback requested
         } else if (al->fb == omp_atv_allocator_fb) {
           KMP_ASSERT(al != al->fb_data);
           al = al->fb_data;
-          return __kmpc_alloc(gtid, size, (omp_allocator_handle_t)al);
+          return __kmp_alloc(gtid, algn, size, (omp_allocator_handle_t)al);
         }
       }
     }
   } else if (allocator < kmp_max_mem_alloc) {
     if (KMP_IS_TARGET_MEM_ALLOC(allocator)) {
       // Use size input directly as the memory may not be accessible on host.
       // Use default device for now.
       if (__kmp_target_mem_available) {
if (used + desc.size_a > al->pool_size) {
       if (al->fb == omp_atv_default_mem_fb) {
         al = (kmp_allocator_t *)omp_default_mem_alloc;
         ptr = __kmp_thread_malloc(__kmp_thread_from_gtid(gtid), desc.size_a);
       } else if (al->fb == omp_atv_abort_fb) {
         KMP_ASSERT(0); // abort fallback requested
       } else if (al->fb == omp_atv_allocator_fb) {
         KMP_ASSERT(al != al->fb_data);
         al = al->fb_data;
-        return __kmpc_alloc(gtid, size, (omp_allocator_handle_t)al);
+        return __kmp_alloc(gtid, algn, size, (omp_allocator_handle_t)al);
       } // else ptr == NULL;
     } else {
       // pool has enough space
       ptr = __kmp_thread_malloc(__kmp_thread_from_gtid(gtid), desc.size_a);
       if (ptr == NULL && al->fb == omp_atv_abort_fb) {
         KMP_ASSERT(0); // abort fallback requested
       } // no sense to look for another fallback because of same internal alloc
     }
   } else {
     // custom allocator, pool size not requested
     ptr = __kmp_thread_malloc(__kmp_thread_from_gtid(gtid), desc.size_a);
     if (ptr == NULL && al->fb == omp_atv_abort_fb) {
       KMP_ASSERT(0); // abort fallback requested
     } // no sense to look for another fallback because of same internal alloc
   }
-  KE_TRACE(10, ("__kmpc_alloc: T#%d %p=alloc(%d)\n", gtid, ptr, desc.size_a));
+  KE_TRACE(10, ("__kmp_alloc: T#%d %p=alloc(%d)\n", gtid, ptr, desc.size_a));
   if (ptr == NULL)
     return NULL;
 
   addr = (kmp_uintptr_t)ptr;
   addr_align = (addr + sz_desc + align - 1) & ~(align - 1);
   addr_descr = addr_align - sz_desc;
 
   desc.ptr_alloc = ptr;
   desc.ptr_align = (void *)addr_align;
   desc.allocator = al;
   *((kmp_mem_desc_t *)addr_descr) = desc; // save descriptor contents
   KMP_MB();
 
-  KE_TRACE(25, ("__kmpc_alloc returns %p, T#%d\n", desc.ptr_align, gtid));
   return desc.ptr_align;
 }
 
-void *__kmpc_calloc(int gtid, size_t nmemb, size_t size,
-                    omp_allocator_handle_t allocator) {
+void *__kmp_calloc(int gtid, size_t algn, size_t nmemb, size_t size,
   void *ptr = NULL;
   kmp_allocator_t *al;
   KMP_DEBUG_ASSERT(__kmp_init_serial);
 
   if (allocator == omp_null_allocator)
     allocator = __kmp_threads[gtid]->th.th_def_allocator;
 
-  KE_TRACE(25, ("__kmpc_calloc: T#%d (%d, %d, %p)\n", gtid, (int)nmemb,
-                (int)size, allocator));
-
-  al = RCAST(kmp_allocator_t *, CCAST(omp_allocator_handle_t, allocator));
+  al = RCAST(kmp_allocator_t *, allocator);
 
   if (nmemb == 0 || size == 0)
     return ptr;
 
   if ((SIZE_MAX - sizeof(kmp_mem_desc_t)) / size < nmemb) {
     if (al->fb == omp_atv_abort_fb) {
       KMP_ASSERT(0);
     }
     return ptr;
   }
 
-  ptr = __kmpc_alloc(gtid, nmemb * size, allocator);
+  ptr = __kmp_alloc(gtid, algn, nmemb * size, allocator);
 
   if (ptr) {
     memset(ptr, 0x00, nmemb * size);
   }
-  KE_TRACE(25, ("__kmpc_calloc returns %p, T#%d\n", ptr, gtid));
   return ptr;
 }
 
-void *__kmpc_realloc(int gtid, void *ptr, size_t size,
-                     omp_allocator_handle_t allocator,
-                     omp_allocator_handle_t free_allocator) {
+void *__kmp_realloc(int gtid, void *ptr, size_t size,
   void *nptr = NULL;
   KMP_DEBUG_ASSERT(__kmp_init_serial);
 
   if (size == 0) {
     if (ptr != NULL)
-      __kmpc_free(gtid, ptr, free_allocator);
+      ___kmpc_free(gtid, ptr, free_allocator);
     return nptr;
   }
 
-  KE_TRACE(25, ("__kmpc_realloc: T#%d (%p, %d, %p, %p)\n", gtid, ptr, (int)size,
-                allocator, free_allocator));
-
-  nptr = __kmpc_alloc(gtid, size, allocator);
+  nptr = __kmp_alloc(gtid, 0, size, allocator);
 
   if (nptr != NULL && ptr != NULL) {
     kmp_mem_desc_t desc;
     kmp_uintptr_t addr_align; // address to return to caller
     kmp_uintptr_t addr_descr; // address of memory block descriptor
 
     addr_align = (kmp_uintptr_t)ptr;
     addr_descr = addr_align - sizeof(kmp_mem_desc_t);
     desc = *((kmp_mem_desc_t *)addr_descr); // read descriptor
 
     KMP_DEBUG_ASSERT(desc.ptr_align == ptr);
     KMP_DEBUG_ASSERT(desc.size_orig > 0);
     KMP_DEBUG_ASSERT(desc.size_orig < desc.size_a);
     KMP_MEMCPY((char *)nptr, (char *)ptr,
                (size_t)((size < desc.size_orig) ? size : desc.size_orig));
   }
 
   if (nptr != NULL) {
-    __kmpc_free(gtid, ptr, free_allocator);
+    ___kmpc_free(gtid, ptr, free_allocator);
   }
 
-  KE_TRACE(25, ("__kmpc_realloc returns %p, T#%d\n", nptr, gtid));
   return nptr;
 }
 
-void __kmpc_free(int gtid, void *ptr, const omp_allocator_handle_t allocator) {
-  KE_TRACE(25, ("__kmpc_free: T#%d free(%p,%p)\n", gtid, ptr, allocator));
+void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t allocator) {
   if (ptr == NULL)
     return;
 
   kmp_allocator_t *al;
   omp_allocator_handle_t oal;
   al = RCAST(kmp_allocator_t *, CCAST(omp_allocator_handle_t, allocator));
   kmp_mem_desc_t desc;
   kmp_uintptr_t addr_align; // address to return to caller
if (__kmp_memkind_available) {
     if (oal > kmp_max_mem_alloc && al->pool_size > 0) {
       kmp_uint64 used =
           KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a);
       (void)used; // to suppress compiler warning
       KMP_DEBUG_ASSERT(used >= desc.size_a);
     }
     __kmp_thread_free(__kmp_thread_from_gtid(gtid), desc.ptr_alloc);
   }
-  KE_TRACE(10, ("__kmpc_free: T#%d freed %p (%p)\n", gtid, desc.ptr_alloc,
-                allocator));
 }
 
 /* If LEAK_MEMORY is defined, __kmp_free() will *not* free memory. It causes
    memory leaks, but it may be useful for debugging memory corruptions, used
    freed pointers, etc. */
 /* #define LEAK_MEMORY */
 struct kmp_mem_descr { // Memory block descriptor.
   void *ptr_allocated; // Pointer returned by malloc(), subject for free().

openmp/runtime/src/kmp_csupport.cpp

void __kmpc_doacross_fini(ident_t *loc, int gtid) {
   }
   // free private resources (need to keep buffer index forever)
   pr_buf->th_doacross_flags = NULL;
   __kmp_thread_free(th, (void *)pr_buf->th_doacross_info);
   pr_buf->th_doacross_info = NULL;
   KA_TRACE(20, ("__kmpc_doacross_fini() exit: T#%d\n", gtid));
 }
 
-/* omp_alloc/omp_calloc/omp_free only defined for C/C++, not for Fortran */
+/* OpenMP 5.1 Memory Management routines */
 void *omp_alloc(size_t size, omp_allocator_handle_t allocator) {
-  return __kmpc_alloc(__kmp_entry_gtid(), size, allocator);
+  return __kmp_alloc(__kmp_entry_gtid(), 0, size, allocator);
+}
+
+void *omp_aligned_alloc(size_t align, size_t size,
+                        omp_allocator_handle_t allocator) {
+  return __kmp_alloc(__kmp_entry_gtid(), align, size, allocator);
 }
 
 void *omp_calloc(size_t nmemb, size_t size, omp_allocator_handle_t allocator) {
-  return __kmpc_calloc(__kmp_entry_gtid(), nmemb, size, allocator);
+  return __kmp_calloc(__kmp_entry_gtid(), 0, nmemb, size, allocator);
+}
+
+void *omp_aligned_calloc(size_t align, size_t nmemb, size_t size,
+                         omp_allocator_handle_t allocator) {
+  return __kmp_calloc(__kmp_entry_gtid(), align, nmemb, size, allocator);
 }
 
 void *omp_realloc(void *ptr, size_t size, omp_allocator_handle_t allocator,
                   omp_allocator_handle_t free_allocator) {
-  return __kmpc_realloc(__kmp_entry_gtid(), ptr, size, allocator,
+  return __kmp_realloc(__kmp_entry_gtid(), ptr, size, allocator,
                         free_allocator);
 }
 
 void omp_free(void *ptr, omp_allocator_handle_t allocator) {
-  __kmpc_free(__kmp_entry_gtid(), ptr, allocator);
+  ___kmpc_free(__kmp_entry_gtid(), ptr, allocator);
 }
+/* end of OpenMP 5.1 Memory Management routines */
 
 int __kmpc_get_target_offload(void) {
   if (!__kmp_init_serial) {
     __kmp_serial_initialize();
   }
   return __kmp_target_offload;
 }
 

openmp/runtime/src/kmp_stub.cpp

 // Preview of target memory support
 omp_memspace_handle_t const llvm_omp_target_host_mem_space =
     (omp_memspace_handle_t const)100;
 omp_memspace_handle_t const llvm_omp_target_shared_mem_space =
     (omp_memspace_handle_t const)101;
 omp_memspace_handle_t const llvm_omp_target_device_mem_space =
     (omp_memspace_handle_t const)102;
 #endif /* KMP_OS_WINDOWS */
-void *omp_alloc(size_t size, const omp_allocator_handle_t allocator) {
+
+void *omp_alloc(size_t size, omp_allocator_handle_t allocator) {
+  i;
+  void *res;
+#if KMP_OS_WINDOWS
+  // Returns a pointer to the memory block, or NULL if failed.
+  // Sets errno to ENOMEM or EINVAL if memory allocation failed or parameter
+  // validation failed.
+  res = _aligned_malloc(size, 1);
+#else
+  res = malloc(size);
+#endif
+  return res;
+}
+
+void *omp_aligned_alloc(size_t a, size_t size, omp_allocator_handle_t al) {
   i;
-  return malloc(size);
+  int err;
+  void *res;
+#if KMP_OS_WINDOWS
+  res = _aligned_malloc(size, a);
+#else
+  if (err = posix_memalign(&res, a, size)) {
+    errno = err; // can be EINVAL or ENOMEM
+    res = NULL;
+  }
+#endif
+  return res;
+}
+
+void *omp_calloc(size_t nmemb, size_t size, omp_allocator_handle_t al) {
+  i;
+  void *res;
+#if KMP_OS_WINDOWS
+  res = _aligned_recalloc(NULL, nmemb, size, 1);
+#else
+  res = calloc(nmemb, size);
+#endif
+  return res;
 }
-void *omp_calloc(size_t nmemb, size_t size,
-                 const omp_allocator_handle_t allocator) {
+
+void *omp_aligned_calloc(size_t a, size_t nmemb, size_t size,
+                         omp_allocator_handle_t al) {
   i;
-  return calloc(nmemb, size);
+  int err;
+  void *res;
+#if KMP_OS_WINDOWS
+  res = _aligned_recalloc(NULL, nmemb, size, a);
+#else
+  if (err = posix_memalign(&res, a, nmemb * size)) {
+    errno = err; // can be EINVAL or ENOMEM
+    res = NULL;
   }
-void *omp_realloc(void *ptr, size_t size,
-                  const omp_allocator_handle_t allocator,
-                  const omp_allocator_handle_t free_allocator) {
+  memset(res, 0x00, size);
+#endif
+  return res;
+}
+
+void *omp_realloc(void *ptr, size_t size, omp_allocator_handle_t al,
+                  omp_allocator_handle_t free_al) {
   i;
-  return realloc(ptr, size);
+  void *res;
+#if KMP_OS_WINDOWS
+  res = _aligned_realloc(ptr, size, 1);
+#else
+  res = realloc(ptr, size);
+#endif
+  return res;
 }
-void omp_free(void *ptr, const omp_allocator_handle_t allocator) {
+
+void omp_free(void *ptr, omp_allocator_handle_t allocator) {
   i;
+#if KMP_OS_WINDOWS
+  _aligned_free(ptr);
+#else
   free(ptr);
+#endif
 }
+
 /* OpenMP 5.0 Affinity Format */
 void omp_set_affinity_format(char const *format) { i; }
 size_t omp_get_affinity_format(char *buffer, size_t size) {
   i;
   return 0;
 }
 void omp_display_affinity(char const *format) { i; }
 size_t omp_capture_affinity(char *buffer, size_t buf_size, char const *format) {
   i;
   return 0;
 }
 
 // end of file //

openmp/runtime/test/api/omp_aligned_alloc.c

+// RUN: %libomp-compile-and-run
+// UNSUPPORTED: gnu
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <omp.h>
+#define NTH 8
+#define AL0 64
+#define AL1 128
+
+int main()
+{
+  int err = 0;
+  omp_alloctrait_t at[3];
+  omp_allocator_handle_t a;
+  void *p[NTH];
+  at[0].key = omp_atk_pool_size;
+  at[0].value = 16*1024*1024;
+  at[1].key = omp_atk_fallback;
+  at[1].value = omp_atv_null_fb;
+  a = omp_init_allocator(omp_large_cap_mem_space, 2, at);
+  printf("allocator large created: %p\n", (void *)a);
+  #pragma omp parallel num_threads(8)
+  {
+    int i = omp_get_thread_num();
+    p[i] = omp_aligned_alloc(AL0, 1024 * 1024, a); // API's alignment only
+    #pragma omp barrier
+    printf("th %d, ptr %p\n", i, p[i]);
+    if ((size_t)p[i] % AL0) {
+      #pragma omp atomic
+        err++;
+      printf("Error param: th %d, ptr %p is not %d-byte aligned\n",
+             i, p[i], AL0);
+    }
+    omp_free(p[i], a);
+  }
+  omp_destroy_allocator(a);
+  at[2].key = omp_atk_alignment;
+  at[2].value = AL1;
+  a = omp_init_allocator(omp_large_cap_mem_space, 3, at);
+  printf("allocator large aligned %d created: %p\n", AL1, (void *)a);
+  if (a != omp_null_allocator)
+  #pragma omp parallel num_threads(8)
+  {
+    int i = omp_get_thread_num();
+    p[i] = omp_aligned_alloc(AL0, 1024 * 1024, a); // allocator's alignment wins
+    #pragma omp barrier
+    printf("th %d, ptr %p\n", i, p[i]);
+    if ((size_t)p[i] % AL1) {
+      #pragma omp atomic
+        err++;
+      printf("Error allocator: th %d, ptr %p is not %d-byte aligned\n",
+             i, p[i], AL1);
+    }
+    omp_free(p[i], a);
+  }
+  omp_destroy_allocator(a);
+  at[2].key = omp_atk_alignment;
+  at[2].value = AL0;
+  a = omp_init_allocator(omp_large_cap_mem_space, 3, at);
+  printf("allocator large aligned %d created: %p\n", AL0, (void *)a);
+  #pragma omp parallel num_threads(8)
+  {
+    int i = omp_get_thread_num();
+    p[i] = omp_aligned_alloc(AL1, 1024 * 1024, a); // API's alignment wins
+    #pragma omp barrier
+    printf("th %d, ptr %p\n", i, p[i]);
+    if ((size_t)p[i] % AL1) {
+      #pragma omp atomic
+        err++;
+      printf("Error param: th %d, ptr %p is not %d-byte aligned\n",
+             i, p[i], AL1);
+    }
+    omp_free(p[i], a);
+  }
+  omp_destroy_allocator(a);
+
+  if (err == 0) {
+    printf("passed\n");
+    return 0;
+  } else {
+    printf("failed\n");
+    return 1;
+  }
+}

openmp/runtime/test/api/omp_aligned_calloc.c

+// RUN: %libomp-compile-and-run
+// UNSUPPORTED: gnu
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <omp.h>
+#define NTH 8
+#define AL0 64
+#define AL1 128
+
+int main()
+{
+  int err = 0;
+  omp_alloctrait_t at[3];
+  omp_allocator_handle_t a;
+  void *p[NTH];
+  at[0].key = omp_atk_pool_size;
+  at[0].value = 16*1024*1024;
+  at[1].key = omp_atk_fallback;
+  at[1].value = omp_atv_null_fb;
+  a = omp_init_allocator(omp_large_cap_mem_space, 2, at);
+  printf("allocator large created: %p\n", (void *)a);
+  #pragma omp parallel num_threads(8)
+  {
+    int i = omp_get_thread_num();
+    p[i] = omp_aligned_calloc(AL0, 1024*128, 8, a); // API's alignment only
+    #pragma omp barrier
+    printf("th %d, ptr %p\n", i, p[i]);
+    if ((size_t)p[i] % AL0) {
+      #pragma omp atomic
+        err++;
+      printf("Error param: th %d, ptr %p is not %d-byte aligned\n",
+             i, p[i], AL0);
+    }
+    omp_free(p[i], a);
+  }
+  omp_destroy_allocator(a);
+  at[2].key = omp_atk_alignment;
+  at[2].value = AL1;
+  a = omp_init_allocator(omp_large_cap_mem_space, 3, at);
+  printf("allocator large aligned %d created: %p\n", AL1, (void *)a);
+  if (a != omp_null_allocator)
+  #pragma omp parallel num_threads(8)
+  {
+    int i = omp_get_thread_num();
+    p[i] = omp_aligned_calloc(AL0, 1024*128, 8, a); // allocator's alignment wins
+    #pragma omp barrier
+    printf("th %d, ptr %p\n", i, p[i]);
+    if ((size_t)p[i] % AL1) {
+      #pragma omp atomic
+        err++;
+      printf("Error allocator: th %d, ptr %p is not %d-byte aligned\n",
+             i, p[i], AL1);
+    }
+    omp_free(p[i], a);
+  }
+  omp_destroy_allocator(a);
+  at[2].key = omp_atk_alignment;
+  at[2].value = AL0;
+  a = omp_init_allocator(omp_large_cap_mem_space, 3, at);
+  printf("allocator large aligned %d created: %p\n", AL0, (void *)a);
+  #pragma omp parallel num_threads(8)
+  {
+    int i = omp_get_thread_num();
+    p[i] = omp_aligned_calloc(AL1, 1024*128, 8, a); // API's alignment wins
+    #pragma omp barrier
+    printf("th %d, ptr %p\n", i, p[i]);
+    if ((size_t)p[i] % AL1) {
+      #pragma omp atomic
+        err++;
+      printf("Error param: th %d, ptr %p is not %d-byte aligned\n",
+             i, p[i], AL1);
+    }
+    omp_free(p[i], a);
+  }
+  omp_destroy_allocator(a);
+
+  if (err == 0) {
+    printf("passed\n");
+    return 0;
+  } else {
+    printf("failed\n");
+    return 1;
+  }
+}

openmp/runtime/tools/generate-def.pl

 
 sub process(\%) {
 
     my ( $entries ) = @_;
 
     foreach my $entry ( keys( %$entries ) ) {
         if ( not $entries->{ $entry }->{ obsolete } ) {
             my $ordinal = $entries->{ $entry }->{ ordinal };
-            # omp_alloc, omp_calloc, omp_realloc and omp_free are C/C++ only functions, skip "1000+ordinal" for them
-            if ( $entry =~ m{\A[ok]mp_} and $entry ne "omp_alloc" and $entry ne "omp_calloc" and
-                $entry ne "omp_realloc" and $entry ne "omp_free" ) {
+            # omp_alloc and omp_free are C/C++ only functions, skip "1000+ordinal" for them
+            if ( $entry =~ m{\A[ok]mp_} and $entry ne "omp_alloc" and $entry ne "omp_free" and
+                $entry ne "omp_calloc" and $entry ne "omp_realloc" and
+                $entry ne "omp_aligned_alloc" and $entry ne "omp_aligned_calloc" ) {
                 if ( not defined( $ordinal ) ) {
                     runtime_error(
                         "Bad entry \"$entry\": ordinal number is not specified."
                     );
                 }; # if
                 if ( $ordinal ne "DATA" ) {
                     $entries->{ uc( $entry ) } = { ordinal => 1000 + $ordinal };
                 }