Add omp_get_device_num() and update several other device API functions

[tlwilmar]

Summary

Add omp_get_device_num() function for 5.0 which returns the number of the device the current thread is running on. Also, did some cleanup and updating of device API functions to make them into weak functions that should be replaced with libomptarget functions when libomptarget is present.

[jlpeyton]

LGTM

[ABataev]

This commit conflicts with libomptarget. libomptarget have the same functions and the code may be confused during execution which version of the functions should be executed. This conflict should be resolved somehow. Before that, I think, this patch must be reverted.

[tlwilmar]

In D54342#1313417, @ABataev wrote:

This commit conflicts with libomptarget. libomptarget have the same functions and the code may be confused during execution which version of the functions should be executed. This conflict should be resolved somehow. Before that, I think, this patch must be reverted.

Everything should be implemented as weak functions that libomptarget should override if it is present. Did I miss one?

[gtbercea]

After this patch, omp_get_num_devices() seems to wrongly return 0 even when devices are present.

[RaviNarayanaswamy]

Can you enable LIBOMPTARGET_DEBUG and see if this is printed from libomptarget to confirm libomptarget is loaded and its api is used
DP("Call to omp_get_num_devices returning %zd\n", Devices_size);

[tlwilmar]

In D54342#1313515, @gtbercea wrote:

After this patch, omp_get_num_devices() seems to wrongly return 0 even when devices are present.

OK, on my internal build these are showing up as weak symbols with nm. But on my LLVM build, not. Will look into the LLVM build to make sure it allows for weak attribute.

[tlwilmar]

In D54342#1313542, @tlwilmar wrote:

In D54342#1313515, @gtbercea wrote:

After this patch, omp_get_num_devices() seems to wrongly return 0 even when devices are present.

OK, on my internal build these are showing up as weak symbols with nm. But on my LLVM build, not. Will look into the LLVM build to make sure it allows for weak attribute.

I was wrong -- here are the weak symbols that are showing up in libomp:

nm runtime/exports/lin_32e.50.ompt.optional/lib/libomp.so | grep " W "
00000000000a4540 W omp_get_device_num
00000000000a5dd0 W omp_get_device_num_
00000000000a3f20 W omp_get_initial_device
00000000000a57c0 W omp_get_initial_device_
00000000000a3ee0 W omp_get_num_devices@@VERSION
00000000000a5780 W omp_get_num_devices_@@VERSION
00000000000a3f10 W omp_is_initial_device@@VERSION
00000000000a57b0 W omp_is_initial_device_@@VERSION
00000000000a7d20 W ompt_start_tool

So, it looks like weak symbols work better with static libraries. For dynamic, try linking libomptarget first.

[ABataev]

In D54342#1313628, @tlwilmar wrote:

In D54342#1313542, @tlwilmar wrote:

In D54342#1313515, @gtbercea wrote:

After this patch, omp_get_num_devices() seems to wrongly return 0 even when devices are present.

OK, on my internal build these are showing up as weak symbols with nm. But on my LLVM build, not. Will look into the LLVM build to make sure it allows for weak attribute.

I was wrong -- here are the weak symbols that are showing up in libomp:

nm runtime/exports/lin_32e.50.ompt.optional/lib/libomp.so | grep " W "
00000000000a4540 W omp_get_device_num
00000000000a5dd0 W omp_get_device_num_
00000000000a3f20 W omp_get_initial_device
00000000000a57c0 W omp_get_initial_device_
00000000000a3ee0 W omp_get_num_devices@@VERSION
00000000000a5780 W omp_get_num_devices_@@VERSION
00000000000a3f10 W omp_is_initial_device@@VERSION
00000000000a57b0 W omp_is_initial_device_@@VERSION
00000000000a7d20 W ompt_start_tool

So, it looks like weak symbols work better with static libraries. For dynamic, try linking libomptarget first.

I don't think it is a good idea to produce different results just because of the order of libomp and libomptarget. I consider this as a conflict and incompatibility. It must be fixed.

[jlpeyton]

I've reverted the patch, we will investigate a solution into this issue.

openmp/trunk/runtime/src/dllexports

%ifdef OMP_40
    #omp_curr_proc_bind                      864
     omp_get_num_teams                       865
     omp_get_team_num                        866
     omp_get_cancellation                    867
     kmp_get_cancellation_status             868
     omp_is_initial_device                   869
     omp_set_default_device                  879
     omp_get_default_device                  880
-    %ifdef stub
-        omp_get_num_devices                 881
-    %endif
+    omp_get_num_devices                     881
 %endif # OMP_40
 
 # OpenMP 45
 
 %ifdef OMP_45
     omp_init_lock_with_hint                 870
     omp_init_nest_lock_with_hint            871
     omp_get_max_task_priority               872
     omp_get_num_places                      873
     omp_get_place_num_procs                 874
     omp_get_place_proc_ids                  875
     omp_get_place_num                       876
     omp_get_partition_num_places            877
     omp_get_partition_place_nums            878
-    %ifdef stub
-        omp_get_initial_device              882
+    omp_get_initial_device                  882
+    %ifdef stub
         omp_target_alloc                    883
         omp_target_free                     884
         omp_target_is_present               885
         omp_target_memcpy                   886
         omp_target_memcpy_rect              887
         omp_target_associate_ptr            888
         omp_target_disassociate_ptr         889
     %endif
 %endif # OMP_45
 
 kmp_set_disp_num_buffers                    890
 
 %ifdef OMP_50
     omp_control_tool                        891
     omp_set_default_allocator               892
     omp_get_default_allocator               893
     omp_alloc                               894
     omp_free                                895
+    omp_get_device_num                      896
 
     OMP_NULL_ALLOCATOR                     DATA
     omp_default_mem_alloc                  DATA
     omp_large_cap_mem_alloc                DATA
     omp_const_mem_alloc                    DATA
     omp_high_bw_mem_alloc                  DATA
     omp_low_lat_mem_alloc                  DATA
     omp_cgroup_mem_alloc                   DATA

openmp/trunk/runtime/src/include/45/omp.h.var

#   undef __KAI_KMPC_CONVENTION
     typedef int     omp_int_t;
     typedef double  omp_wtime_t;
 
 #   ifdef __cplusplus
     }
 #   endif
 
 #endif /* __OMP_H */
-

openmp/trunk/runtime/src/include/45/omp_lib.h.var

 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_schedule
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_schedule
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_proc_bind
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_wtime
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_wtick
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_default_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_default_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_is_initial_device
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_initial_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_devices
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_teams
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_team_num
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_init_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_destroy_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_unset_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_test_lock

 !$omp declare target(omp_set_schedule )
 !$omp declare target(omp_get_schedule )
 !$omp declare target(omp_get_proc_bind )
 !$omp declare target(omp_get_wtime )
 !$omp declare target(omp_get_wtick )
 !$omp declare target(omp_get_default_device )
 !$omp declare target(omp_set_default_device )
 !$omp declare target(omp_is_initial_device )
+!$omp declare target(omp_get_initial_device )
 !$omp declare target(omp_get_num_devices )
 !$omp declare target(omp_get_num_teams )
 !$omp declare target(omp_get_team_num )
 !$omp declare target(omp_init_lock )
 !$omp declare target(omp_destroy_lock )
 !$omp declare target(omp_set_lock )
 !$omp declare target(omp_unset_lock )
 !$omp declare target(omp_test_lock )

 !$omp declare target(kmp_realloc )
 !$omp declare target(kmp_free )
 !$omp declare target(kmp_set_warnings_on )
 !$omp declare target(kmp_set_warnings_off )
 !$omp declare target(omp_init_lock_with_hint )
 !$omp declare target(omp_init_nest_lock_with_hint )
 !DIR$ ENDIF
 !DIR$ ENDIF
-

openmp/trunk/runtime/src/include/45/omp_lib.f.var

 !dec$ attributes alias:'OMP_GET_WTICK' :: omp_get_wtick
 !dec$ attributes alias:'OMP_GET_DEFAULT_DEVICE' :: omp_get_default_device
 !dec$ attributes alias:'OMP_SET_DEFAULT_DEVICE' :: omp_set_default_device
 !dec$ attributes alias:'OMP_GET_NUM_DEVICES' :: omp_get_num_devices
 !dec$ attributes alias:'OMP_GET_NUM_TEAMS' :: omp_get_num_teams
 !dec$ attributes alias:'OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
+!dec$ attributes alias:'OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device
 !dec$ attributes alias:'OMP_GET_MAX_TASK_PRIORITY' :: omp_get_max_task_priority
 
 !dec$ attributes alias:'omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint' :: omp_init_lock_with_hint
 !dec$ attributes alias:'omp_destroy_lock' :: omp_destroy_lock
 !dec$ attributes alias:'omp_set_lock' :: omp_set_lock
 !dec$ attributes alias:'omp_unset_lock' :: omp_unset_lock
 !dec$ attributes alias:'omp_test_lock' :: omp_test_lock

 !dec$ attributes alias:'_OMP_GET_WTICK' :: omp_get_wtick
 !dec$ attributes alias:'_OMP_GET_DEFAULT_DEVICE' :: omp_get_default_device
 !dec$ attributes alias:'_OMP_SET_DEFAULT_DEVICE' :: omp_set_default_device
 !dec$ attributes alias:'_OMP_GET_NUM_DEVICES' :: omp_get_num_devices
 !dec$ attributes alias:'_OMP_GET_NUM_TEAMS' :: omp_get_num_teams
 !dec$ attributes alias:'_OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'_OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'_OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
+!dec$ attributes alias:'_OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device
 !dec$ attributes alias:'_OMP_GET_MAX_TASK_PRIORTY' :: omp_get_max_task_priority
 
 !dec$ attributes alias:'_omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint' :: omp_init_lock_with_hint
 !dec$ attributes alias:'_omp_destroy_lock' :: omp_destroy_lock
 !dec$ attributes alias:'_omp_set_lock' :: omp_set_lock
 !dec$ attributes alias:'_omp_unset_lock' :: omp_unset_lock
 !dec$ attributes alias:'_omp_test_lock' :: omp_test_lock

 !dec$ attributes alias:'omp_get_wtick_'::omp_get_wtick
 !dec$ attributes alias:'omp_get_default_device_'::omp_get_default_device
 !dec$ attributes alias:'omp_set_default_device_'::omp_set_default_device
 !dec$ attributes alias:'omp_get_num_devices_'::omp_get_num_devices
 !dec$ attributes alias:'omp_get_num_teams_'::omp_get_num_teams
 !dec$ attributes alias:'omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'omp_is_initial_device_'::omp_is_initial_device
+!dec$ attributes alias:'omp_get_initial_device_'::omp_get_initial_device
 !dec$ attributes alias:'omp_get_max_task_priority_'::omp_get_max_task_priority
 
 !dec$ attributes alias:'omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint_'::omp_init_lock_with_hint
 !dec$ attributes alias:'omp_destroy_lock_'::omp_destroy_lock
 !dec$ attributes alias:'omp_set_lock_'::omp_set_lock
 !dec$ attributes alias:'omp_unset_lock_'::omp_unset_lock
 !dec$ attributes alias:'omp_test_lock_'::omp_test_lock

 !dec$ attributes alias:'_omp_get_active_level_'::omp_get_active_level
 !dec$ attributes alias:'_omp_get_ancestor_thread_num_'::omp_get_ancestor_thread_num
 !dec$ attributes alias:'_omp_get_team_size_'::omp_get_team_size
 !dec$ attributes alias:'_omp_set_schedule_'::omp_set_schedule
 !dec$ attributes alias:'_omp_get_schedule_'::omp_get_schedule
 !dec$ attributes alias:'_omp_get_proc_bind_' :: omp_get_proc_bind
 !dec$ attributes alias:'_omp_get_wtime_'::omp_get_wtime
 !dec$ attributes alias:'_omp_get_wtick_'::omp_get_wtick
+!dec$ attributes alias:'_omp_get_default_device_'::omp_get_default_device
+!dec$ attributes alias:'_omp_set_default_device_'::omp_set_default_device
+!dec$ attributes alias:'_omp_get_num_devices_'::omp_get_num_devices
 !dec$ attributes alias:'_omp_get_num_teams_'::omp_get_num_teams
 !dec$ attributes alias:'_omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'_omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'_omp_is_initial_device_'::omp_is_initial_device
+!dec$ attributes alias:'_omp_get_initial_device_'::omp_get_initial_device
 !dec$ attributes alias:'_omp_get_max_task_priorty_'::omp_get_max_task_priority
 
 !dec$ attributes alias:'_omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint_'::omp_init_lock_with_hint
 !dec$ attributes alias:'_omp_destroy_lock_'::omp_destroy_lock
 !dec$ attributes alias:'_omp_set_lock_'::omp_set_lock
 !dec$ attributes alias:'_omp_unset_lock_'::omp_unset_lock
 !dec$ attributes alias:'_omp_test_lock_'::omp_test_lock

 !dec$ attributes alias:'_kmp_set_warnings_on_'::kmp_set_warnings_on
 !dec$ attributes alias:'_kmp_set_warnings_off_'::kmp_set_warnings_off
 
 !dec$ attributes alias:'_kmp_get_cancellation_status_'::kmp_get_cancellation_status
 
 !dec$ endif
 
       end module omp_lib
-

openmp/trunk/runtime/src/include/50/omp.h.var

#   include <stdlib.h>
     extern void  __KAI_KMPC_CONVENTION  omp_target_free(void *, int);
     extern int   __KAI_KMPC_CONVENTION  omp_target_is_present(void *, int);
     extern int   __KAI_KMPC_CONVENTION  omp_target_memcpy(void *, void *, size_t, size_t, size_t, int, int);
     extern int   __KAI_KMPC_CONVENTION  omp_target_memcpy_rect(void *, void *, size_t, int, const size_t *,
                                             const size_t *, const size_t *, const size_t *, const size_t *, int, int);
     extern int   __KAI_KMPC_CONVENTION  omp_target_associate_ptr(void *, void *, size_t, size_t, int);
     extern int   __KAI_KMPC_CONVENTION  omp_target_disassociate_ptr(void *, int);
 
+    /* OpenMP 5.0 */
+    extern int   __KAI_KMPC_CONVENTION  omp_get_device_num (void);
+
     /* kmp API functions */
     extern int    __KAI_KMPC_CONVENTION  kmp_get_stacksize          (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_stacksize          (int);
     extern size_t __KAI_KMPC_CONVENTION  kmp_get_stacksize_s        (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_stacksize_s        (size_t);
     extern int    __KAI_KMPC_CONVENTION  kmp_get_blocktime          (void);
     extern int    __KAI_KMPC_CONVENTION  kmp_get_library            (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_blocktime          (int);
#   undef __KMP_IMP
     typedef int     omp_int_t;
     typedef double  omp_wtime_t;
 
 #   ifdef __cplusplus
     }
 #   endif
 
 #endif /* __OMP_H */
-

openmp/trunk/runtime/src/include/50/omp_lib.h.var

!       ***
           logical (kind=omp_logical_kind) omp_is_initial_device
         end function omp_is_initial_device
 
         function omp_get_initial_device() bind(c)
           import
           integer (kind=omp_integer_kind) omp_get_initial_device
         end function omp_get_initial_device
 
+        function omp_get_device_num() bind(c)
+          import
+          integer (kind=omp_integer_kind) omp_get_device_num
+        end function omp_get_device_num
+
         subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
           import
           integer (kind=omp_lock_kind) svar
         end subroutine omp_init_lock
 

 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_schedule
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_schedule
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_proc_bind
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_wtime
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_wtick
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_default_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_default_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_is_initial_device
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_initial_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_devices
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_device_num
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_teams
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_team_num
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_init_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_destroy_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_unset_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_test_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_init_nest_lock

 !$omp declare target(omp_set_schedule )
 !$omp declare target(omp_get_schedule )
 !$omp declare target(omp_get_proc_bind )
 !$omp declare target(omp_get_wtime )
 !$omp declare target(omp_get_wtick )
 !$omp declare target(omp_get_default_device )
 !$omp declare target(omp_set_default_device )
 !$omp declare target(omp_is_initial_device )
+!$omp declare target(omp_get_initial_device )
 !$omp declare target(omp_get_num_devices )
+!$omp declare target(omp_get_device_num )
 !$omp declare target(omp_get_num_teams )
 !$omp declare target(omp_get_team_num )
 !$omp declare target(omp_init_lock )
 !$omp declare target(omp_destroy_lock )
 !$omp declare target(omp_set_lock )
 !$omp declare target(omp_unset_lock )
 !$omp declare target(omp_test_lock )
 !$omp declare target(omp_init_nest_lock )

openmp/trunk/runtime/src/include/50/omp_lib.f.var

!         ***
             logical (kind=omp_logical_kind) omp_is_initial_device
           end function omp_is_initial_device
 
           function omp_get_initial_device()
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_initial_device
           end function omp_get_initial_device
 
+          function omp_get_device_num()
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_device_num
+          end function omp_get_device_num
+
           subroutine omp_init_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 

 !dec$ attributes alias:'OMP_GET_WTICK' :: omp_get_wtick
 !dec$ attributes alias:'OMP_GET_DEFAULT_DEVICE' :: omp_get_default_device
 !dec$ attributes alias:'OMP_SET_DEFAULT_DEVICE' :: omp_set_default_device
 !dec$ attributes alias:'OMP_GET_NUM_DEVICES' :: omp_get_num_devices
 !dec$ attributes alias:'OMP_GET_NUM_TEAMS' :: omp_get_num_teams
 !dec$ attributes alias:'OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
+!dec$ attributes alias:'OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device
 !dec$ attributes alias:'OMP_GET_MAX_TASK_PRIORITY' :: omp_get_max_task_priority
+!dec$ attributes alias:'OMP_GET_DEVICE_NUM' :: omp_get_device_num
 !dec$ attributes alias:'OMP_CONTROL_TOOL' :: omp_control_tool
 
 !dec$ attributes alias:'omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint' :: omp_init_lock_with_hint
 !dec$ attributes alias:'omp_destroy_lock' :: omp_destroy_lock
 !dec$ attributes alias:'omp_set_lock' :: omp_set_lock
 !dec$ attributes alias:'omp_unset_lock' :: omp_unset_lock
 !dec$ attributes alias:'omp_test_lock' :: omp_test_lock

 !dec$ attributes alias:'_OMP_GET_WTICK' :: omp_get_wtick
 !dec$ attributes alias:'_OMP_GET_DEFAULT_DEVICE' :: omp_get_default_device
 !dec$ attributes alias:'_OMP_SET_DEFAULT_DEVICE' :: omp_set_default_device
 !dec$ attributes alias:'_OMP_GET_NUM_DEVICES' :: omp_get_num_devices
 !dec$ attributes alias:'_OMP_GET_NUM_TEAMS' :: omp_get_num_teams
 !dec$ attributes alias:'_OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'_OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'_OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
+!dec$ attributes alias:'_OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device
 !dec$ attributes alias:'_OMP_GET_MAX_TASK_PRIORTY' :: omp_get_max_task_priority
+!dec$ attributes alias:'_OMP_GET_DEVICE_NUM' :: omp_get_device_num
 !dec$ attributes alias:'_OMP_CONTROL_TOOL' :: omp_control_tool
 
 !dec$ attributes alias:'_omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint' :: omp_init_lock_with_hint
 !dec$ attributes alias:'_omp_destroy_lock' :: omp_destroy_lock
 !dec$ attributes alias:'_omp_set_lock' :: omp_set_lock
 !dec$ attributes alias:'_omp_unset_lock' :: omp_unset_lock
 !dec$ attributes alias:'_omp_test_lock' :: omp_test_lock

 !dec$ attributes alias:'omp_get_wtick_'::omp_get_wtick
 !dec$ attributes alias:'omp_get_default_device_'::omp_get_default_device
 !dec$ attributes alias:'omp_set_default_device_'::omp_set_default_device
 !dec$ attributes alias:'omp_get_num_devices_'::omp_get_num_devices
 !dec$ attributes alias:'omp_get_num_teams_'::omp_get_num_teams
 !dec$ attributes alias:'omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'omp_is_initial_device_'::omp_is_initial_device
+!dec$ attributes alias:'omp_get_initial_device_'::omp_get_initial_device
 !dec$ attributes alias:'omp_get_max_task_priority_'::omp_get_max_task_priority
+!dec$ attributes alias:'omp_get_device_num_'::omp_get_device_num
 
 !dec$ attributes alias:'omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint_'::omp_init_lock_with_hint
 !dec$ attributes alias:'omp_destroy_lock_'::omp_destroy_lock
 !dec$ attributes alias:'omp_set_lock_'::omp_set_lock
 !dec$ attributes alias:'omp_unset_lock_'::omp_unset_lock
 !dec$ attributes alias:'omp_test_lock_'::omp_test_lock
 !dec$ attributes alias:'omp_init_nest_lock_'::omp_init_nest_lock

 !dec$ attributes alias:'_omp_get_active_level_'::omp_get_active_level
 !dec$ attributes alias:'_omp_get_ancestor_thread_num_'::omp_get_ancestor_thread_num
 !dec$ attributes alias:'_omp_get_team_size_'::omp_get_team_size
 !dec$ attributes alias:'_omp_set_schedule_'::omp_set_schedule
 !dec$ attributes alias:'_omp_get_schedule_'::omp_get_schedule
 !dec$ attributes alias:'_omp_get_proc_bind_' :: omp_get_proc_bind
 !dec$ attributes alias:'_omp_get_wtime_'::omp_get_wtime
 !dec$ attributes alias:'_omp_get_wtick_'::omp_get_wtick
+!dec$ attributes alias:'_omp_get_default_device_'::omp_get_default_device
+!dec$ attributes alias:'_omp_set_default_device_'::omp_set_default_device
+!dec$ attributes alias:'_omp_get_num_devices_'::omp_get_num_devices
 !dec$ attributes alias:'_omp_get_num_teams_'::omp_get_num_teams
 !dec$ attributes alias:'_omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'_omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'_omp_is_initial_device_'::omp_is_initial_device
+!dec$ attributes alias:'_omp_get_initial_device_'::omp_get_initial_device
 !dec$ attributes alias:'_omp_get_max_task_priorty_'::omp_get_max_task_priority
-
+!dec$ attributes alias:'_omp_get_device_num_'::omp_get_device_num
 !dec$ attributes alias:'_omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint_'::omp_init_lock_with_hint
 !dec$ attributes alias:'_omp_destroy_lock_'::omp_destroy_lock
 !dec$ attributes alias:'_omp_set_lock_'::omp_set_lock
 !dec$ attributes alias:'_omp_unset_lock_'::omp_unset_lock
 !dec$ attributes alias:'_omp_test_lock_'::omp_test_lock
 !dec$ attributes alias:'_omp_init_nest_lock_'::omp_init_nest_lock
 !dec$ attributes alias:'_omp_init_nest_lock_with_hint_'::omp_init_nest_lock_with_hint

 !dec$ attributes alias:'_kmp_set_warnings_on_'::kmp_set_warnings_on
 !dec$ attributes alias:'_kmp_set_warnings_off_'::kmp_set_warnings_off
 
 !dec$ attributes alias:'_kmp_get_cancellation_status_'::kmp_get_cancellation_status
 
 !dec$ endif
 
       end module omp_lib
-

openmp/trunk/runtime/src/include/50/omp_lib.f90.var

!         ***
             logical (kind=omp_logical_kind) omp_is_initial_device
           end function omp_is_initial_device
 
           function omp_get_initial_device() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_initial_device
           end function omp_get_initial_device
 
+          function omp_get_device_num() bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_device_num
+          end function omp_get_device_num
+
           subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 

openmp/trunk/runtime/src/kmp.h

enum kmp_target_offload_kind {
   tgt_mandatory = 2
 };
 typedef enum kmp_target_offload_kind kmp_target_offload_kind_t;
 // Set via OMP_TARGET_OFFLOAD if specified, defaults to tgt_default otherwise
 extern kmp_target_offload_kind_t __kmp_target_offload;
 extern int __kmpc_get_target_offload();
 #endif
 
+#if OMP_40_ENABLED
+// Constants used in libomptarget
+#define KMP_DEVICE_DEFAULT -1 // This is libomptarget's default device.
+#define KMP_HOST_DEVICE -10 // This is what it is in libomptarget, go figure.
+#define KMP_DEVICE_ALL -11 // This is libomptarget's "all devices".
+#endif // OMP_40_ENABLED
+
 #ifdef __cplusplus
 }
 #endif
 
 #endif /* KMP_H */

openmp/trunk/runtime/src/kmp_ftn_entry.h

 #if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
 // Nothing.
 #else
   __kmp_entry_thread()->th.th_current_task->td_icvs.default_device =
       KMP_DEREF arg;
 #endif
 }
 
-#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
-
-int FTN_STDCALL FTN_GET_NUM_DEVICES(void) { return 0; }
-
-#endif // KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
-
-#if !KMP_OS_LINUX
-
-int FTN_STDCALL KMP_EXPAND_NAME(FTN_IS_INITIAL_DEVICE)(void) { return 1; }
-
+// Get number of NON-HOST devices.
+// libomptarget, if loaded, provides this function in api.cpp.
+int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NUM_DEVICES)(void) KMP_WEAK_ATTRIBUTE;
+int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NUM_DEVICES)(void) {
+#if KMP_MIC || KMP_OS_DARWIN || KMP_OS_WINDOWS || defined(KMP_STUB)
+  return 0;
 #else
-
-// This internal function is used when the entry from the offload library
-// is not found.
-int _Offload_get_device_number(void) KMP_WEAK_ATTRIBUTE;
-
-int FTN_STDCALL KMP_EXPAND_NAME(FTN_IS_INITIAL_DEVICE)(void) {
-  if (_Offload_get_device_number) {
-    return _Offload_get_device_number() == -1;
-  } else {
-    return 1;
+  int (*fptr)();
+  if ((*(void **)(&fptr) = dlsym(RTLD_DEFAULT, "_Offload_number_of_devices"))) {
+    return (*fptr)();
+  } else { // liboffload & libomptarget don't exist
+    return 0;
   }
+#endif // KMP_MIC || KMP_OS_DARWIN || KMP_OS_WINDOWS || defined(KMP_STUB)
 }
 
-#endif // ! KMP_OS_LINUX
+// This function always returns true when called on host device.
+// Compilier/libomptarget should handle when it is called inside target region.
+int FTN_STDCALL KMP_EXPAND_NAME(FTN_IS_INITIAL_DEVICE)(void) KMP_WEAK_ATTRIBUTE;
+int FTN_STDCALL KMP_EXPAND_NAME(FTN_IS_INITIAL_DEVICE)(void) {
+  return 1; // This is the host
+}
 
 #endif // OMP_40_ENABLED
 
-#if OMP_45_ENABLED && defined(KMP_STUB)
-// OpenMP 4.5 entries for stubs library
+#if OMP_45_ENABLED
+// OpenMP 4.5 entries
 
-int FTN_STDCALL FTN_GET_INITIAL_DEVICE(void) { return -1; }
+// libomptarget, if loaded, provides this function
+int FTN_STDCALL FTN_GET_INITIAL_DEVICE(void) KMP_WEAK_ATTRIBUTE;
+int FTN_STDCALL FTN_GET_INITIAL_DEVICE(void) { return KMP_HOST_DEVICE; }
 
+#if defined(KMP_STUB)
+// Entries for stubs library
 // As all *target* functions are C-only parameters always passed by value
 void *FTN_STDCALL FTN_TARGET_ALLOC(size_t size, int device_num) { return 0; }
 
 void FTN_STDCALL FTN_TARGET_FREE(void *device_ptr, int device_num) {}
 
 int FTN_STDCALL FTN_TARGET_IS_PRESENT(void *ptr, int device_num) { return 0; }
 
 int FTN_STDCALL FTN_TARGET_MEMCPY(void *dst, void *src, size_t length,
int FTN_STDCALL FTN_TARGET_ASSOCIATE_PTR(void *host_ptr, void *device_ptr,
                                          size_t size, size_t device_offset,
                                          int device_num) {
   return -1;
 }
 
 int FTN_STDCALL FTN_TARGET_DISASSOCIATE_PTR(void *host_ptr, int device_num) {
   return -1;
 }
-#endif // OMP_45_ENABLED && defined(KMP_STUB)
+#endif // defined(KMP_STUB)
+#endif // OMP_45_ENABLED
 
 #ifdef KMP_STUB
 typedef enum { UNINIT = -1, UNLOCKED, LOCKED } kmp_stub_lock_t;
 #endif /* KMP_STUB */
 
 #if KMP_USE_DYNAMIC_LOCK
 void FTN_STDCALL FTN_INIT_LOCK_WITH_HINT(void **user_lock,
                                          uintptr_t KMP_DEREF hint) {
#else
   if (!__kmp_init_serial) {
     __kmp_serial_initialize();
   }
   return __kmp_max_task_priority;
 #endif
 }
 #endif
 
+#if OMP_50_ENABLED
+// This function will be defined in libomptarget. When libomptarget is not
+// loaded, we assume we are on the host and return KMP_HOST_DEVICE.
+// Compiler/libomptarget will handle this if called inside target.
+int FTN_STDCALL FTN_GET_DEVICE_NUM(void) KMP_WEAK_ATTRIBUTE;
+int FTN_STDCALL FTN_GET_DEVICE_NUM(void) { return KMP_HOST_DEVICE; }
+#endif // OMP_50_ENABLED
+
 // GCC compatibility (versioned symbols)
 #ifdef KMP_USE_VERSION_SYMBOLS
 
 /* These following sections create versioned symbols for the
    omp_* routines. The KMP_VERSION_SYMBOL macro expands the API name and
    then maps it to a versioned symbol.
    libgomp ``versions'' its symbols (OMP_1.0, OMP_2.0, OMP_3.0, ...) while also
    retaining the default version which libomp uses: VERSION (defined in

 // OMP_4.0 versioned symbols
 KMP_VERSION_SYMBOL(FTN_GET_PROC_BIND, 40, "OMP_4.0");
 KMP_VERSION_SYMBOL(FTN_GET_NUM_TEAMS, 40, "OMP_4.0");
 KMP_VERSION_SYMBOL(FTN_GET_TEAM_NUM, 40, "OMP_4.0");
 KMP_VERSION_SYMBOL(FTN_GET_CANCELLATION, 40, "OMP_4.0");
 KMP_VERSION_SYMBOL(FTN_GET_DEFAULT_DEVICE, 40, "OMP_4.0");
 KMP_VERSION_SYMBOL(FTN_SET_DEFAULT_DEVICE, 40, "OMP_4.0");
 KMP_VERSION_SYMBOL(FTN_IS_INITIAL_DEVICE, 40, "OMP_4.0");
+KMP_VERSION_SYMBOL(FTN_GET_NUM_DEVICES, 40, "OMP_4.0");
 #endif /* OMP_40_ENABLED */
 
 #if OMP_45_ENABLED
 // OMP_4.5 versioned symbols
 KMP_VERSION_SYMBOL(FTN_GET_MAX_TASK_PRIORITY, 45, "OMP_4.5");
 KMP_VERSION_SYMBOL(FTN_GET_NUM_PLACES, 45, "OMP_4.5");
 KMP_VERSION_SYMBOL(FTN_GET_PLACE_NUM_PROCS, 45, "OMP_4.5");
 KMP_VERSION_SYMBOL(FTN_GET_PLACE_PROC_IDS, 45, "OMP_4.5");
 KMP_VERSION_SYMBOL(FTN_GET_PLACE_NUM, 45, "OMP_4.5");
 KMP_VERSION_SYMBOL(FTN_GET_PARTITION_NUM_PLACES, 45, "OMP_4.5");
 KMP_VERSION_SYMBOL(FTN_GET_PARTITION_PLACE_NUMS, 45, "OMP_4.5");
+// KMP_VERSION_SYMBOL(FTN_GET_INITIAL_DEVICE, 45, "OMP_4.5");
 #endif
 
 #if OMP_50_ENABLED
 // OMP_5.0 versioned symbols
+// KMP_VERSION_SYMBOL(FTN_GET_DEVICE_NUM, 50, "OMP_5.0");
 #endif
 
 #endif // KMP_USE_VERSION_SYMBOLS
 
 #ifdef __cplusplus
 } // extern "C"
 #endif // __cplusplus
 
 // end of file //

openmp/trunk/runtime/src/kmp_ftn_os.h

 
 #define FTN_SET_WARNINGS_ON kmp_set_warnings_on
 #define FTN_SET_WARNINGS_OFF kmp_set_warnings_off
 
 #define FTN_GET_WTIME omp_get_wtime
 #define FTN_GET_WTICK omp_get_wtick
 
 #if OMP_40_ENABLED
-#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
 #define FTN_GET_NUM_DEVICES omp_get_num_devices
-#endif
 #define FTN_GET_DEFAULT_DEVICE omp_get_default_device
 #define FTN_SET_DEFAULT_DEVICE omp_set_default_device
 #define FTN_IS_INITIAL_DEVICE omp_is_initial_device
 #endif
 
 #if OMP_40_ENABLED
 #define FTN_GET_CANCELLATION omp_get_cancellation
 #define FTN_GET_CANCELLATION_STATUS kmp_get_cancellation_status
 #endif
 
 #if OMP_45_ENABLED
 #define FTN_GET_MAX_TASK_PRIORITY omp_get_max_task_priority
 #define FTN_GET_NUM_PLACES omp_get_num_places
 #define FTN_GET_PLACE_NUM_PROCS omp_get_place_num_procs
 #define FTN_GET_PLACE_PROC_IDS omp_get_place_proc_ids
 #define FTN_GET_PLACE_NUM omp_get_place_num
 #define FTN_GET_PARTITION_NUM_PLACES omp_get_partition_num_places
 #define FTN_GET_PARTITION_PLACE_NUMS omp_get_partition_place_nums
-#ifdef KMP_STUB
 #define FTN_GET_INITIAL_DEVICE omp_get_initial_device
+#ifdef KMP_STUB
 #define FTN_TARGET_ALLOC omp_target_alloc
 #define FTN_TARGET_FREE omp_target_free
 #define FTN_TARGET_IS_PRESENT omp_target_is_present
 #define FTN_TARGET_MEMCPY omp_target_memcpy
 #define FTN_TARGET_MEMCPY_RECT omp_target_memcpy_rect
 #define FTN_TARGET_ASSOCIATE_PTR omp_target_associate_ptr
 #define FTN_TARGET_DISASSOCIATE_PTR omp_target_disassociate_ptr
 #endif
 #endif
 
 #if OMP_50_ENABLED
 #define FTN_CONTROL_TOOL omp_control_tool
 #define FTN_SET_DEFAULT_ALLOCATOR omp_set_default_allocator
 #define FTN_GET_DEFAULT_ALLOCATOR omp_get_default_allocator
 #define FTN_ALLOC omp_alloc
 #define FTN_FREE omp_free
+#define FTN_GET_DEVICE_NUM omp_get_device_num
 #endif
 
 #endif /* KMP_FTN_PLAIN */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_APPEND
 

 
 #define FTN_SET_WARNINGS_ON kmp_set_warnings_on_
 #define FTN_SET_WARNINGS_OFF kmp_set_warnings_off_
 
 #define FTN_GET_WTIME omp_get_wtime_
 #define FTN_GET_WTICK omp_get_wtick_
 
 #if OMP_40_ENABLED
-#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
 #define FTN_GET_NUM_DEVICES omp_get_num_devices_
-#endif
 #define FTN_GET_DEFAULT_DEVICE omp_get_default_device_
 #define FTN_SET_DEFAULT_DEVICE omp_set_default_device_
 #define FTN_IS_INITIAL_DEVICE omp_is_initial_device_
 #endif
 
 #if OMP_40_ENABLED
 #define FTN_GET_CANCELLATION omp_get_cancellation_
 #define FTN_GET_CANCELLATION_STATUS kmp_get_cancellation_status_
 #endif
 
 #if OMP_45_ENABLED
 #define FTN_GET_MAX_TASK_PRIORITY omp_get_max_task_priority_
 #define FTN_GET_NUM_PLACES omp_get_num_places_
 #define FTN_GET_PLACE_NUM_PROCS omp_get_place_num_procs_
 #define FTN_GET_PLACE_PROC_IDS omp_get_place_proc_ids_
 #define FTN_GET_PLACE_NUM omp_get_place_num_
 #define FTN_GET_PARTITION_NUM_PLACES omp_get_partition_num_places_
 #define FTN_GET_PARTITION_PLACE_NUMS omp_get_partition_place_nums_
-#ifdef KMP_STUB
 #define FTN_GET_INITIAL_DEVICE omp_get_initial_device_
+#ifdef KMP_STUB
 #define FTN_TARGET_ALLOC omp_target_alloc_
 #define FTN_TARGET_FREE omp_target_free_
 #define FTN_TARGET_IS_PRESENT omp_target_is_present_
 #define FTN_TARGET_MEMCPY omp_target_memcpy_
 #define FTN_TARGET_MEMCPY_RECT omp_target_memcpy_rect_
 #define FTN_TARGET_ASSOCIATE_PTR omp_target_associate_ptr_
 #define FTN_TARGET_DISASSOCIATE_PTR omp_target_disassociate_ptr_
 #endif
 #endif
 
 #if OMP_50_ENABLED
 #define FTN_CONTROL_TOOL omp_control_tool_
 #define FTN_SET_DEFAULT_ALLOCATOR omp_set_default_allocator_
 #define FTN_GET_DEFAULT_ALLOCATOR omp_get_default_allocator_
 #define FTN_ALLOC omp_alloc_
 #define FTN_FREE omp_free_
+#define FTN_GET_DEVICE_NUM omp_get_device_num_
 #endif
 
 #endif /* KMP_FTN_APPEND */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_UPPER
 

 
 #define FTN_SET_WARNINGS_ON KMP_SET_WARNINGS_ON
 #define FTN_SET_WARNINGS_OFF KMP_SET_WARNINGS_OFF
 
 #define FTN_GET_WTIME OMP_GET_WTIME
 #define FTN_GET_WTICK OMP_GET_WTICK
 
 #if OMP_40_ENABLED
-#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
 #define FTN_GET_NUM_DEVICES OMP_GET_NUM_DEVICES
-#endif
 #define FTN_GET_DEFAULT_DEVICE OMP_GET_DEFAULT_DEVICE
 #define FTN_SET_DEFAULT_DEVICE OMP_SET_DEFAULT_DEVICE
 #define FTN_IS_INITIAL_DEVICE OMP_IS_INITIAL_DEVICE
 #endif
 
 #if OMP_40_ENABLED
 #define FTN_GET_CANCELLATION OMP_GET_CANCELLATION
 #define FTN_GET_CANCELLATION_STATUS KMP_GET_CANCELLATION_STATUS
 #endif
 
 #if OMP_45_ENABLED
 #define FTN_GET_MAX_TASK_PRIORITY OMP_GET_MAX_TASK_PRIORITY
 #define FTN_GET_NUM_PLACES OMP_GET_NUM_PLACES
 #define FTN_GET_PLACE_NUM_PROCS OMP_GET_PLACE_NUM_PROCS
 #define FTN_GET_PLACE_PROC_IDS OMP_GET_PLACE_PROC_IDS
 #define FTN_GET_PLACE_NUM OMP_GET_PLACE_NUM
 #define FTN_GET_PARTITION_NUM_PLACES OMP_GET_PARTITION_NUM_PLACES
 #define FTN_GET_PARTITION_PLACE_NUMS OMP_GET_PARTITION_PLACE_NUMS
-#ifdef KMP_STUB
 #define FTN_GET_INITIAL_DEVICE OMP_GET_INITIAL_DEVICE
+#ifdef KMP_STUB
 #define FTN_TARGET_ALLOC OMP_TARGET_ALLOC
 #define FTN_TARGET_FREE OMP_TARGET_FREE
 #define FTN_TARGET_IS_PRESENT OMP_TARGET_IS_PRESENT
 #define FTN_TARGET_MEMCPY OMP_TARGET_MEMCPY
 #define FTN_TARGET_MEMCPY_RECT OMP_TARGET_MEMCPY_RECT
 #define FTN_TARGET_ASSOCIATE_PTR OMP_TARGET_ASSOCIATE_PTR
 #define FTN_TARGET_DISASSOCIATE_PTR OMP_TARGET_DISASSOCIATE_PTR
 #endif
 #endif
 
 #if OMP_50_ENABLED
 #define FTN_CONTROL_TOOL OMP_CONTROL_TOOL
 #define FTN_SET_DEFAULT_ALLOCATOR OMP_SET_DEFAULT_ALLOCATOR
 #define FTN_GET_DEFAULT_ALLOCATOR OMP_GET_DEFAULT_ALLOCATOR
 #define FTN_ALLOC OMP_ALLOC
 #define FTN_FREE OMP_FREE
+#define FTN_GET_DEVICE_NUM OMP_GET_DEVICE_NUM
 #endif
 
 #endif /* KMP_FTN_UPPER */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_UAPPEND
 

 
 #define FTN_SET_WARNINGS_ON KMP_SET_WARNINGS_ON_
 #define FTN_SET_WARNINGS_OFF KMP_SET_WARNINGS_OFF_
 
 #define FTN_GET_WTIME OMP_GET_WTIME_
 #define FTN_GET_WTICK OMP_GET_WTICK_
 
 #if OMP_40_ENABLED
-#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
 #define FTN_GET_NUM_DEVICES OMP_GET_NUM_DEVICES_
-#endif
 #define FTN_GET_DEFAULT_DEVICE OMP_GET_DEFAULT_DEVICE_
 #define FTN_SET_DEFAULT_DEVICE OMP_SET_DEFAULT_DEVICE_
 #define FTN_IS_INITIAL_DEVICE OMP_IS_INITIAL_DEVICE_
 #endif
 
 #if OMP_40_ENABLED
 #define FTN_GET_CANCELLATION OMP_GET_CANCELLATION_
 #define FTN_GET_CANCELLATION_STATUS KMP_GET_CANCELLATION_STATUS_
 #endif
 
 #if OMP_45_ENABLED
 #define FTN_GET_MAX_TASK_PRIORITY OMP_GET_MAX_TASK_PRIORITY_
 #define FTN_GET_NUM_PLACES OMP_GET_NUM_PLACES_
 #define FTN_GET_PLACE_NUM_PROCS OMP_GET_PLACE_NUM_PROCS_
 #define FTN_GET_PLACE_PROC_IDS OMP_GET_PLACE_PROC_IDS_
 #define FTN_GET_PLACE_NUM OMP_GET_PLACE_NUM_
 #define FTN_GET_PARTITION_NUM_PLACES OMP_GET_PARTITION_NUM_PLACES_
 #define FTN_GET_PARTITION_PLACE_NUMS OMP_GET_PARTITION_PLACE_NUMS_
-#ifdef KMP_STUB
 #define FTN_GET_INITIAL_DEVICE OMP_GET_INITIAL_DEVICE_
+#ifdef KMP_STUB
 #define FTN_TARGET_ALLOC OMP_TARGET_ALLOC_
 #define FTN_TARGET_FREE OMP_TARGET_FREE_
 #define FTN_TARGET_IS_PRESENT OMP_TARGET_IS_PRESENT_
 #define FTN_TARGET_MEMCPY OMP_TARGET_MEMCPY_
 #define FTN_TARGET_MEMCPY_RECT OMP_TARGET_MEMCPY_RECT_
 #define FTN_TARGET_ASSOCIATE_PTR OMP_TARGET_ASSOCIATE_PTR_
 #define FTN_TARGET_DISASSOCIATE_PTR OMP_TARGET_DISASSOCIATE_PTR_
 #endif
 #endif
 
 #if OMP_50_ENABLED
 #define FTN_CONTROL_TOOL OMP_CONTROL_TOOL_
 #define FTN_SET_DEFAULT_ALLOCATOR OMP_SET_DEFAULT_ALLOCATOR_
 #define FTN_GET_DEFAULT_ALLOCATOR OMP_GET_DEFAULT_ALLOCATOR_
 #define FTN_ALLOC OMP_ALLOC_
 #define FTN_FREE OMP_FREE_
+#define FTN_GET_DEVICE_NUM OMP_GET_DEVICE_NUM_
 #endif
 
 #endif /* KMP_FTN_UAPPEND */
 
 /* -------------------------- GOMP API NAMES ------------------------ */
 // All GOMP_1.0 symbols
 #define KMP_API_NAME_GOMP_ATOMIC_END GOMP_atomic_end
 #define KMP_API_NAME_GOMP_ATOMIC_START GOMP_atomic_start