Add omp_pause_resource* API

[tlwilmar]

Summary

Add omp_pause_resource and omp_pause_resource_all API and enum, plus stub for internal implementation. Implemented callable helper function to do local pause, and added basic functionality for hard and soft pause.

[AndreyChurbanov]

runtime/src/include/50/omp_lib.f.var

292

Parameters should be re-named as (kind, device_num). Because in Fortran user can provide named arguments, so we should use same names as in the specification.

299

Parameter "level" should be renamed to "kind" to correspond to the specification.

runtime/src/include/50/omp_lib.f90.var

308

Parameters should be re-named as (kind, device_num). Because in Fortran user can provide named arguments, so we should use same names as in the specification.

315

Parameter "level" should be renamed to "kind" to correspond to the specification.

runtime/src/include/50/omp_lib.h.var

340

Parameters should be re-named as (kind, device_num). Because in Fortran user can provide named arguments, so we should use same names as in the specification.

347

Parameter "level" should be renamed to "kind" to correspond to the specification.

[tlwilmar]

Renamed parameters to match 5.0 spec. Added test.

[AndreyChurbanov]

LGTM

[tlwilmar]

Sorry Andrey -- I missed the Windows fix. Added now.

[pawosm01]

This commit causes Flang unit tests to fail.

openmp/trunk/runtime/src/include/50/omp_lib.h.var

143 ↗

(On Diff #182117)

These three lines are too long causing OpenMP test cases from Flang fortran compiler to fail. See how this problem is addressed in the lines above and below.

[pawosm01]

Please correct.

[AndreyChurbanov]

openmp/trunk/runtime/src/include/50/omp_lib.h.var

143 ↗

(On Diff #182117)

Paul, sorry about that. I've missed on review that these lines exceed 72 chars.
Now fixed by commit rL351745.

[pawosm01]

It's ok, works for me now.

[kkwli0]

openmp/trunk/runtime/src/include/50/omp_lib.h.var

44 ↗

(On Diff #182117)

Should use the same syntax (declaration + parameter statement) as the others in this file.

143 ↗

(On Diff #182117)

declaration + parameter statement

[AndreyChurbanov]

openmp/trunk/runtime/src/include/50/omp_lib.h.var

44 ↗

(On Diff #182117)

Sure, fixed this as well.

Thanks, Kelvin.

runtime/src/dllexports

 # OpenMP 5.0 entry points
 %ifndef stub
     %ifdef OMP_50
         __kmpc_task_reduction_init          268
         __kmpc_task_reduction_get_th_data   269
 # USED FOR 4.5 __kmpc_critical_with_hint    270
         __kmpc_get_target_offload           271
   __kmpc_omp_reg_task_with_affinity   272
+  __kmpc_pause_resource               273
     %endif
 %endif
 
 # User API entry points that have both lower- and upper- case versions for Fortran.
 # Number for lowercase version is indicated.  Number for uppercase is obtained by adding 1000.
 # User API entry points are entry points that start with 'kmp_' or 'omp_'.
 
 omp_destroy_lock                            700

 %ifdef OMP_50
     omp_control_tool                        891
     omp_set_default_allocator               892
     omp_get_default_allocator               893
     omp_alloc                               894
     omp_free                                895
     omp_get_device_num                      896
 
+    omp_pause_resource                      756
+    omp_pause_resource_all                  757
+
     OMP_NULL_ALLOCATOR                     DATA
     omp_default_mem_alloc                  DATA
     omp_large_cap_mem_alloc                DATA
     omp_const_mem_alloc                    DATA
     omp_high_bw_mem_alloc                  DATA
     omp_low_lat_mem_alloc                  DATA
     omp_cgroup_mem_alloc                   DATA
     omp_pteam_mem_alloc                    DATA

runtime/src/include/50/omp.h.var

 #ifdef __cplusplus
     extern void *__KAI_KMPC_CONVENTION omp_alloc(size_t size, const omp_allocator_t *allocator = OMP_NULL_ALLOCATOR);
     extern void __KAI_KMPC_CONVENTION omp_free(void * ptr, const omp_allocator_t *allocator = OMP_NULL_ALLOCATOR);
 #else
     extern void *__KAI_KMPC_CONVENTION omp_alloc(size_t size, const omp_allocator_t *allocator);
     extern void __KAI_KMPC_CONVENTION omp_free(void *ptr, const omp_allocator_t *allocator);
 #endif
 
+    /* OpenMP 5.0 Pause Resources */
+    typedef enum omp_pause_resource_t {
+      omp_pause_resume = 0,
+      omp_pause_soft = 1,
+      omp_pause_hard = 2
+    } omp_pause_resource_t;
+    extern int __KAI_KMPC_CONVENTION omp_pause_resource(omp_pause_resource_t, int);
+    extern int __KAI_KMPC_CONVENTION omp_pause_resource_all(omp_pause_resource_t);
+
 #   undef __KAI_KMPC_CONVENTION
 #   undef __KMP_IMP
 
     /* Warning:
        The following typedefs are not standard, deprecated and will be removed in a future release.
     */
     typedef int     omp_int_t;
     typedef double  omp_wtime_t;
 
 #   ifdef __cplusplus
     }
 #   endif
 
 #endif /* __OMP_H */

runtime/src/include/50/omp_lib.h.var

!
       integer omp_lock_hint_kind
       parameter(omp_lock_hint_kind=omp_sync_hint_kind)
       integer omp_control_tool_kind
       parameter(omp_control_tool_kind=omp_integer_kind)
       integer omp_control_tool_result_kind
       parameter(omp_control_tool_result_kind=omp_integer_kind)
       integer omp_allocator_kind
       parameter(omp_allocator_kind=int_ptr_kind())
+      integer, parameter :: omp_pause_resource_kind = omp_integer_kind
 
       integer(kind=omp_integer_kind)openmp_version
       parameter(openmp_version=@LIBOMP_OMP_YEAR_MONTH@)
       integer(kind=omp_integer_kind)kmp_version_major
       parameter(kmp_version_major=@LIBOMP_VERSION_MAJOR@)
       integer(kind=omp_integer_kind)kmp_version_minor
       parameter(kmp_version_minor=@LIBOMP_VERSION_MINOR@)
       integer(kind=omp_integer_kind)kmp_version_build
!
       parameter(omp_low_lat_mem_alloc=5)
       integer(kind=omp_allocator_kind)omp_cgroup_mem_alloc
       parameter(omp_cgroup_mem_alloc=6)
       integer(kind=omp_allocator_kind)omp_pteam_mem_alloc
       parameter(omp_pteam_mem_alloc=7)
       integer(kind=omp_allocator_kind)omp_thread_mem_alloc
       parameter(omp_thread_mem_alloc=8)
 
+      integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0
+      integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1
+      integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2
+
       interface
 
 !       ***
 !       *** omp_* entry points
 !       ***
 
         subroutine omp_set_num_threads(num_threads) bind(c)
           import
!       ***
           integer (kind=omp_integer_kind) omp_get_initial_device
         end function omp_get_initial_device
 
         function omp_get_device_num() bind(c)
           import
           integer (kind=omp_integer_kind) omp_get_device_num
         end function omp_get_device_num
 
+        function omp_pause_resource(level, device) bind(c)
+          import
+          integer (kind=omp_pause_resource_kind), value :: level
+          integer (kind=omp_integer_kind), value :: device
+          integer (kind=omp_integer_kind) omp_pause_resource
+        end function omp_pause_resource
+
+        function omp_pause_resource_all(level) bind(c)
+          import
+          integer (kind=omp_pause_resource_kind), value :: level
+          integer (kind=omp_integer_kind) omp_pause_resource_all
+        end function omp_pause_resource_all
+
         subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
           import
           integer (kind=omp_lock_kind) svar
         end subroutine omp_init_lock
 

 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_wtime
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_wtick
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_default_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_default_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_is_initial_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_initial_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_devices
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_device_num
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_pause_resource
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_pause_resource_all
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_teams
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_team_num
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_init_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_destroy_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_unset_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_test_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_init_nest_lock

 !$omp declare target(omp_get_wtime )
 !$omp declare target(omp_get_wtick )
 !$omp declare target(omp_get_default_device )
 !$omp declare target(omp_set_default_device )
 !$omp declare target(omp_is_initial_device )
 !$omp declare target(omp_get_initial_device )
 !$omp declare target(omp_get_num_devices )
 !$omp declare target(omp_get_device_num )
+!$omp declare target(omp_pause_resource )
+!$omp declare target(omp_pause_resource_all )
 !$omp declare target(omp_get_num_teams )
 !$omp declare target(omp_get_team_num )
 !$omp declare target(omp_init_lock )
 !$omp declare target(omp_destroy_lock )
 !$omp declare target(omp_set_lock )
 !$omp declare target(omp_unset_lock )
 !$omp declare target(omp_test_lock )
 !$omp declare target(omp_init_nest_lock )

runtime/src/include/50/omp_lib.f.var

!dec$ fixedformlinesize:132
         integer, parameter :: kmp_size_t_kind        = int_ptr_kind()
         integer, parameter :: kmp_affinity_mask_kind = int_ptr_kind()
         integer, parameter :: kmp_cancel_kind        = omp_integer_kind
         integer, parameter :: omp_lock_hint_kind     = omp_integer_kind
         integer, parameter :: omp_control_tool_kind  = omp_integer_kind
         integer, parameter :: omp_control_tool_result_kind = omp_integer_kind
         integer, parameter :: omp_allocator_kind = int_ptr_kind()
 
+        integer, parameter :: omp_pause_resource_kind = omp_integer_kind
+
       end module omp_lib_kinds
 
       module omp_lib
 
         use omp_lib_kinds
 
         integer (kind=omp_integer_kind), parameter :: kmp_version_major = @LIBOMP_VERSION_MAJOR@
         integer (kind=omp_integer_kind), parameter :: kmp_version_minor = @LIBOMP_VERSION_MINOR@
!dec$ fixedformlinesize:132
         integer (kind=omp_allocator_kind), parameter :: omp_large_cap_mem_alloc = 2
         integer (kind=omp_allocator_kind), parameter :: omp_const_mem_alloc = 3
         integer (kind=omp_allocator_kind), parameter :: omp_high_bw_mem_alloc = 4
         integer (kind=omp_allocator_kind), parameter :: omp_low_lat_mem_alloc = 5
         integer (kind=omp_allocator_kind), parameter :: omp_cgroup_mem_alloc = 6
         integer (kind=omp_allocator_kind), parameter :: omp_pteam_mem_alloc = 7
         integer (kind=omp_allocator_kind), parameter :: omp_thread_mem_alloc = 8
 
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2
+
         interface
 
 !         ***
 !         *** omp_* entry points
 !         ***
 
           subroutine omp_set_num_threads(num_threads)
             use omp_lib_kinds
!         ***
             integer (kind=omp_integer_kind) omp_get_initial_device
           end function omp_get_initial_device
 
           function omp_get_device_num()
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
+          function omp_pause_resource(level, device)
+            use omp_lib_kinds
+            integer (kind=omp_pause_resource_kind) level
+            integer (kind=omp_integer_kind) device
+            integer (kind=omp_integer_kind) omp_pause_resource
+          end function omp_pause_resource
+
+          function omp_pause_resource_all(level)
+            use omp_lib_kinds
+            integer (kind=omp_pause_resource_kind) level
+            integer (kind=omp_integer_kind) omp_pause_resource_all
+          end function omp_pause_resource_all
+
           subroutine omp_init_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 

 !dec$ attributes alias:'OMP_GET_NUM_DEVICES' :: omp_get_num_devices
 !dec$ attributes alias:'OMP_GET_NUM_TEAMS' :: omp_get_num_teams
 !dec$ attributes alias:'OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
 !dec$ attributes alias:'OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device
 !dec$ attributes alias:'OMP_GET_MAX_TASK_PRIORITY' :: omp_get_max_task_priority
 !dec$ attributes alias:'OMP_GET_DEVICE_NUM' :: omp_get_device_num
+!dec$ attributes alias:'OMP_PAUSE_RESOURCE' :: omp_pause_resource
+!dec$ attributes alias:'OMP_PAUSE_RESOURCE_ALL' :: omp_pause_resource_all
+
 !dec$ attributes alias:'OMP_CONTROL_TOOL' :: omp_control_tool
 
 !dec$ attributes alias:'omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint' :: omp_init_lock_with_hint
 !dec$ attributes alias:'omp_destroy_lock' :: omp_destroy_lock
 !dec$ attributes alias:'omp_set_lock' :: omp_set_lock
 !dec$ attributes alias:'omp_unset_lock' :: omp_unset_lock
 !dec$ attributes alias:'omp_test_lock' :: omp_test_lock

 !dec$ attributes alias:'_OMP_GET_NUM_DEVICES' :: omp_get_num_devices
 !dec$ attributes alias:'_OMP_GET_NUM_TEAMS' :: omp_get_num_teams
 !dec$ attributes alias:'_OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'_OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'_OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
 !dec$ attributes alias:'_OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device
 !dec$ attributes alias:'_OMP_GET_MAX_TASK_PRIORTY' :: omp_get_max_task_priority
 !dec$ attributes alias:'_OMP_GET_DEVICE_NUM' :: omp_get_device_num
+!dec$ attributes alias:'_OMP_PAUSE_RESOURCE' :: omp_pause_resource
+!dec$ attributes alias:'_OMP_PAUSE_RESOURCE_ALL' :: omp_pause_resource_all
+
 !dec$ attributes alias:'_OMP_CONTROL_TOOL' :: omp_control_tool
 
 !dec$ attributes alias:'_omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint' :: omp_init_lock_with_hint
 !dec$ attributes alias:'_omp_destroy_lock' :: omp_destroy_lock
 !dec$ attributes alias:'_omp_set_lock' :: omp_set_lock
 !dec$ attributes alias:'_omp_unset_lock' :: omp_unset_lock
 !dec$ attributes alias:'_omp_test_lock' :: omp_test_lock

 !dec$ attributes alias:'omp_get_num_devices_'::omp_get_num_devices
 !dec$ attributes alias:'omp_get_num_teams_'::omp_get_num_teams
 !dec$ attributes alias:'omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'omp_is_initial_device_'::omp_is_initial_device
 !dec$ attributes alias:'omp_get_initial_device_'::omp_get_initial_device
 !dec$ attributes alias:'omp_get_max_task_priority_'::omp_get_max_task_priority
 !dec$ attributes alias:'omp_get_device_num_'::omp_get_device_num
+!dec$ attributes alias:'omp_pause_resource_' :: omp_pause_resource
+!dec$ attributes alias:'omp_pause_resource_all_' :: omp_pause_resource_all
 
 !dec$ attributes alias:'omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint_'::omp_init_lock_with_hint
 !dec$ attributes alias:'omp_destroy_lock_'::omp_destroy_lock
 !dec$ attributes alias:'omp_set_lock_'::omp_set_lock
 !dec$ attributes alias:'omp_unset_lock_'::omp_unset_lock
 !dec$ attributes alias:'omp_test_lock_'::omp_test_lock
 !dec$ attributes alias:'omp_init_nest_lock_'::omp_init_nest_lock

 !dec$ attributes alias:'_omp_get_num_devices_'::omp_get_num_devices
 !dec$ attributes alias:'_omp_get_num_teams_'::omp_get_num_teams
 !dec$ attributes alias:'_omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'_omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'_omp_is_initial_device_'::omp_is_initial_device
 !dec$ attributes alias:'_omp_get_initial_device_'::omp_get_initial_device
 !dec$ attributes alias:'_omp_get_max_task_priorty_'::omp_get_max_task_priority
 !dec$ attributes alias:'_omp_get_device_num_'::omp_get_device_num
+!dec$ attributes alias:'_omp_pause_resource_' :: omp_pause_resource
+!dec$ attributes alias:'_omp_pause_resource_all_' :: omp_pause_resource_all
+
 !dec$ attributes alias:'_omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint_'::omp_init_lock_with_hint
 !dec$ attributes alias:'_omp_destroy_lock_'::omp_destroy_lock
 !dec$ attributes alias:'_omp_set_lock_'::omp_set_lock
 !dec$ attributes alias:'_omp_unset_lock_'::omp_unset_lock
 !dec$ attributes alias:'_omp_test_lock_'::omp_test_lock
 !dec$ attributes alias:'_omp_init_nest_lock_'::omp_init_nest_lock
 !dec$ attributes alias:'_omp_init_nest_lock_with_hint_'::omp_init_nest_lock_with_hint

runtime/src/include/50/omp_lib.f90.var

!
         integer, parameter :: kmp_affinity_mask_kind = c_intptr_t
         integer, parameter :: kmp_cancel_kind        = omp_integer_kind
         integer, parameter :: omp_sync_hint_kind     = omp_integer_kind
         integer, parameter :: omp_lock_hint_kind     = omp_sync_hint_kind
         integer, parameter :: omp_control_tool_kind  = omp_integer_kind
         integer, parameter :: omp_control_tool_result_kind = omp_integer_kind
         integer, parameter :: omp_allocator_kind = c_intptr_t
 
+        integer, parameter :: omp_pause_resource_kind = omp_integer_kind
+
       end module omp_lib_kinds
 
       module omp_lib
 
         use omp_lib_kinds
 
         integer (kind=omp_integer_kind), parameter :: openmp_version    = @LIBOMP_OMP_YEAR_MONTH@
         integer (kind=omp_integer_kind), parameter :: kmp_version_major = @LIBOMP_VERSION_MAJOR@
!
         integer (kind=omp_allocator_kind), parameter :: omp_large_cap_mem_alloc = 2
         integer (kind=omp_allocator_kind), parameter :: omp_const_mem_alloc = 3
         integer (kind=omp_allocator_kind), parameter :: omp_high_bw_mem_alloc = 4
         integer (kind=omp_allocator_kind), parameter :: omp_low_lat_mem_alloc = 5
         integer (kind=omp_allocator_kind), parameter :: omp_cgroup_mem_alloc = 6
         integer (kind=omp_allocator_kind), parameter :: omp_pteam_mem_alloc = 7
         integer (kind=omp_allocator_kind), parameter :: omp_thread_mem_alloc = 8
 
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2
+
         interface
 
 !         ***
 !         *** omp_* entry points
 !         ***
 
           subroutine omp_set_num_threads(num_threads) bind(c)
             use omp_lib_kinds
!         ***
             integer (kind=omp_integer_kind) omp_get_initial_device
           end function omp_get_initial_device
 
           function omp_get_device_num() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
+          function omp_pause_resource(level, device) bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_pause_resource_kind), value :: level
+            integer (kind=omp_integer_kind), value :: device
+            integer (kind=omp_integer_kind) omp_pause_resource
+          end function omp_pause_resource
+
+          function omp_pause_resource_all(level) bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_pause_resource_kind), value :: level
+            integer (kind=omp_integer_kind) omp_pause_resource_all
+          end function omp_pause_resource_all
+
           subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 

runtime/src/kmp.h

 #endif
 
 #if KMP_USE_MONITOR
 extern void __kmp_reap_monitor(kmp_info_t *th);
 #endif
 extern void __kmp_reap_worker(kmp_info_t *th);
 extern void __kmp_terminate_thread(int gtid);
 
+extern int __kmp_try_suspend_mx(kmp_info_t *th);
+extern void __kmp_lock_suspend_mx(kmp_info_t *th);
+extern void __kmp_unlock_suspend_mx(kmp_info_t *th);
+
 extern void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag);
 extern void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag);
 extern void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag);
 extern void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag);
 extern void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag);
 extern void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag);
 
 extern void __kmp_elapsed(double *);

 
 #if OMP_40_ENABLED
 // Constants used in libomptarget
 #define KMP_DEVICE_DEFAULT -1 // This is libomptarget's default device.
 #define KMP_HOST_DEVICE -10 // This is what it is in libomptarget, go figure.
 #define KMP_DEVICE_ALL -11 // This is libomptarget's "all devices".
 #endif // OMP_40_ENABLED
 
+#if OMP_50_ENABLED
+// OMP Pause Resource
+
+// The following enum is used both to set the status in __kmp_pause_status, and
+// as the internal equivalent of the externally-visible omp_pause_resource_t.
+typedef enum kmp_pause_status_t {
+  kmp_not_paused = 0, // status is not paused, or, requesting resume
+  kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause
+  kmp_hard_paused = 2 // status is hard-paused, or, requesting hard pause
+} kmp_pause_status_t;
+
+// This stores the pause state of the runtime
+extern kmp_pause_status_t __kmp_pause_status;
+
+// Weak function for pausing a device resource.
+extern int tgt_pause_resource(kmp_pause_status_t level,
+                              int devnum) KMP_WEAK_ATTRIBUTE;
+extern int __kmpc_pause_resource(kmp_pause_status_t level);
+extern int __kmp_pause_resource(kmp_pause_status_t level);
+// Soft resume sets __kmp_pause_status, and wakes up all threads.
+extern void __kmp_resume_if_soft_paused();
+// Hard resume simply resets the status to not paused. Library will appear to
+// be uninitialized after hard pause. Let OMP constructs trigger required
+// initializations.
+static inline void __kmp_resume_if_hard_paused() {
+  if (__kmp_pause_status == kmp_hard_paused) {
+    __kmp_pause_status = kmp_not_paused;
+  }
+}
+#endif // OMP_50_ENABLED
+
 #ifdef __cplusplus
 }
 #endif
 
 #endif /* KMP_H */

runtime/src/kmp_csupport.cpp

/* skip all this code for autopar serialized loops since it results in
      unacceptable overhead */
   if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
     return;
 
   // Not autopar code
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   this_thr = __kmp_threads[global_tid];
   serial_team = this_thr->th.th_serial_team;
 
 #if OMP_45_ENABLED
   kmp_task_team_t *task_team = this_thr->th.th_task_team;
 
   // we need to wait for the proxy tasks before finishing the thread
   if (task_team != NULL && task_team->tt.tt_found_proxy_tasks)

 */
 void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) {
   KMP_COUNT_BLOCK(OMP_BARRIER);
   KC_TRACE(10, ("__kmpc_barrier: called T#%d\n", global_tid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   if (__kmp_env_consistency_check) {
     if (loc == 0) {
       KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
     }
 
     __kmp_check_barrier(global_tid, ct_barrier, loc);
   }
 

 kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid) {
   int status = 0;
 
   KC_TRACE(10, ("__kmpc_master: called T#%d\n", global_tid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   if (KMP_MASTER_GTID(global_tid)) {
     KMP_COUNT_BLOCK(OMP_MASTER);
     KMP_PUSH_PARTITIONED_TIMER(OMP_master);
     status = 1;
   }
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (status) {
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) {
   kmp_info_t *th;
   KMP_DEBUG_ASSERT(__kmp_init_serial);
 
   KC_TRACE(10, ("__kmpc_ordered: called T#%d\n", gtid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
 #if USE_ITT_BUILD
   __kmp_itt_ordered_prep(gtid);
 // TODO: ordered_wait_id
 #endif /* USE_ITT_BUILD */
 
   th = __kmp_threads[gtid];
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL

 kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid) {
   int status;
 
   KC_TRACE(10, ("__kmpc_barrier_master: called T#%d\n", global_tid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   if (__kmp_env_consistency_check)
     __kmp_check_barrier(global_tid, ct_barrier, loc);
 
 #if OMPT_SUPPORT
   omp_frame_t *ompt_frame;
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
     if (ompt_frame->enter_frame == NULL)

 kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid) {
   kmp_int32 ret;
 
   KC_TRACE(10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   if (__kmp_env_consistency_check) {
     if (loc == 0) {
       KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
     }
     __kmp_check_barrier(global_tid, ct_barrier, loc);
   }
 
 #if OMPT_SUPPORT
#endif
   // Reduction clause can not be used as a stand-alone directive.
 
   // do not call __kmp_serial_initialize(), it will be called by
   // __kmp_parallel_initialize() if needed
   // possible detection of false-positive race by the threadchecker ???
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
 // check correctness of reduce block nesting
 #if KMP_USE_DYNAMIC_LOCK
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
 #else
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
 #endif
#endif
   // Reduction clause can not be a stand-alone directive.
 
   // do not call __kmp_serial_initialize(), it will be called by
   // __kmp_parallel_initialize() if needed
   // possible detection of false-positive race by the threadchecker ???
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
 // check correctness of reduce block nesting
 #if KMP_USE_DYNAMIC_LOCK
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
 #else
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
 #endif

 
 #if OMP_50_ENABLED
 int __kmpc_get_target_offload(void) {
   if (!__kmp_init_serial) {
     __kmp_serial_initialize();
   }
   return __kmp_target_offload;
 }
+
+// Weak function for pausing device resources provided by libomptarget
+int tgt_pause_resource(kmp_pause_status_t level, int devnum) {
+  return 1; // just fail if there is no libomptarget
+}
+
+int __kmpc_pause_resource(kmp_pause_status_t level) {
+  if (!__kmp_init_serial) {
+    return 1; // Can't pause if runtime is not initialized
+  }
+  return __kmp_pause_resource(level);
+}
 #endif // OMP_50_ENABLED
 
 // end of file //

runtime/src/kmp_dispatch.cpp

__kmp_dispatch_init(ident_t *loc, int gtid, enum sched_type schedule, T lb,
   KMP_BUILD_ASSERT(sizeof(dispatch_private_info_template<T>) ==
                    sizeof(dispatch_private_info));
   KMP_BUILD_ASSERT(sizeof(dispatch_shared_info_template<UT>) ==
                    sizeof(dispatch_shared_info));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
 #if INCLUDE_SSC_MARKS
   SSC_MARK_DISPATCH_INIT();
 #endif
 #ifdef KMP_DEBUG
   typedef typename traits_t<T>::signed_t ST;
   {
     char *buff;
     // create format specifiers before the debug output

runtime/src/kmp_dispatch_hier.h

#ifdef KMP_DEBUG
   }
 #endif // KMP_DEBUG
   KMP_DEBUG_ASSERT(n > 0);
   KMP_DEBUG_ASSERT(new_layers);
   KMP_DEBUG_ASSERT(new_scheds);
   KMP_DEBUG_ASSERT(new_chunks);
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   th = __kmp_threads[gtid];
   team = th->th.th_team;
   active = !team->t.t_serialized;
   th->th.th_ident = loc;
   num_hw_threads = __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1];
   if (!active) {
     KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d not active parallel. "
                   "Using normal dispatch functions.\n",

runtime/src/kmp_ftn_entry.h

 #endif
 
 #if OMP_50_ENABLED
 // This function will be defined in libomptarget. When libomptarget is not
 // loaded, we assume we are on the host and return KMP_HOST_DEVICE.
 // Compiler/libomptarget will handle this if called inside target.
 int FTN_STDCALL FTN_GET_DEVICE_NUM(void) KMP_WEAK_ATTRIBUTE;
 int FTN_STDCALL FTN_GET_DEVICE_NUM(void) { return KMP_HOST_DEVICE; }
+
+// Compiler will ensure that this is only called from host in sequential region
+int FTN_STDCALL FTN_PAUSE_RESOURCE(kmp_pause_status_t level, int device) {
+#ifdef KMP_STUB
+  return 1; // just fail
+#else
+  if (device == KMP_HOST_DEVICE)
+    return __kmpc_pause_resource(level);
+  else
+    return tgt_pause_resource(level, device);
+#endif
+}
+
+// Compiler will ensure that this is only called from host in sequential region
+int FTN_STDCALL FTN_PAUSE_RESOURCE_ALL(kmp_pause_status_t level) {
+#ifdef KMP_STUB
+  return 1; // just fail
+#else
+  int fails = tgt_pause_resource(level, KMP_DEVICE_ALL); // pause devices
+  fails += __kmpc_pause_resource(level); // pause host
+  return fails;
+#endif
+}
+
 #endif // OMP_50_ENABLED
 
 // GCC compatibility (versioned symbols)
 #ifdef KMP_USE_VERSION_SYMBOLS
 
 /* These following sections create versioned symbols for the
    omp_* routines. The KMP_VERSION_SYMBOL macro expands the API name and
    then maps it to a versioned symbol.

 KMP_VERSION_SYMBOL(FTN_GET_PARTITION_NUM_PLACES, 45, "OMP_4.5");
 KMP_VERSION_SYMBOL(FTN_GET_PARTITION_PLACE_NUMS, 45, "OMP_4.5");
 // KMP_VERSION_SYMBOL(FTN_GET_INITIAL_DEVICE, 45, "OMP_4.5");
 #endif
 
 #if OMP_50_ENABLED
 // OMP_5.0 versioned symbols
 // KMP_VERSION_SYMBOL(FTN_GET_DEVICE_NUM, 50, "OMP_5.0");
+// KMP_VERSION_SYMBOL(FTN_PAUSE_RESOURCE, 50, "OMP_5.0");
+// KMP_VERSION_SYMBOL(FTN_PAUSE_RESOURCE_ALL, 50, "OMP_5.0");
 #endif
 
 #endif // KMP_USE_VERSION_SYMBOLS
 
 #ifdef __cplusplus
 } // extern "C"
 #endif // __cplusplus
 
 // end of file //

runtime/src/kmp_ftn_os.h

 
 #if OMP_50_ENABLED
 #define FTN_CONTROL_TOOL omp_control_tool
 #define FTN_SET_DEFAULT_ALLOCATOR omp_set_default_allocator
 #define FTN_GET_DEFAULT_ALLOCATOR omp_get_default_allocator
 #define FTN_ALLOC omp_alloc
 #define FTN_FREE omp_free
 #define FTN_GET_DEVICE_NUM omp_get_device_num
+#define FTN_PAUSE_RESOURCE omp_pause_resource
+#define FTN_PAUSE_RESOURCE_ALL omp_pause_resource_all
 #endif
 
 #endif /* KMP_FTN_PLAIN */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_APPEND
 

 
 #if OMP_50_ENABLED
 #define FTN_CONTROL_TOOL omp_control_tool_
 #define FTN_SET_DEFAULT_ALLOCATOR omp_set_default_allocator_
 #define FTN_GET_DEFAULT_ALLOCATOR omp_get_default_allocator_
 #define FTN_ALLOC omp_alloc_
 #define FTN_FREE omp_free_
 #define FTN_GET_DEVICE_NUM omp_get_device_num_
+#define FTN_PAUSE_RESOURCE omp_pause_resource_
+#define FTN_PAUSE_RESOURCE_ALL omp_pause_resource_all_
 #endif
 
 #endif /* KMP_FTN_APPEND */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_UPPER
 

 
 #if OMP_50_ENABLED
 #define FTN_CONTROL_TOOL OMP_CONTROL_TOOL
 #define FTN_SET_DEFAULT_ALLOCATOR OMP_SET_DEFAULT_ALLOCATOR
 #define FTN_GET_DEFAULT_ALLOCATOR OMP_GET_DEFAULT_ALLOCATOR
 #define FTN_ALLOC OMP_ALLOC
 #define FTN_FREE OMP_FREE
 #define FTN_GET_DEVICE_NUM OMP_GET_DEVICE_NUM
+#define FTN_PAUSE_RESOURCE OMP_PAUSE_RESOURCE
+#define FTN_PAUSE_RESOURCE_ALL OMP_PAUSE_RESOURCE_ALL
 #endif
 
 #endif /* KMP_FTN_UPPER */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_UAPPEND
 

 
 #if OMP_50_ENABLED
 #define FTN_CONTROL_TOOL OMP_CONTROL_TOOL_
 #define FTN_SET_DEFAULT_ALLOCATOR OMP_SET_DEFAULT_ALLOCATOR_
 #define FTN_GET_DEFAULT_ALLOCATOR OMP_GET_DEFAULT_ALLOCATOR_
 #define FTN_ALLOC OMP_ALLOC_
 #define FTN_FREE OMP_FREE_
 #define FTN_GET_DEVICE_NUM OMP_GET_DEVICE_NUM_
+#define FTN_PAUSE_RESOURCE OMP_PAUSE_RESOURCE_
+#define FTN_PAUSE_RESOURCE_ALL OMP_PAUSE_RESOURCE_ALL_
 #endif
 
 #endif /* KMP_FTN_UAPPEND */
 
 /* -------------------------- GOMP API NAMES ------------------------ */
 // All GOMP_1.0 symbols
 #define KMP_API_NAME_GOMP_ATOMIC_END GOMP_atomic_end
 #define KMP_API_NAME_GOMP_ATOMIC_START GOMP_atomic_start

runtime/src/kmp_global.cpp

 int _You_must_link_with_exactly_one_OpenMP_library = 1;
 int _You_must_link_with_Intel_OpenMP_library = 1;
 #if KMP_OS_WINDOWS && (KMP_VERSION_MAJOR > 4)
 int _You_must_link_with_Microsoft_OpenMP_library = 1;
 #endif
 
 #if OMP_50_ENABLED
 kmp_target_offload_kind_t __kmp_target_offload = tgt_default;
-#endif
+
+// OMP Pause Resources
+kmp_pause_status_t __kmp_pause_status = kmp_not_paused;
+#endif // OMP_50_ENABLED
+
 // end of file //

runtime/src/kmp_gsupport.cpp

 int KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_START)(void) {
   int gtid = __kmp_entry_gtid();
   MKLOC(loc, "GOMP_single_start");
   KA_TRACE(20, ("GOMP_single_start: T#%d\n", gtid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   // 3rd parameter == FALSE prevents kmp_enter_single from pushing a
   // workshare when USE_CHECKS is defined.  We need to avoid the push,
   // as there is no corresponding GOMP_single_end() call.
   kmp_int32 rc = __kmp_enter_single(gtid, &loc, FALSE);
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   kmp_info_t *this_thr = __kmp_threads[gtid];
   kmp_team_t *team = this_thr->th.th_team;
void *KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_START)(void) {
   void *retval;
   int gtid = __kmp_entry_gtid();
   MKLOC(loc, "GOMP_single_copy_start");
   KA_TRACE(20, ("GOMP_single_copy_start: T#%d\n", gtid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   // If this is the first thread to enter, return NULL.  The generated code will
   // then call GOMP_single_copy_end() for this thread only, with the
   // copyprivate data pointer as an argument.
   if (__kmp_enter_single(gtid, &loc, FALSE))
     return NULL;
 
 // Wait for the first thread to set the copyprivate data pointer,
 // and for all other threads to reach this point.

runtime/src/kmp_runtime.cpp

 int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
   int status;
   kmp_info_t *th;
   kmp_team_t *team;
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   th = __kmp_threads[gtid];
   team = th->th.th_team;
   status = 0;
 
   th->th.th_ident = id_ref;
 
   if (team->t.t_serialized) {
     status = 1;
void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
   /* Skip all this code for autopar serialized loops since it results in
      unacceptable overhead */
   if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
     return;
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   this_thr = __kmp_threads[global_tid];
   serial_team = this_thr->th.th_serial_team;
 
   /* utilize the serialized team held by this thread */
   KMP_DEBUG_ASSERT(serial_team);
   KMP_MB();
 
   if (__kmp_tasking_mode != tskm_immediate_exec) {
#endif
     }
 
     /* initialize if needed */
     KMP_DEBUG_ASSERT(
         __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown
     if (!TCR_4(__kmp_init_parallel))
       __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+    __kmp_resume_if_soft_paused();
+#endif
+
     /* setup current data */
     master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with
     // shutdown
     parent_team = master_th->th.th_team;
     master_tid = master_th->th.th_info.ds.ds_tid;
     master_this_cons = master_th->th.th_local.this_construct;
     root = master_th->th.th_root;
     master_active = root->r.r_active;
static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
 
   int gtid;
 
   KMP_DEBUG_ASSERT(thread != NULL);
 
   gtid = thread->th.th_info.ds.ds_gtid;
 
   if (!is_root) {
-
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
       /* Assume the threads are at the fork barrier here */
       KA_TRACE(
           20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
                gtid));
       /* Need release fence here to prevent seg faults for tree forkjoin barrier
        * (GEH) */
       ANNOTATE_HAPPENS_BEFORE(thread);
void __kmp_parallel_initialize(void) {
 
   /* jc: The lock __kmp_initz_lock is already held, so calling
      __kmp_serial_initialize would cause a deadlock.  So we call
      __kmp_do_serial_initialize directly. */
   if (!__kmp_init_middle) {
     __kmp_do_middle_initialize();
   }
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_hard_paused();
+#endif
+
   /* begin initialization */
   KA_TRACE(10, ("__kmp_parallel_initialize: enter\n"));
   KMP_ASSERT(KMP_UBER_GTID(gtid));
 
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
   // Save the FP control regs.
   // Worker threads will set theirs to these values at thread startup.
   __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
#undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
 
   return (retval);
 }
 
 // this function is for testing set/get/determine reduce method
 kmp_int32 __kmp_get_reduce_method(void) {
   return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
 }
+
+#if OMP_50_ENABLED
+
+// Soft pause sets up threads to ignore blocktime and just go to sleep.
+// Spin-wait code checks __kmp_pause_status and reacts accordingly.
+void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }
+
+// Hard pause shuts down the runtime completely.  Resume happens naturally when
+// OpenMP is used subsequently.
+void __kmp_hard_pause() {
+  __kmp_pause_status = kmp_hard_paused;
+  __kmp_internal_end_thread(-1);
+}
+
+// Soft resume sets __kmp_pause_status, and wakes up all threads.
+void __kmp_resume_if_soft_paused() {
+  if (__kmp_pause_status == kmp_soft_paused) {
+    __kmp_pause_status = kmp_not_paused;
+
+    for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
+      kmp_info_t *thread = __kmp_threads[gtid];
+      if (thread) { // Wake it if sleeping
+        kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
+        if (fl.is_sleeping())
+          fl.resume(gtid);
+        else if (__kmp_try_suspend_mx(thread)) { // got suspend lock
+          __kmp_unlock_suspend_mx(thread); // unlock it; it won't sleep
+        } else { // thread holds the lock and may sleep soon
+          do { // until either the thread sleeps, or we can get the lock
+            if (fl.is_sleeping()) {
+              fl.resume(gtid);
+              break;
+            } else if (__kmp_try_suspend_mx(thread)) {
+              __kmp_unlock_suspend_mx(thread);
+              break;
+            }
+          } while (1);
+        }
+      }
+    }
+  }
+}
+
+// This function is called via __kmpc_pause_resource. Returns 0 if successful.
+// TODO: add warning messages
+int __kmp_pause_resource(kmp_pause_status_t level) {
+  if (level == kmp_not_paused) { // requesting resume
+    if (__kmp_pause_status == kmp_not_paused) {
+      // error message about runtime not being paused, so can't resume
+      return 1;
+    } else {
+      KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||
+                       __kmp_pause_status == kmp_hard_paused);
+      __kmp_pause_status = kmp_not_paused;
+      return 0;
+    }
+  } else if (level == kmp_soft_paused) { // requesting soft pause
+    if (__kmp_pause_status != kmp_not_paused) {
+      // error message about already being paused
+      return 1;
+    } else {
+      __kmp_soft_pause();
+      return 0;
+    }
+  } else if (level == kmp_hard_paused) { // requesting hard pause
+    if (__kmp_pause_status != kmp_not_paused) {
+      // error message about already being paused
+      return 1;
+    } else {
+      __kmp_hard_pause();
+      return 0;
+    }
+  } else {
+    // error message about invalid level
+    return 1;
+  }
+}
+
+#endif // OMP_50_ENABLED

runtime/src/kmp_wait_release.h

if (!final_spin || ompt_entry_state != omp_state_wait_barrier_implicit ||
       __ompt_implicit_task_end(this_thr, ompt_entry_state, tId, pId);
     }
   }
 #endif
 
   // Setup for waiting
   KMP_INIT_YIELD(spins);
 
-  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME
+#if OMP_50_ENABLED
+      || __kmp_pause_status == kmp_soft_paused
+#endif
+      ) {
 #if KMP_USE_MONITOR
 // The worker threads cannot rely on the team struct existing at this point.
 // Use the bt values cached in the thread struct instead.
 #ifdef KMP_ADJUST_BLOCKTIME
-    if (__kmp_zero_bt && !this_thr->th.th_team_bt_set)
+    if (
+#if OMP_50_ENABLED
+        __kmp_pause_status == kmp_soft_paused ||
+#endif
+        (__kmp_zero_bt && !this_thr->th.th_team_bt_set))
       // Force immediate suspend if not set by user and more threads than
       // available procs
       hibernate = 0;
     else
       hibernate = this_thr->th.th_team_bt_intervals;
 #else
     hibernate = this_thr->th.th_team_bt_intervals;
 #endif /* KMP_ADJUST_BLOCKTIME */
 
     /* If the blocktime is nonzero, we want to make sure that we spin wait for
        the entirety of the specified #intervals, plus up to one interval more.
        This increment make certain that this thread doesn't go to sleep too
        soon.  */
     if (hibernate != 0)
       hibernate++;
 
     // Add in the current time value.
     hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value);
     KF_TRACE(20, ("__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",
                   th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,
                   hibernate - __kmp_global.g.g_time.dt.t_value));
 #else
+#if OMP_50_ENABLED
+    if (__kmp_pause_status == kmp_soft_paused) {
+      // Force immediate suspend
+      hibernate_goal = KMP_NOW();
+    } else
+#endif
       hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals;
     poll_count = 0;
 #endif // KMP_USE_MONITOR
   }
 
   oversubscribed = (TCR_4(__kmp_nth) > __kmp_avail_proc);
   KMP_MB();
 
   // Main wait spin loop
#if KMP_STATS_ENABLED
     if (this_thr->th.th_stats->isIdle() &&
         KMP_GET_THREAD_STATE() == FORK_JOIN_BARRIER) {
       KMP_SET_THREAD_STATE(IDLE);
       KMP_PUSH_PARTITIONED_TIMER(OMP_idle);
     }
 #endif
 
     // Don't suspend if KMP_BLOCKTIME is set to "infinite"
-    if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
+    if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME
+#if OMP_50_ENABLED
+        && __kmp_pause_status != kmp_soft_paused
+#endif
+        )
       continue;
 
     // Don't suspend if there is a likelihood of new tasks being spawned.
     if ((task_team != NULL) && TCR_4(task_team->tt.tt_found_tasks))
       continue;
 
 #if KMP_USE_MONITOR
     // If we have waited a bit more, fall asleep
     if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate)
       continue;
 #else
     if (KMP_BLOCKING(hibernate_goal, poll_count++))
       continue;
 #endif
 
+#if OMP_50_ENABLED
+    if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
+        __kmp_pause_status != kmp_soft_paused)
+      continue;
+#endif
+
     KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
+
 #if KMP_OS_UNIX
     if (final_spin)
       KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);
 #endif
     flag->suspend(th_gtid);
 #if KMP_OS_UNIX
     if (final_spin)
       KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);

runtime/src/z_Linux_util.cpp

if (th->th.th_suspend_init_count > __kmp_fork_count) {
     if (status != 0 && status != EBUSY) {
       KMP_SYSFAIL("pthread_mutex_destroy", status);
     }
     --th->th.th_suspend_init_count;
     KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
   }
 }
 
+// return true if lock obtained, false otherwise
+int __kmp_try_suspend_mx(kmp_info_t *th) {
+  return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0);
+}
+
+void __kmp_lock_suspend_mx(kmp_info_t *th) {
+  int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
+  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
+}
+
+void __kmp_unlock_suspend_mx(kmp_info_t *th) {
+  int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
+  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+}
+
 /* This routine puts the calling thread to sleep after setting the
    sleep bit for the indicated flag variable to true. */
 template <class C>
 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
   kmp_info_t *th = __kmp_threads[th_gtid];
   int status;
   typename C::flag_t old_spin;
 
   KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
                 flag->get()));
 
   __kmp_suspend_initialize_thread(th);
 
   status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
   KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
 
   KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
                 th_gtid, flag->get()));
 
   /* TODO: shouldn't this use release semantics to ensure that
      __kmp_suspend_initialize_thread gets called first? */
   old_spin = flag->set_sleeping();
-
+#if OMP_50_ENABLED
+  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
+      __kmp_pause_status != kmp_soft_paused) {
+    flag->unset_sleeping();
+    status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
+    KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+    return;
+  }
+#endif
   KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
                " was %x\n",
                th_gtid, flag->get(), flag->load(), old_spin));
 
   if (flag->done_check_val(old_spin)) {
     old_spin = flag->unset_sleeping();
     KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
                  "for spin(%p)\n",

runtime/src/z_Windows_NT_util.cpp

 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
   DeleteCriticalSection(&mx->cs);
 }
 
 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
   EnterCriticalSection(&mx->cs);
 }
 
+int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
+  return TryEnterCriticalSection(&mx->cs);
+}
+
 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
   LeaveCriticalSection(&mx->cs);
 }
 
 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
   cv->waiters_count_ = 0;
   cv->wait_generation_count_ = 0;
   cv->release_count_ = 0;
if (TCR_4(th->th.th_suspend_init)) {
     /* this means we have initialize the suspension pthread objects for this
        thread in this instance of the process */
     __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
     __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
     TCW_4(th->th.th_suspend_init, FALSE);
   }
 }
 
+int __kmp_try_suspend_mx(kmp_info_t *th) {
+  return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
+}
+
+void __kmp_lock_suspend_mx(kmp_info_t *th) {
+  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
+}
+
+void __kmp_unlock_suspend_mx(kmp_info_t *th) {
+  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
+}
+
 /* This routine puts the calling thread to sleep after setting the
    sleep bit for the indicated flag variable to true. */
 template <class C>
 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
   kmp_info_t *th = __kmp_threads[th_gtid];
   int status;
   typename C::flag_t old_spin;
 
   KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
                 th_gtid, flag->get()));
 
   __kmp_suspend_initialize_thread(th);
   __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
 
   KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
                 " loc(%p)\n",
                 th_gtid, flag->get()));
 
   /* TODO: shouldn't this use release semantics to ensure that
      __kmp_suspend_initialize_thread gets called first? */
   old_spin = flag->set_sleeping();
+#if OMP_50_ENABLED
+  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
+      __kmp_pause_status != kmp_soft_paused) {
+    flag->unset_sleeping();
+    __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
+    return;
+  }
+#endif
 
   KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
                " loc(%p)==%d\n",
                th_gtid, flag->get(), *(flag->get())));
 
   if (flag->done_check_val(old_spin)) {
     old_spin = flag->unset_sleeping();
     KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "