Add omp_pause_resource* API

[tlwilmar]

Summary

Add omp_pause_resource and omp_pause_resource_all API and enum, plus stub for internal implementation. Implemented callable helper function to do local pause, and added basic functionality for hard and soft pause.

[AndreyChurbanov]

runtime/src/include/50/omp_lib.f.var

292 ↗

(On Diff #175958)

Parameters should be re-named as (kind, device_num). Because in Fortran user can provide named arguments, so we should use same names as in the specification.

299 ↗

(On Diff #175958)

Parameter "level" should be renamed to "kind" to correspond to the specification.

runtime/src/include/50/omp_lib.f90.var

308 ↗

(On Diff #175958)

Parameters should be re-named as (kind, device_num). Because in Fortran user can provide named arguments, so we should use same names as in the specification.

315 ↗

(On Diff #175958)

Parameter "level" should be renamed to "kind" to correspond to the specification.

runtime/src/include/50/omp_lib.h.var

340 ↗

(On Diff #175958)

Parameters should be re-named as (kind, device_num). Because in Fortran user can provide named arguments, so we should use same names as in the specification.

347 ↗

(On Diff #175958)

Parameter "level" should be renamed to "kind" to correspond to the specification.

[tlwilmar]

Renamed parameters to match 5.0 spec. Added test.

[AndreyChurbanov]

LGTM

[tlwilmar]

Sorry Andrey -- I missed the Windows fix. Added now.

[pawosm01]

This commit causes Flang unit tests to fail.

openmp/trunk/runtime/src/include/50/omp_lib.h.var

143

These three lines are too long causing OpenMP test cases from Flang fortran compiler to fail. See how this problem is addressed in the lines above and below.

[pawosm01]

Please correct.

[AndreyChurbanov]

openmp/trunk/runtime/src/include/50/omp_lib.h.var

143

Paul, sorry about that. I've missed on review that these lines exceed 72 chars.
Now fixed by commit rL351745.

[pawosm01]

It's ok, works for me now.

[kkwli0]

openmp/trunk/runtime/src/include/50/omp_lib.h.var

44

Should use the same syntax (declaration + parameter statement) as the others in this file.

143

declaration + parameter statement

[AndreyChurbanov]

openmp/trunk/runtime/src/include/50/omp_lib.h.var

44

Sure, fixed this as well.

Thanks, Kelvin.

openmp/trunk/runtime/src/dllexports

 # OpenMP 5.0 entry points
 %ifndef stub
     %ifdef OMP_50
         __kmpc_task_reduction_init          268
         __kmpc_task_reduction_get_th_data   269
 # USED FOR 4.5 __kmpc_critical_with_hint    270
         __kmpc_get_target_offload           271
   __kmpc_omp_reg_task_with_affinity   272
+  __kmpc_pause_resource               273
     %endif
 %endif
 
 # User API entry points that have both lower- and upper- case versions for Fortran.
 # Number for lowercase version is indicated.  Number for uppercase is obtained by adding 1000.
 # User API entry points are entry points that start with 'kmp_' or 'omp_'.
 
 omp_destroy_lock                            700
%ifdef OMP_50
     omp_set_affinity_format                 748
     omp_get_affinity_format                 749
     omp_display_affinity                    750
     omp_capture_affinity                    751
     ompc_set_affinity_format                752
     ompc_get_affinity_format                753
     ompc_display_affinity                   754
     ompc_capture_affinity                   755
+    omp_pause_resource                      756
+    omp_pause_resource_all                  757
 
     OMP_NULL_ALLOCATOR                     DATA
     omp_default_mem_alloc                  DATA
     omp_large_cap_mem_alloc                DATA
     omp_const_mem_alloc                    DATA
     omp_high_bw_mem_alloc                  DATA
     omp_low_lat_mem_alloc                  DATA
     omp_cgroup_mem_alloc                   DATA

openmp/trunk/runtime/src/include/50/omp.h.var

 #endif
 
     /* OpenMP 5.0 Affinity Format */
     extern void __KAI_KMPC_CONVENTION omp_set_affinity_format(char const *);
     extern size_t __KAI_KMPC_CONVENTION omp_get_affinity_format(char *, size_t);
     extern void __KAI_KMPC_CONVENTION omp_display_affinity(char const *);
     extern size_t __KAI_KMPC_CONVENTION omp_capture_affinity(char *, size_t, char const *);
 
+    /* OpenMP 5.0 Pause Resources */
+    typedef enum omp_pause_resource_t {
+      omp_pause_resume = 0,
+      omp_pause_soft = 1,
+      omp_pause_hard = 2
+    } omp_pause_resource_t;
+    extern int __KAI_KMPC_CONVENTION omp_pause_resource(omp_pause_resource_t, int);
+    extern int __KAI_KMPC_CONVENTION omp_pause_resource_all(omp_pause_resource_t);
+
 #   undef __KAI_KMPC_CONVENTION
 #   undef __KMP_IMP
 
     /* Warning:
        The following typedefs are not standard, deprecated and will be removed in a future release.
     */
     typedef int     omp_int_t;
     typedef double  omp_wtime_t;
 
 #   ifdef __cplusplus
     }
 #   endif
 
 #endif /* __OMP_H */

openmp/trunk/runtime/src/include/50/omp_lib.h.var

!
       integer omp_lock_hint_kind
       parameter(omp_lock_hint_kind=omp_sync_hint_kind)
       integer omp_control_tool_kind
       parameter(omp_control_tool_kind=omp_integer_kind)
       integer omp_control_tool_result_kind
       parameter(omp_control_tool_result_kind=omp_integer_kind)
       integer omp_allocator_kind
       parameter(omp_allocator_kind=int_ptr_kind())
+      integer, parameter :: omp_pause_resource_kind = omp_integer_kind
 
       integer(kind=omp_integer_kind)openmp_version
       parameter(openmp_version=@LIBOMP_OMP_YEAR_MONTH@)
       integer(kind=omp_integer_kind)kmp_version_major
       parameter(kmp_version_major=@LIBOMP_VERSION_MAJOR@)
       integer(kind=omp_integer_kind)kmp_version_minor
       parameter(kmp_version_minor=@LIBOMP_VERSION_MINOR@)
       integer(kind=omp_integer_kind)kmp_version_build
!
       parameter(omp_low_lat_mem_alloc=5)
       integer(kind=omp_allocator_kind)omp_cgroup_mem_alloc
       parameter(omp_cgroup_mem_alloc=6)
       integer(kind=omp_allocator_kind)omp_pteam_mem_alloc
       parameter(omp_pteam_mem_alloc=7)
       integer(kind=omp_allocator_kind)omp_thread_mem_alloc
       parameter(omp_thread_mem_alloc=8)
 
+      integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0
+      integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1
+      integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2
+
       interface
 
 !       ***
 !       *** omp_* entry points
 !       ***
 
         subroutine omp_set_num_threads(num_threads) bind(c)
           import
!       ***
           integer (kind=omp_integer_kind) omp_get_initial_device
         end function omp_get_initial_device
 
         function omp_get_device_num() bind(c)
           import
           integer (kind=omp_integer_kind) omp_get_device_num
         end function omp_get_device_num
 
+        function omp_pause_resource(kind, device_num) bind(c)
+          import
+          integer (kind=omp_pause_resource_kind), value :: kind
+          integer (kind=omp_integer_kind), value :: device_num
+          integer (kind=omp_integer_kind) omp_pause_resource
+        end function omp_pause_resource
+
+        function omp_pause_resource_all(kind) bind(c)
+          import
+          integer (kind=omp_pause_resource_kind), value :: kind
+          integer (kind=omp_integer_kind) omp_pause_resource_all
+        end function omp_pause_resource_all
+
         subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
           import
           integer (kind=omp_lock_kind) svar
         end subroutine omp_init_lock
 

 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_wtime
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_wtick
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_default_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_default_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_is_initial_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_initial_device
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_devices
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_device_num
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_pause_resource
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_pause_resource_all
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_teams
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_team_num
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_init_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_destroy_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_unset_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_test_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_init_nest_lock

 !$omp declare target(omp_get_wtime )
 !$omp declare target(omp_get_wtick )
 !$omp declare target(omp_get_default_device )
 !$omp declare target(omp_set_default_device )
 !$omp declare target(omp_is_initial_device )
 !$omp declare target(omp_get_initial_device )
 !$omp declare target(omp_get_num_devices )
 !$omp declare target(omp_get_device_num )
+!$omp declare target(omp_pause_resource )
+!$omp declare target(omp_pause_resource_all )
 !$omp declare target(omp_get_num_teams )
 !$omp declare target(omp_get_team_num )
 !$omp declare target(omp_init_lock )
 !$omp declare target(omp_destroy_lock )
 !$omp declare target(omp_set_lock )
 !$omp declare target(omp_unset_lock )
 !$omp declare target(omp_test_lock )
 !$omp declare target(omp_init_nest_lock )

openmp/trunk/runtime/src/include/50/omp_lib.f.var

!dec$ fixedformlinesize:132
         integer, parameter :: kmp_size_t_kind        = int_ptr_kind()
         integer, parameter :: kmp_affinity_mask_kind = int_ptr_kind()
         integer, parameter :: kmp_cancel_kind        = omp_integer_kind
         integer, parameter :: omp_lock_hint_kind     = omp_integer_kind
         integer, parameter :: omp_control_tool_kind  = omp_integer_kind
         integer, parameter :: omp_control_tool_result_kind = omp_integer_kind
         integer, parameter :: omp_allocator_kind = int_ptr_kind()
 
+        integer, parameter :: omp_pause_resource_kind = omp_integer_kind
+
       end module omp_lib_kinds
 
       module omp_lib
 
         use omp_lib_kinds
 
         integer (kind=omp_integer_kind), parameter :: kmp_version_major = @LIBOMP_VERSION_MAJOR@
         integer (kind=omp_integer_kind), parameter :: kmp_version_minor = @LIBOMP_VERSION_MINOR@
!dec$ fixedformlinesize:132
         integer (kind=omp_allocator_kind), parameter :: omp_large_cap_mem_alloc = 2
         integer (kind=omp_allocator_kind), parameter :: omp_const_mem_alloc = 3
         integer (kind=omp_allocator_kind), parameter :: omp_high_bw_mem_alloc = 4
         integer (kind=omp_allocator_kind), parameter :: omp_low_lat_mem_alloc = 5
         integer (kind=omp_allocator_kind), parameter :: omp_cgroup_mem_alloc = 6
         integer (kind=omp_allocator_kind), parameter :: omp_pteam_mem_alloc = 7
         integer (kind=omp_allocator_kind), parameter :: omp_thread_mem_alloc = 8
 
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2
+
         interface
 
 !         ***
 !         *** omp_* entry points
 !         ***
 
           subroutine omp_set_num_threads(num_threads)
             use omp_lib_kinds
!         ***
             integer (kind=omp_integer_kind) omp_get_initial_device
           end function omp_get_initial_device
 
           function omp_get_device_num()
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
+          function omp_pause_resource(kind, device_num)
+            use omp_lib_kinds
+            integer (kind=omp_pause_resource_kind) kind
+            integer (kind=omp_integer_kind) device_num
+            integer (kind=omp_integer_kind) omp_pause_resource
+          end function omp_pause_resource
+
+          function omp_pause_resource_all(kind)
+            use omp_lib_kinds
+            integer (kind=omp_pause_resource_kind) kind
+            integer (kind=omp_integer_kind) omp_pause_resource_all
+          end function omp_pause_resource_all
+
           subroutine omp_init_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 

 !dec$ attributes alias:'OMP_GET_NUM_DEVICES' :: omp_get_num_devices
 !dec$ attributes alias:'OMP_GET_NUM_TEAMS' :: omp_get_num_teams
 !dec$ attributes alias:'OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
 !dec$ attributes alias:'OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device
 !dec$ attributes alias:'OMP_GET_MAX_TASK_PRIORITY' :: omp_get_max_task_priority
 !dec$ attributes alias:'OMP_GET_DEVICE_NUM' :: omp_get_device_num
+!dec$ attributes alias:'OMP_PAUSE_RESOURCE' :: omp_pause_resource
+!dec$ attributes alias:'OMP_PAUSE_RESOURCE_ALL' :: omp_pause_resource_all
+
 !dec$ attributes alias:'OMP_CONTROL_TOOL' :: omp_control_tool
 !dec$ attributes alias:'OMP_SET_AFFINITY_FORMAT' :: omp_set_affinity_format
 !dec$ attributes alias:'OMP_GET_AFFINITY_FORMAT' :: omp_get_affinity_format
 !dec$ attributes alias:'OMP_DISPLAY_AFFINITY' :: omp_display_affinity
 !dec$ attributes alias:'OMP_CAPTURE_AFFINITY' :: omp_capture_affinity
 
 !dec$ attributes alias:'omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint' :: omp_init_lock_with_hint

 !dec$ attributes alias:'_OMP_GET_NUM_DEVICES' :: omp_get_num_devices
 !dec$ attributes alias:'_OMP_GET_NUM_TEAMS' :: omp_get_num_teams
 !dec$ attributes alias:'_OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'_OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'_OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
 !dec$ attributes alias:'_OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device
 !dec$ attributes alias:'_OMP_GET_MAX_TASK_PRIORTY' :: omp_get_max_task_priority
 !dec$ attributes alias:'_OMP_GET_DEVICE_NUM' :: omp_get_device_num
+!dec$ attributes alias:'_OMP_PAUSE_RESOURCE' :: omp_pause_resource
+!dec$ attributes alias:'_OMP_PAUSE_RESOURCE_ALL' :: omp_pause_resource_all
+
 !dec$ attributes alias:'_OMP_CONTROL_TOOL' :: omp_control_tool
 !dec$ attributes alias:'_OMP_SET_AFFINITY_FORMAT' :: omp_set_affinity_format
 !dec$ attributes alias:'_OMP_GET_AFFINITY_FORMAT' :: omp_get_affinity_format
 !dec$ attributes alias:'_OMP_DISPLAY_AFFINITY' :: omp_display_affinity
 !dec$ attributes alias:'_OMP_CAPTURE_AFFINITY' :: omp_capture_affinity
 
 !dec$ attributes alias:'_omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint' :: omp_init_lock_with_hint

 !dec$ attributes alias:'omp_get_num_devices_'::omp_get_num_devices
 !dec$ attributes alias:'omp_get_num_teams_'::omp_get_num_teams
 !dec$ attributes alias:'omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'omp_is_initial_device_'::omp_is_initial_device
 !dec$ attributes alias:'omp_get_initial_device_'::omp_get_initial_device
 !dec$ attributes alias:'omp_get_max_task_priority_'::omp_get_max_task_priority
 !dec$ attributes alias:'omp_get_device_num_'::omp_get_device_num
+!dec$ attributes alias:'omp_pause_resource_' :: omp_pause_resource
+!dec$ attributes alias:'omp_pause_resource_all_' :: omp_pause_resource_all
+
 !dec$ attributes alias:'omp_set_affinity_format_' :: omp_set_affinity_format
 !dec$ attributes alias:'omp_get_affinity_format_' :: omp_get_affinity_format
 !dec$ attributes alias:'omp_display_affinity_' :: omp_display_affinity
 !dec$ attributes alias:'omp_capture_affinity_' :: omp_capture_affinity
 
 !dec$ attributes alias:'omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint_'::omp_init_lock_with_hint
 !dec$ attributes alias:'omp_destroy_lock_'::omp_destroy_lock

 !dec$ attributes alias:'_omp_get_num_devices_'::omp_get_num_devices
 !dec$ attributes alias:'_omp_get_num_teams_'::omp_get_num_teams
 !dec$ attributes alias:'_omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'_omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'_omp_is_initial_device_'::omp_is_initial_device
 !dec$ attributes alias:'_omp_get_initial_device_'::omp_get_initial_device
 !dec$ attributes alias:'_omp_get_max_task_priorty_'::omp_get_max_task_priority
 !dec$ attributes alias:'_omp_get_device_num_'::omp_get_device_num
+!dec$ attributes alias:'_omp_pause_resource_' :: omp_pause_resource
+!dec$ attributes alias:'_omp_pause_resource_all_' :: omp_pause_resource_all
+
 !dec$ attributes alias:'_omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint_'::omp_init_lock_with_hint
 !dec$ attributes alias:'_omp_destroy_lock_'::omp_destroy_lock
 !dec$ attributes alias:'_omp_set_lock_'::omp_set_lock
 !dec$ attributes alias:'_omp_unset_lock_'::omp_unset_lock
 !dec$ attributes alias:'_omp_test_lock_'::omp_test_lock
 !dec$ attributes alias:'_omp_init_nest_lock_'::omp_init_nest_lock
 !dec$ attributes alias:'_omp_init_nest_lock_with_hint_'::omp_init_nest_lock_with_hint

openmp/trunk/runtime/src/include/50/omp_lib.f90.var

!
         integer, parameter :: kmp_affinity_mask_kind = c_intptr_t
         integer, parameter :: kmp_cancel_kind        = omp_integer_kind
         integer, parameter :: omp_sync_hint_kind     = omp_integer_kind
         integer, parameter :: omp_lock_hint_kind     = omp_sync_hint_kind
         integer, parameter :: omp_control_tool_kind  = omp_integer_kind
         integer, parameter :: omp_control_tool_result_kind = omp_integer_kind
         integer, parameter :: omp_allocator_kind = c_intptr_t
 
+        integer, parameter :: omp_pause_resource_kind = omp_integer_kind
+
       end module omp_lib_kinds
 
       module omp_lib
 
         use omp_lib_kinds
 
         integer (kind=omp_integer_kind), parameter :: openmp_version    = @LIBOMP_OMP_YEAR_MONTH@
         integer (kind=omp_integer_kind), parameter :: kmp_version_major = @LIBOMP_VERSION_MAJOR@
!
         integer (kind=omp_allocator_kind), parameter :: omp_large_cap_mem_alloc = 2
         integer (kind=omp_allocator_kind), parameter :: omp_const_mem_alloc = 3
         integer (kind=omp_allocator_kind), parameter :: omp_high_bw_mem_alloc = 4
         integer (kind=omp_allocator_kind), parameter :: omp_low_lat_mem_alloc = 5
         integer (kind=omp_allocator_kind), parameter :: omp_cgroup_mem_alloc = 6
         integer (kind=omp_allocator_kind), parameter :: omp_pteam_mem_alloc = 7
         integer (kind=omp_allocator_kind), parameter :: omp_thread_mem_alloc = 8
 
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1
+        integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2
+
         interface
 
 !         ***
 !         *** omp_* entry points
 !         ***
 
           subroutine omp_set_num_threads(num_threads) bind(c)
             use omp_lib_kinds
!         ***
             integer (kind=omp_integer_kind) omp_get_initial_device
           end function omp_get_initial_device
 
           function omp_get_device_num() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
+          function omp_pause_resource(kind, device_num) bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_pause_resource_kind), value :: kind
+            integer (kind=omp_integer_kind), value :: device_num
+            integer (kind=omp_integer_kind) omp_pause_resource
+          end function omp_pause_resource
+
+          function omp_pause_resource_all(kind) bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_pause_resource_kind), value :: kind
+            integer (kind=omp_integer_kind) omp_pause_resource_all
+          end function omp_pause_resource_all
+
           subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 

openmp/trunk/runtime/src/kmp.h

 #endif
 
 #if KMP_USE_MONITOR
 extern void __kmp_reap_monitor(kmp_info_t *th);
 #endif
 extern void __kmp_reap_worker(kmp_info_t *th);
 extern void __kmp_terminate_thread(int gtid);
 
+extern int __kmp_try_suspend_mx(kmp_info_t *th);
+extern void __kmp_lock_suspend_mx(kmp_info_t *th);
+extern void __kmp_unlock_suspend_mx(kmp_info_t *th);
+
 extern void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag);
 extern void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag);
 extern void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag);
 extern void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag);
 extern void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag);
 extern void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag);
 
 extern void __kmp_elapsed(double *);

 
 #if OMP_40_ENABLED
 // Constants used in libomptarget
 #define KMP_DEVICE_DEFAULT -1 // This is libomptarget's default device.
 #define KMP_HOST_DEVICE -10 // This is what it is in libomptarget, go figure.
 #define KMP_DEVICE_ALL -11 // This is libomptarget's "all devices".
 #endif // OMP_40_ENABLED
 
+#if OMP_50_ENABLED
+// OMP Pause Resource
+
+// The following enum is used both to set the status in __kmp_pause_status, and
+// as the internal equivalent of the externally-visible omp_pause_resource_t.
+typedef enum kmp_pause_status_t {
+  kmp_not_paused = 0, // status is not paused, or, requesting resume
+  kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause
+  kmp_hard_paused = 2 // status is hard-paused, or, requesting hard pause
+} kmp_pause_status_t;
+
+// This stores the pause state of the runtime
+extern kmp_pause_status_t __kmp_pause_status;
+extern int __kmpc_pause_resource(kmp_pause_status_t level);
+extern int __kmp_pause_resource(kmp_pause_status_t level);
+// Soft resume sets __kmp_pause_status, and wakes up all threads.
+extern void __kmp_resume_if_soft_paused();
+// Hard resume simply resets the status to not paused. Library will appear to
+// be uninitialized after hard pause. Let OMP constructs trigger required
+// initializations.
+static inline void __kmp_resume_if_hard_paused() {
+  if (__kmp_pause_status == kmp_hard_paused) {
+    __kmp_pause_status = kmp_not_paused;
+  }
+}
+#endif // OMP_50_ENABLED
+
 #ifdef __cplusplus
 }
 #endif
 
 #endif /* KMP_H */

openmp/trunk/runtime/src/kmp_csupport.cpp

/* skip all this code for autopar serialized loops since it results in
      unacceptable overhead */
   if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
     return;
 
   // Not autopar code
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   this_thr = __kmp_threads[global_tid];
   serial_team = this_thr->th.th_serial_team;
 
 #if OMP_45_ENABLED
   kmp_task_team_t *task_team = this_thr->th.th_task_team;
 
   // we need to wait for the proxy tasks before finishing the thread
   if (task_team != NULL && task_team->tt.tt_found_proxy_tasks)

 */
 void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) {
   KMP_COUNT_BLOCK(OMP_BARRIER);
   KC_TRACE(10, ("__kmpc_barrier: called T#%d\n", global_tid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   if (__kmp_env_consistency_check) {
     if (loc == 0) {
       KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
     }
 
     __kmp_check_barrier(global_tid, ct_barrier, loc);
   }
 

 kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid) {
   int status = 0;
 
   KC_TRACE(10, ("__kmpc_master: called T#%d\n", global_tid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   if (KMP_MASTER_GTID(global_tid)) {
     KMP_COUNT_BLOCK(OMP_MASTER);
     KMP_PUSH_PARTITIONED_TIMER(OMP_master);
     status = 1;
   }
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (status) {
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) {
   kmp_info_t *th;
   KMP_DEBUG_ASSERT(__kmp_init_serial);
 
   KC_TRACE(10, ("__kmpc_ordered: called T#%d\n", gtid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
 #if USE_ITT_BUILD
   __kmp_itt_ordered_prep(gtid);
 // TODO: ordered_wait_id
 #endif /* USE_ITT_BUILD */
 
   th = __kmp_threads[gtid];
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL

 kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid) {
   int status;
 
   KC_TRACE(10, ("__kmpc_barrier_master: called T#%d\n", global_tid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   if (__kmp_env_consistency_check)
     __kmp_check_barrier(global_tid, ct_barrier, loc);
 
 #if OMPT_SUPPORT
   ompt_frame_t *ompt_frame;
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
     if (ompt_frame->enter_frame.ptr == NULL)

 kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid) {
   kmp_int32 ret;
 
   KC_TRACE(10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   if (__kmp_env_consistency_check) {
     if (loc == 0) {
       KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
     }
     __kmp_check_barrier(global_tid, ct_barrier, loc);
   }
 
 #if OMPT_SUPPORT
#endif
   // Reduction clause can not be used as a stand-alone directive.
 
   // do not call __kmp_serial_initialize(), it will be called by
   // __kmp_parallel_initialize() if needed
   // possible detection of false-positive race by the threadchecker ???
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
 // check correctness of reduce block nesting
 #if KMP_USE_DYNAMIC_LOCK
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
 #else
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
 #endif
#endif
   // Reduction clause can not be a stand-alone directive.
 
   // do not call __kmp_serial_initialize(), it will be called by
   // __kmp_parallel_initialize() if needed
   // possible detection of false-positive race by the threadchecker ???
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
 // check correctness of reduce block nesting
 #if KMP_USE_DYNAMIC_LOCK
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
 #else
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
 #endif

 
 #if OMP_50_ENABLED
 int __kmpc_get_target_offload(void) {
   if (!__kmp_init_serial) {
     __kmp_serial_initialize();
   }
   return __kmp_target_offload;
 }
+
+int __kmpc_pause_resource(kmp_pause_status_t level) {
+  if (!__kmp_init_serial) {
+    return 1; // Can't pause if runtime is not initialized
+  }
+  return __kmp_pause_resource(level);
+}
 #endif // OMP_50_ENABLED
 
 // end of file //

openmp/trunk/runtime/src/kmp_dispatch.cpp

__kmp_dispatch_init(ident_t *loc, int gtid, enum sched_type schedule, T lb,
   KMP_BUILD_ASSERT(sizeof(dispatch_private_info_template<T>) ==
                    sizeof(dispatch_private_info));
   KMP_BUILD_ASSERT(sizeof(dispatch_shared_info_template<UT>) ==
                    sizeof(dispatch_shared_info));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
 #if INCLUDE_SSC_MARKS
   SSC_MARK_DISPATCH_INIT();
 #endif
 #ifdef KMP_DEBUG
   typedef typename traits_t<T>::signed_t ST;
   {
     char *buff;
     // create format specifiers before the debug output

openmp/trunk/runtime/src/kmp_dispatch_hier.h

#ifdef KMP_DEBUG
   }
 #endif // KMP_DEBUG
   KMP_DEBUG_ASSERT(n > 0);
   KMP_DEBUG_ASSERT(new_layers);
   KMP_DEBUG_ASSERT(new_scheds);
   KMP_DEBUG_ASSERT(new_chunks);
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   th = __kmp_threads[gtid];
   team = th->th.th_team;
   active = !team->t.t_serialized;
   th->th.th_ident = loc;
   num_hw_threads = __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1];
   if (!active) {
     KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d not active parallel. "
                   "Using normal dispatch functions.\n",

openmp/trunk/runtime/src/kmp_ftn_entry.h

 #endif
 
 #if OMP_50_ENABLED
 // This function will be defined in libomptarget. When libomptarget is not
 // loaded, we assume we are on the host and return KMP_HOST_DEVICE.
 // Compiler/libomptarget will handle this if called inside target.
 int FTN_STDCALL FTN_GET_DEVICE_NUM(void) KMP_WEAK_ATTRIBUTE;
 int FTN_STDCALL FTN_GET_DEVICE_NUM(void) { return KMP_HOST_DEVICE; }
+
+// Compiler will ensure that this is only called from host in sequential region
+int FTN_STDCALL FTN_PAUSE_RESOURCE(kmp_pause_status_t kind, int device_num) {
+#ifdef KMP_STUB
+  return 1; // just fail
+#else
+  if (device_num == KMP_HOST_DEVICE)
+    return __kmpc_pause_resource(kind);
+  else {
+#if !KMP_OS_WINDOWS
+    int (*fptr)(kmp_pause_status_t, int);
+    if ((*(void **)(&fptr) = dlsym(RTLD_DEFAULT, "tgt_pause_resource")))
+      return (*fptr)(kind, device_num);
+    else
+#endif
+      return 1; // just fail if there is no libomptarget
+  }
+#endif
+}
+
+// Compiler will ensure that this is only called from host in sequential region
+int FTN_STDCALL FTN_PAUSE_RESOURCE_ALL(kmp_pause_status_t kind) {
+#ifdef KMP_STUB
+  return 1; // just fail
+#else
+  int fails = 0;
+#if !KMP_OS_WINDOWS
+  int (*fptr)(kmp_pause_status_t, int);
+  if ((*(void **)(&fptr) = dlsym(RTLD_DEFAULT, "tgt_pause_resource")))
+    fails = (*fptr)(kind, KMP_DEVICE_ALL); // pause devices
+#endif
+  fails += __kmpc_pause_resource(kind); // pause host
+  return fails;
+#endif
+}
+
 #endif // OMP_50_ENABLED
 
 // GCC compatibility (versioned symbols)
 #ifdef KMP_USE_VERSION_SYMBOLS
 
 /* These following sections create versioned symbols for the
    omp_* routines. The KMP_VERSION_SYMBOL macro expands the API name and
    then maps it to a versioned symbol.

 KMP_VERSION_SYMBOL(FTN_GET_PARTITION_NUM_PLACES, 45, "OMP_4.5");
 KMP_VERSION_SYMBOL(FTN_GET_PARTITION_PLACE_NUMS, 45, "OMP_4.5");
 // KMP_VERSION_SYMBOL(FTN_GET_INITIAL_DEVICE, 45, "OMP_4.5");
 #endif
 
 #if OMP_50_ENABLED
 // OMP_5.0 versioned symbols
 // KMP_VERSION_SYMBOL(FTN_GET_DEVICE_NUM, 50, "OMP_5.0");
+// KMP_VERSION_SYMBOL(FTN_PAUSE_RESOURCE, 50, "OMP_5.0");
+// KMP_VERSION_SYMBOL(FTN_PAUSE_RESOURCE_ALL, 50, "OMP_5.0");
 #endif
 
 #endif // KMP_USE_VERSION_SYMBOLS
 
 #ifdef __cplusplus
 } // extern "C"
 #endif // __cplusplus
 
 // end of file //

openmp/trunk/runtime/src/kmp_ftn_os.h

 #define FTN_GET_DEFAULT_ALLOCATOR omp_get_default_allocator
 #define FTN_ALLOC omp_alloc
 #define FTN_FREE omp_free
 #define FTN_GET_DEVICE_NUM omp_get_device_num
 #define FTN_SET_AFFINITY_FORMAT omp_set_affinity_format
 #define FTN_GET_AFFINITY_FORMAT omp_get_affinity_format
 #define FTN_DISPLAY_AFFINITY omp_display_affinity
 #define FTN_CAPTURE_AFFINITY omp_capture_affinity
+#define FTN_PAUSE_RESOURCE omp_pause_resource
+#define FTN_PAUSE_RESOURCE_ALL omp_pause_resource_all
 #endif
 
 #endif /* KMP_FTN_PLAIN */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_APPEND
 

 #define FTN_GET_DEFAULT_ALLOCATOR omp_get_default_allocator_
 #define FTN_ALLOC omp_alloc_
 #define FTN_FREE omp_free_
 #define FTN_GET_DEVICE_NUM omp_get_device_num_
 #define FTN_SET_AFFINITY_FORMAT omp_set_affinity_format_
 #define FTN_GET_AFFINITY_FORMAT omp_get_affinity_format_
 #define FTN_DISPLAY_AFFINITY omp_display_affinity_
 #define FTN_CAPTURE_AFFINITY omp_capture_affinity_
+#define FTN_PAUSE_RESOURCE omp_pause_resource_
+#define FTN_PAUSE_RESOURCE_ALL omp_pause_resource_all_
 #endif
 
 #endif /* KMP_FTN_APPEND */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_UPPER
 

 #define FTN_GET_DEFAULT_ALLOCATOR OMP_GET_DEFAULT_ALLOCATOR
 #define FTN_ALLOC OMP_ALLOC
 #define FTN_FREE OMP_FREE
 #define FTN_GET_DEVICE_NUM OMP_GET_DEVICE_NUM
 #define FTN_SET_AFFINITY_FORMAT OMP_SET_AFFINITY_FORMAT
 #define FTN_GET_AFFINITY_FORMAT OMP_GET_AFFINITY_FORMAT
 #define FTN_DISPLAY_AFFINITY OMP_DISPLAY_AFFINITY
 #define FTN_CAPTURE_AFFINITY OMP_CAPTURE_AFFINITY
+#define FTN_PAUSE_RESOURCE OMP_PAUSE_RESOURCE
+#define FTN_PAUSE_RESOURCE_ALL OMP_PAUSE_RESOURCE_ALL
 #endif
 
 #endif /* KMP_FTN_UPPER */
 
 /* ------------------------------------------------------------------------ */
 
 #if KMP_FTN_ENTRIES == KMP_FTN_UAPPEND
 

 #define FTN_GET_DEFAULT_ALLOCATOR OMP_GET_DEFAULT_ALLOCATOR_
 #define FTN_ALLOC OMP_ALLOC_
 #define FTN_FREE OMP_FREE_
 #define FTN_GET_DEVICE_NUM OMP_GET_DEVICE_NUM_
 #define FTN_SET_AFFINITY_FORMAT OMP_SET_AFFINITY_FORMAT_
 #define FTN_GET_AFFINITY_FORMAT OMP_GET_AFFINITY_FORMAT_
 #define FTN_DISPLAY_AFFINITY OMP_DISPLAY_AFFINITY_
 #define FTN_CAPTURE_AFFINITY OMP_CAPTURE_AFFINITY_
+#define FTN_PAUSE_RESOURCE OMP_PAUSE_RESOURCE_
+#define FTN_PAUSE_RESOURCE_ALL OMP_PAUSE_RESOURCE_ALL_
 #endif
 
 #endif /* KMP_FTN_UAPPEND */
 
 /* -------------------------- GOMP API NAMES ------------------------ */
 // All GOMP_1.0 symbols
 #define KMP_API_NAME_GOMP_ATOMIC_END GOMP_atomic_end
 #define KMP_API_NAME_GOMP_ATOMIC_START GOMP_atomic_start

openmp/trunk/runtime/src/kmp_global.cpp

 int _You_must_link_with_exactly_one_OpenMP_library = 1;
 int _You_must_link_with_Intel_OpenMP_library = 1;
 #if KMP_OS_WINDOWS && (KMP_VERSION_MAJOR > 4)
 int _You_must_link_with_Microsoft_OpenMP_library = 1;
 #endif
 
 #if OMP_50_ENABLED
 kmp_target_offload_kind_t __kmp_target_offload = tgt_default;
-#endif
+
+// OMP Pause Resources
+kmp_pause_status_t __kmp_pause_status = kmp_not_paused;
+#endif // OMP_50_ENABLED
+
 // end of file //

openmp/trunk/runtime/src/kmp_gsupport.cpp

 int KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_START)(void) {
   int gtid = __kmp_entry_gtid();
   MKLOC(loc, "GOMP_single_start");
   KA_TRACE(20, ("GOMP_single_start: T#%d\n", gtid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   // 3rd parameter == FALSE prevents kmp_enter_single from pushing a
   // workshare when USE_CHECKS is defined.  We need to avoid the push,
   // as there is no corresponding GOMP_single_end() call.
   kmp_int32 rc = __kmp_enter_single(gtid, &loc, FALSE);
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   kmp_info_t *this_thr = __kmp_threads[gtid];
   kmp_team_t *team = this_thr->th.th_team;
void *KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_START)(void) {
   void *retval;
   int gtid = __kmp_entry_gtid();
   MKLOC(loc, "GOMP_single_copy_start");
   KA_TRACE(20, ("GOMP_single_copy_start: T#%d\n", gtid));
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   // If this is the first thread to enter, return NULL.  The generated code will
   // then call GOMP_single_copy_end() for this thread only, with the
   // copyprivate data pointer as an argument.
   if (__kmp_enter_single(gtid, &loc, FALSE))
     return NULL;
 
 // Wait for the first thread to set the copyprivate data pointer,
 // and for all other threads to reach this point.

openmp/trunk/runtime/src/kmp_runtime.cpp

 int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
   int status;
   kmp_info_t *th;
   kmp_team_t *team;
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   th = __kmp_threads[gtid];
   team = th->th.th_team;
   status = 0;
 
   th->th.th_ident = id_ref;
 
   if (team->t.t_serialized) {
     status = 1;
void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
   /* Skip all this code for autopar serialized loops since it results in
      unacceptable overhead */
   if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
     return;
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_soft_paused();
+#endif
+
   this_thr = __kmp_threads[global_tid];
   serial_team = this_thr->th.th_serial_team;
 
   /* utilize the serialized team held by this thread */
   KMP_DEBUG_ASSERT(serial_team);
   KMP_MB();
 
   if (__kmp_tasking_mode != tskm_immediate_exec) {
#endif
     }
 
     /* initialize if needed */
     KMP_DEBUG_ASSERT(
         __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown
     if (!TCR_4(__kmp_init_parallel))
       __kmp_parallel_initialize();
 
+#if OMP_50_ENABLED
+    __kmp_resume_if_soft_paused();
+#endif
+
     /* setup current data */
     master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with
     // shutdown
     parent_team = master_th->th.th_team;
     master_tid = master_th->th.th_info.ds.ds_tid;
     master_this_cons = master_th->th.th_local.this_construct;
     root = master_th->th.th_root;
     master_active = root->r.r_active;
static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
 
   int gtid;
 
   KMP_DEBUG_ASSERT(thread != NULL);
 
   gtid = thread->th.th_info.ds.ds_gtid;
 
   if (!is_root) {
-
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
       /* Assume the threads are at the fork barrier here */
       KA_TRACE(
           20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
                gtid));
       /* Need release fence here to prevent seg faults for tree forkjoin barrier
        * (GEH) */
       ANNOTATE_HAPPENS_BEFORE(thread);
#if KMP_DYNAMIC_LIB
   // AC: lets not shutdown the Linux* OS dynamic library at the exit of uber
   // thread, because we will better shutdown later in the library destructor.
   // The reason of this change is performance problem when non-openmp thread in
   // a loop forks and joins many openmp threads. We can save a lot of time
   // keeping worker threads alive until the program shutdown.
   // OM: Removed Linux* OS restriction to fix the crash on OS X* (DPD200239966)
   // and Windows(DPD200287443) that occurs when using critical sections from
   // foreign threads.
+  if (__kmp_pause_status != kmp_hard_paused) {
     KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
     return;
+  }
 #endif
   /* synchronize the termination process */
   __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
 
   /* have we already finished */
   if (__kmp_global.g.g_abort) {
     KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n"));
     /* TODO abort? */
void __kmp_parallel_initialize(void) {
 
   /* jc: The lock __kmp_initz_lock is already held, so calling
      __kmp_serial_initialize would cause a deadlock.  So we call
      __kmp_do_serial_initialize directly. */
   if (!__kmp_init_middle) {
     __kmp_do_middle_initialize();
   }
 
+#if OMP_50_ENABLED
+  __kmp_resume_if_hard_paused();
+#endif
+
   /* begin initialization */
   KA_TRACE(10, ("__kmp_parallel_initialize: enter\n"));
   KMP_ASSERT(KMP_UBER_GTID(gtid));
 
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
   // Save the FP control regs.
   // Worker threads will set theirs to these values at thread startup.
   __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
#undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
 
   return (retval);
 }
 
 // this function is for testing set/get/determine reduce method
 kmp_int32 __kmp_get_reduce_method(void) {
   return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
 }
+
+#if OMP_50_ENABLED
+
+// Soft pause sets up threads to ignore blocktime and just go to sleep.
+// Spin-wait code checks __kmp_pause_status and reacts accordingly.
+void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }
+
+// Hard pause shuts down the runtime completely.  Resume happens naturally when
+// OpenMP is used subsequently.
+void __kmp_hard_pause() {
+  __kmp_pause_status = kmp_hard_paused;
+  __kmp_internal_end_thread(-1);
+}
+
+// Soft resume sets __kmp_pause_status, and wakes up all threads.
+void __kmp_resume_if_soft_paused() {
+  if (__kmp_pause_status == kmp_soft_paused) {
+    __kmp_pause_status = kmp_not_paused;
+
+    for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
+      kmp_info_t *thread = __kmp_threads[gtid];
+      if (thread) { // Wake it if sleeping
+        kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
+        if (fl.is_sleeping())
+          fl.resume(gtid);
+        else if (__kmp_try_suspend_mx(thread)) { // got suspend lock
+          __kmp_unlock_suspend_mx(thread); // unlock it; it won't sleep
+        } else { // thread holds the lock and may sleep soon
+          do { // until either the thread sleeps, or we can get the lock
+            if (fl.is_sleeping()) {
+              fl.resume(gtid);
+              break;
+            } else if (__kmp_try_suspend_mx(thread)) {
+              __kmp_unlock_suspend_mx(thread);
+              break;
+            }
+          } while (1);
+        }
+      }
+    }
+  }
+}
+
+// This function is called via __kmpc_pause_resource. Returns 0 if successful.
+// TODO: add warning messages
+int __kmp_pause_resource(kmp_pause_status_t level) {
+  if (level == kmp_not_paused) { // requesting resume
+    if (__kmp_pause_status == kmp_not_paused) {
+      // error message about runtime not being paused, so can't resume
+      return 1;
+    } else {
+      KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||
+                       __kmp_pause_status == kmp_hard_paused);
+      __kmp_pause_status = kmp_not_paused;
+      return 0;
+    }
+  } else if (level == kmp_soft_paused) { // requesting soft pause
+    if (__kmp_pause_status != kmp_not_paused) {
+      // error message about already being paused
+      return 1;
+    } else {
+      __kmp_soft_pause();
+      return 0;
+    }
+  } else if (level == kmp_hard_paused) { // requesting hard pause
+    if (__kmp_pause_status != kmp_not_paused) {
+      // error message about already being paused
+      return 1;
+    } else {
+      __kmp_hard_pause();
+      return 0;
+    }
+  } else {
+    // error message about invalid level
+    return 1;
+  }
+}
+
+#endif // OMP_50_ENABLED

openmp/trunk/runtime/src/kmp_tasking.cpp

KA_TRACE(
         20,
         ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",
          __kmp_gtid_from_thread(this_thr)));
     return;
   }
   threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
   KMP_DEBUG_ASSERT(threads_data != NULL);
 
-  if ((__kmp_tasking_mode == tskm_task_teams) &&
+  if (__kmp_tasking_mode == tskm_task_teams &&
       (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) {
     // Release any threads sleeping at the barrier, so that they can steal
     // tasks and execute them.  In extra barrier mode, tasks do not sleep
     // at the separate tasking barrier, so this isn't a problem.
     for (i = 0; i < nthreads; i++) {
       volatile void *sleep_loc;
       kmp_info_t *thread = threads_data[i].td.td_thr;
 

openmp/trunk/runtime/src/kmp_wait_release.h

if (final_spin && (__kmp_tasking_mode == tskm_immediate_exec ||
       __ompt_implicit_task_end(this_thr, ompt_entry_state, tId);
     }
   }
 #endif
 
   // Setup for waiting
   KMP_INIT_YIELD(spins);
 
-  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME
+#if OMP_50_ENABLED
+      || __kmp_pause_status == kmp_soft_paused
+#endif
+      ) {
 #if KMP_USE_MONITOR
 // The worker threads cannot rely on the team struct existing at this point.
 // Use the bt values cached in the thread struct instead.
 #ifdef KMP_ADJUST_BLOCKTIME
-    if (__kmp_zero_bt && !this_thr->th.th_team_bt_set)
+    if (
+#if OMP_50_ENABLED
+        __kmp_pause_status == kmp_soft_paused ||
+#endif
+        (__kmp_zero_bt && !this_thr->th.th_team_bt_set))
       // Force immediate suspend if not set by user and more threads than
       // available procs
       hibernate = 0;
     else
       hibernate = this_thr->th.th_team_bt_intervals;
 #else
     hibernate = this_thr->th.th_team_bt_intervals;
 #endif /* KMP_ADJUST_BLOCKTIME */
 
     /* If the blocktime is nonzero, we want to make sure that we spin wait for
        the entirety of the specified #intervals, plus up to one interval more.
        This increment make certain that this thread doesn't go to sleep too
        soon.  */
     if (hibernate != 0)
       hibernate++;
 
     // Add in the current time value.
     hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value);
     KF_TRACE(20, ("__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",
                   th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,
                   hibernate - __kmp_global.g.g_time.dt.t_value));
 #else
+#if OMP_50_ENABLED
+    if (__kmp_pause_status == kmp_soft_paused) {
+      // Force immediate suspend
+      hibernate_goal = KMP_NOW();
+    } else
+#endif
       hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals;
     poll_count = 0;
 #endif // KMP_USE_MONITOR
   }
 
   oversubscribed = (TCR_4(__kmp_nth) > __kmp_avail_proc);
   KMP_MB();
 
   // Main wait spin loop
#if KMP_STATS_ENABLED
     if (this_thr->th.th_stats->isIdle() &&
         KMP_GET_THREAD_STATE() == FORK_JOIN_BARRIER) {
       KMP_SET_THREAD_STATE(IDLE);
       KMP_PUSH_PARTITIONED_TIMER(OMP_idle);
     }
 #endif
 
     // Don't suspend if KMP_BLOCKTIME is set to "infinite"
-    if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
+    if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME
+#if OMP_50_ENABLED
+        && __kmp_pause_status != kmp_soft_paused
+#endif
+        )
       continue;
 
     // Don't suspend if there is a likelihood of new tasks being spawned.
     if ((task_team != NULL) && TCR_4(task_team->tt.tt_found_tasks))
       continue;
 
 #if KMP_USE_MONITOR
     // If we have waited a bit more, fall asleep
     if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate)
       continue;
 #else
     if (KMP_BLOCKING(hibernate_goal, poll_count++))
       continue;
 #endif
 
+#if OMP_50_ENABLED
+    if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
+        __kmp_pause_status != kmp_soft_paused)
+      continue;
+#endif
+
     KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
+
 #if KMP_OS_UNIX
     if (final_spin)
       KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);
 #endif
     flag->suspend(th_gtid);
 #if KMP_OS_UNIX
     if (final_spin)
       KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);

openmp/trunk/runtime/src/z_Linux_util.cpp

if (th->th.th_suspend_init_count > __kmp_fork_count) {
     if (status != 0 && status != EBUSY) {
       KMP_SYSFAIL("pthread_mutex_destroy", status);
     }
     --th->th.th_suspend_init_count;
     KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
   }
 }
 
+// return true if lock obtained, false otherwise
+int __kmp_try_suspend_mx(kmp_info_t *th) {
+  return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0);
+}
+
+void __kmp_lock_suspend_mx(kmp_info_t *th) {
+  int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
+  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
+}
+
+void __kmp_unlock_suspend_mx(kmp_info_t *th) {
+  int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
+  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+}
+
 /* This routine puts the calling thread to sleep after setting the
    sleep bit for the indicated flag variable to true. */
 template <class C>
 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
   kmp_info_t *th = __kmp_threads[th_gtid];
   int status;
   typename C::flag_t old_spin;
 
   KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
                 flag->get()));
 
   __kmp_suspend_initialize_thread(th);
 
   status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
   KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
 
   KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
                 th_gtid, flag->get()));
 
   /* TODO: shouldn't this use release semantics to ensure that
      __kmp_suspend_initialize_thread gets called first? */
   old_spin = flag->set_sleeping();
-
+#if OMP_50_ENABLED
+  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
+      __kmp_pause_status != kmp_soft_paused) {
+    flag->unset_sleeping();
+    status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
+    KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+    return;
+  }
+#endif
   KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
                " was %x\n",
                th_gtid, flag->get(), flag->load(), old_spin));
 
   if (flag->done_check_val(old_spin)) {
     old_spin = flag->unset_sleeping();
     KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
                  "for spin(%p)\n",

openmp/trunk/runtime/src/z_Windows_NT_util.cpp

 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
   DeleteCriticalSection(&mx->cs);
 }
 
 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
   EnterCriticalSection(&mx->cs);
 }
 
+int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
+  return TryEnterCriticalSection(&mx->cs);
+}
+
 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
   LeaveCriticalSection(&mx->cs);
 }
 
 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
   cv->waiters_count_ = 0;
   cv->wait_generation_count_ = 0;
   cv->release_count_ = 0;
if (TCR_4(th->th.th_suspend_init)) {
     /* this means we have initialize the suspension pthread objects for this
        thread in this instance of the process */
     __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
     __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
     TCW_4(th->th.th_suspend_init, FALSE);
   }
 }
 
+int __kmp_try_suspend_mx(kmp_info_t *th) {
+  return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
+}
+
+void __kmp_lock_suspend_mx(kmp_info_t *th) {
+  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
+}
+
+void __kmp_unlock_suspend_mx(kmp_info_t *th) {
+  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
+}
+
 /* This routine puts the calling thread to sleep after setting the
    sleep bit for the indicated flag variable to true. */
 template <class C>
 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
   kmp_info_t *th = __kmp_threads[th_gtid];
   int status;
   typename C::flag_t old_spin;
 
   KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
                 th_gtid, flag->get()));
 
   __kmp_suspend_initialize_thread(th);
   __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
 
   KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
                 " loc(%p)\n",
                 th_gtid, flag->get()));
 
   /* TODO: shouldn't this use release semantics to ensure that
      __kmp_suspend_initialize_thread gets called first? */
   old_spin = flag->set_sleeping();
+#if OMP_50_ENABLED
+  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
+      __kmp_pause_status != kmp_soft_paused) {
+    flag->unset_sleeping();
+    __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
+    return;
+  }
+#endif
 
   KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
                " loc(%p)==%d\n",
                th_gtid, flag->get(), *(flag->get())));
 
   if (flag->done_check_val(old_spin)) {
     old_spin = flag->unset_sleeping();
     KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "

openmp/trunk/runtime/test/api/omp_pause_resource.c

+// RUN: %libomp-compile-and-run
+#include <stdio.h>
+#include "omp_testsuite.h"
+
+int test_omp_pause_resource() {
+  int fails, nthreads, my_dev;
+
+  fails = 0;
+  nthreads = 0;
+  my_dev = omp_get_initial_device();
+
+#pragma omp parallel
+#pragma omp single
+  nthreads = omp_get_num_threads();
+
+  if (omp_pause_resource(omp_pause_soft, my_dev))
+    fails++;
+
+#pragma omp parallel shared(nthreads)
+#pragma omp single
+  nthreads = omp_get_num_threads();
+
+  if (nthreads == 0)
+    fails++;
+  if (omp_pause_resource(omp_pause_hard, my_dev))
+    fails++;
+  nthreads = 0;
+
+#pragma omp parallel shared(nthreads)
+#pragma omp single
+  nthreads = omp_get_num_threads();
+
+  if (nthreads == 0)
+    fails++;
+  if (omp_pause_resource_all(omp_pause_soft))
+    fails++;
+  nthreads = 0;
+
+#pragma omp parallel shared(nthreads)
+#pragma omp single
+  nthreads = omp_get_num_threads();
+
+  if (nthreads == 0)
+    fails++;
+  return fails == 0;
+}
+
+int main() {
+  int i;
+  int num_failed = 0;
+
+  for (i = 0; i < REPETITIONS; i++) {
+    if (!test_omp_pause_resource()) {
+      num_failed++;
+    }
+  }
+  return num_failed;
+}