Index: runtime/src/dllexports =================================================================== --- runtime/src/dllexports +++ runtime/src/dllexports @@ -406,6 +406,7 @@ # USED FOR 4.5 __kmpc_critical_with_hint 270 __kmpc_get_target_offload 271 __kmpc_omp_reg_task_with_affinity 272 + __kmpc_pause_resource 273 %endif %endif @@ -547,6 +548,9 @@ omp_free 895 omp_get_device_num 896 + omp_pause_resource 756 + omp_pause_resource_all 757 + OMP_NULL_ALLOCATOR DATA omp_default_mem_alloc DATA omp_large_cap_mem_alloc DATA Index: runtime/src/include/50/omp.h.var =================================================================== --- runtime/src/include/50/omp.h.var +++ runtime/src/include/50/omp.h.var @@ -238,6 +238,15 @@ extern void __KAI_KMPC_CONVENTION omp_free(void *ptr, const omp_allocator_t *allocator); #endif + /* OpenMP 5.0 Pause Resources */ + typedef enum omp_pause_resource_t { + omp_pause_resume = 0, + omp_pause_soft = 1, + omp_pause_hard = 2 + } omp_pause_resource_t; + extern int __KAI_KMPC_CONVENTION omp_pause_resource(omp_pause_resource_t, int); + extern int __KAI_KMPC_CONVENTION omp_pause_resource_all(omp_pause_resource_t); + # undef __KAI_KMPC_CONVENTION # undef __KMP_IMP Index: runtime/src/include/50/omp_lib.h.var =================================================================== --- runtime/src/include/50/omp_lib.h.var +++ runtime/src/include/50/omp_lib.h.var @@ -41,6 +41,7 @@ parameter(omp_control_tool_result_kind=omp_integer_kind) integer omp_allocator_kind parameter(omp_allocator_kind=int_ptr_kind()) + integer, parameter :: omp_pause_resource_kind = omp_integer_kind integer(kind=omp_integer_kind)openmp_version parameter(openmp_version=@LIBOMP_OMP_YEAR_MONTH@) @@ -137,6 +138,10 @@ integer(kind=omp_allocator_kind)omp_thread_mem_alloc parameter(omp_thread_mem_alloc=8) + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0 + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1 + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2 + interface ! *** @@ -332,6 +337,19 @@ integer (kind=omp_integer_kind) omp_get_device_num end function omp_get_device_num + function omp_pause_resource(level, device) bind(c) + import + integer (kind=omp_pause_resource_kind), value :: level + integer (kind=omp_integer_kind), value :: device + integer (kind=omp_integer_kind) omp_pause_resource + end function omp_pause_resource + + function omp_pause_resource_all(level) bind(c) + import + integer (kind=omp_pause_resource_kind), value :: level + integer (kind=omp_integer_kind) omp_pause_resource_all + end function omp_pause_resource_all + subroutine omp_init_lock(svar) bind(c) !DIR$ IF(__INTEL_COMPILER.GE.1400) !DIR$ attributes known_intrinsic :: omp_init_lock @@ -631,6 +649,8 @@ !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_initial_device !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_devices !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_device_num +!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_pause_resource +!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_pause_resource_all !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_num_teams !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_team_num !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_init_lock @@ -706,6 +726,8 @@ !$omp declare target(omp_get_initial_device ) !$omp declare target(omp_get_num_devices ) !$omp declare target(omp_get_device_num ) +!$omp declare target(omp_pause_resource ) +!$omp declare target(omp_pause_resource_all ) !$omp declare target(omp_get_num_teams ) !$omp declare target(omp_get_team_num ) !$omp declare target(omp_init_lock ) Index: runtime/src/include/50/omp_lib.f.var =================================================================== --- runtime/src/include/50/omp_lib.f.var +++ runtime/src/include/50/omp_lib.f.var @@ -36,6 +36,8 @@ integer, parameter :: omp_control_tool_result_kind = omp_integer_kind integer, parameter :: omp_allocator_kind = int_ptr_kind() + integer, parameter :: omp_pause_resource_kind = omp_integer_kind + end module omp_lib_kinds module omp_lib @@ -83,6 +85,10 @@ integer (kind=omp_allocator_kind), parameter :: omp_pteam_mem_alloc = 7 integer (kind=omp_allocator_kind), parameter :: omp_thread_mem_alloc = 8 + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0 + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1 + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2 + interface ! *** @@ -283,6 +289,19 @@ integer (kind=omp_integer_kind) omp_get_device_num end function omp_get_device_num + function omp_pause_resource(level, device) + use omp_lib_kinds + integer (kind=omp_pause_resource_kind) level + integer (kind=omp_integer_kind) device + integer (kind=omp_integer_kind) omp_pause_resource + end function omp_pause_resource + + function omp_pause_resource_all(level) + use omp_lib_kinds + integer (kind=omp_pause_resource_kind) level + integer (kind=omp_integer_kind) omp_pause_resource_all + end function omp_pause_resource_all + subroutine omp_init_lock(svar) !DIR$ IF(__INTEL_COMPILER.GE.1400) !DIR$ attributes known_intrinsic :: omp_init_lock @@ -600,6 +619,9 @@ !dec$ attributes alias:'OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device !dec$ attributes alias:'OMP_GET_MAX_TASK_PRIORITY' :: omp_get_max_task_priority !dec$ attributes alias:'OMP_GET_DEVICE_NUM' :: omp_get_device_num +!dec$ attributes alias:'OMP_PAUSE_RESOURCE' :: omp_pause_resource +!dec$ attributes alias:'OMP_PAUSE_RESOURCE_ALL' :: omp_pause_resource_all + !dec$ attributes alias:'OMP_CONTROL_TOOL' :: omp_control_tool !dec$ attributes alias:'omp_init_lock' :: omp_init_lock @@ -683,6 +705,9 @@ !dec$ attributes alias:'_OMP_GET_INITIAL_DEVICE' :: omp_get_initial_device !dec$ attributes alias:'_OMP_GET_MAX_TASK_PRIORTY' :: omp_get_max_task_priority !dec$ attributes alias:'_OMP_GET_DEVICE_NUM' :: omp_get_device_num +!dec$ attributes alias:'_OMP_PAUSE_RESOURCE' :: omp_pause_resource +!dec$ attributes alias:'_OMP_PAUSE_RESOURCE_ALL' :: omp_pause_resource_all + !dec$ attributes alias:'_OMP_CONTROL_TOOL' :: omp_control_tool !dec$ attributes alias:'_omp_init_lock' :: omp_init_lock @@ -769,6 +794,8 @@ !dec$ attributes alias:'omp_get_initial_device_'::omp_get_initial_device !dec$ attributes alias:'omp_get_max_task_priority_'::omp_get_max_task_priority !dec$ attributes alias:'omp_get_device_num_'::omp_get_device_num +!dec$ attributes alias:'omp_pause_resource_' :: omp_pause_resource +!dec$ attributes alias:'omp_pause_resource_all_' :: omp_pause_resource_all !dec$ attributes alias:'omp_init_lock_'::omp_init_lock !dec$ attributes alias:'omp_init_lock_with_hint_'::omp_init_lock_with_hint @@ -854,6 +881,9 @@ !dec$ attributes alias:'_omp_get_initial_device_'::omp_get_initial_device !dec$ attributes alias:'_omp_get_max_task_priorty_'::omp_get_max_task_priority !dec$ attributes alias:'_omp_get_device_num_'::omp_get_device_num +!dec$ attributes alias:'_omp_pause_resource_' :: omp_pause_resource +!dec$ attributes alias:'_omp_pause_resource_all_' :: omp_pause_resource_all + !dec$ attributes alias:'_omp_init_lock_'::omp_init_lock !dec$ attributes alias:'_omp_init_lock_with_hint_'::omp_init_lock_with_hint !dec$ attributes alias:'_omp_destroy_lock_'::omp_destroy_lock Index: runtime/src/include/50/omp_lib.f90.var =================================================================== --- runtime/src/include/50/omp_lib.f90.var +++ runtime/src/include/50/omp_lib.f90.var @@ -33,6 +33,8 @@ integer, parameter :: omp_control_tool_result_kind = omp_integer_kind integer, parameter :: omp_allocator_kind = c_intptr_t + integer, parameter :: omp_pause_resource_kind = omp_integer_kind + end module omp_lib_kinds module omp_lib @@ -97,6 +99,10 @@ integer (kind=omp_allocator_kind), parameter :: omp_pteam_mem_alloc = 7 integer (kind=omp_allocator_kind), parameter :: omp_thread_mem_alloc = 8 + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_resume = 0 + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_soft = 1 + integer (kind=omp_pause_resource_kind), parameter :: omp_pause_hard = 2 + interface ! *** @@ -299,6 +305,19 @@ integer (kind=omp_integer_kind) omp_get_device_num end function omp_get_device_num + function omp_pause_resource(level, device) bind(c) + use omp_lib_kinds + integer (kind=omp_pause_resource_kind), value :: level + integer (kind=omp_integer_kind), value :: device + integer (kind=omp_integer_kind) omp_pause_resource + end function omp_pause_resource + + function omp_pause_resource_all(level) bind(c) + use omp_lib_kinds + integer (kind=omp_pause_resource_kind), value :: level + integer (kind=omp_integer_kind) omp_pause_resource_all + end function omp_pause_resource_all + subroutine omp_init_lock(svar) bind(c) !DIR$ IF(__INTEL_COMPILER.GE.1400) !DIR$ attributes known_intrinsic :: omp_init_lock Index: runtime/src/kmp.h =================================================================== --- runtime/src/kmp.h +++ runtime/src/kmp.h @@ -3427,6 +3427,10 @@ extern void __kmp_reap_worker(kmp_info_t *th); extern void __kmp_terminate_thread(int gtid); +extern int __kmp_try_suspend_mx(kmp_info_t *th); +extern void __kmp_lock_suspend_mx(kmp_info_t *th); +extern void __kmp_unlock_suspend_mx(kmp_info_t *th); + extern void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag); extern void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag); extern void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag); @@ -3965,6 +3969,37 @@ #define KMP_DEVICE_ALL -11 // This is libomptarget's "all devices". #endif // OMP_40_ENABLED +#if OMP_50_ENABLED +// OMP Pause Resource + +// The following enum is used both to set the status in __kmp_pause_status, and +// as the internal equivalent of the externally-visible omp_pause_resource_t. +typedef enum kmp_pause_status_t { + kmp_not_paused = 0, // status is not paused, or, requesting resume + kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause + kmp_hard_paused = 2 // status is hard-paused, or, requesting hard pause +} kmp_pause_status_t; + +// This stores the pause state of the runtime +extern kmp_pause_status_t __kmp_pause_status; + +// Weak function for pausing a device resource. +extern int tgt_pause_resource(kmp_pause_status_t level, + int devnum) KMP_WEAK_ATTRIBUTE; +extern int __kmpc_pause_resource(kmp_pause_status_t level); +extern int __kmp_pause_resource(kmp_pause_status_t level); +// Soft resume sets __kmp_pause_status, and wakes up all threads. +extern void __kmp_resume_if_soft_paused(); +// Hard resume simply resets the status to not paused. Library will appear to +// be uninitialized after hard pause. Let OMP constructs trigger required +// initializations. +static inline void __kmp_resume_if_hard_paused() { + if (__kmp_pause_status == kmp_hard_paused) { + __kmp_pause_status = kmp_not_paused; + } +} +#endif // OMP_50_ENABLED + #ifdef __cplusplus } #endif Index: runtime/src/kmp_csupport.cpp =================================================================== --- runtime/src/kmp_csupport.cpp +++ runtime/src/kmp_csupport.cpp @@ -485,6 +485,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + this_thr = __kmp_threads[global_tid]; serial_team = this_thr->th.th_serial_team; @@ -696,6 +700,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + if (__kmp_env_consistency_check) { if (loc == 0) { KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user? @@ -744,6 +752,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + if (KMP_MASTER_GTID(global_tid)) { KMP_COUNT_BLOCK(OMP_MASTER); KMP_PUSH_PARTITIONED_TIMER(OMP_master); @@ -834,6 +846,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + #if USE_ITT_BUILD __kmp_itt_ordered_prep(gtid); // TODO: ordered_wait_id @@ -1590,6 +1606,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + if (__kmp_env_consistency_check) __kmp_check_barrier(global_tid, ct_barrier, loc); @@ -1648,6 +1668,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + if (__kmp_env_consistency_check) { if (loc == 0) { KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user? @@ -3313,6 +3337,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + // check correctness of reduce block nesting #if KMP_USE_DYNAMIC_LOCK if (__kmp_env_consistency_check) @@ -3533,6 +3561,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + // check correctness of reduce block nesting #if KMP_USE_DYNAMIC_LOCK if (__kmp_env_consistency_check) @@ -4106,6 +4138,18 @@ } return __kmp_target_offload; } + +// Weak function for pausing device resources provided by libomptarget +int tgt_pause_resource(kmp_pause_status_t level, int devnum) { + return 1; // just fail if there is no libomptarget +} + +int __kmpc_pause_resource(kmp_pause_status_t level) { + if (!__kmp_init_serial) { + return 1; // Can't pause if runtime is not initialized + } + return __kmp_pause_resource(level); +} #endif // OMP_50_ENABLED // end of file // Index: runtime/src/kmp_dispatch.cpp =================================================================== --- runtime/src/kmp_dispatch.cpp +++ runtime/src/kmp_dispatch.cpp @@ -740,6 +740,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + #if INCLUDE_SSC_MARKS SSC_MARK_DISPATCH_INIT(); #endif Index: runtime/src/kmp_dispatch_hier.h =================================================================== --- runtime/src/kmp_dispatch_hier.h +++ runtime/src/kmp_dispatch_hier.h @@ -924,6 +924,10 @@ KMP_DEBUG_ASSERT(new_chunks); if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + th = __kmp_threads[gtid]; team = th->th.th_team; active = !team->t.t_serialized; Index: runtime/src/kmp_ftn_entry.h =================================================================== --- runtime/src/kmp_ftn_entry.h +++ runtime/src/kmp_ftn_entry.h @@ -1242,6 +1242,30 @@ // Compiler/libomptarget will handle this if called inside target. int FTN_STDCALL FTN_GET_DEVICE_NUM(void) KMP_WEAK_ATTRIBUTE; int FTN_STDCALL FTN_GET_DEVICE_NUM(void) { return KMP_HOST_DEVICE; } + +// Compiler will ensure that this is only called from host in sequential region +int FTN_STDCALL FTN_PAUSE_RESOURCE(kmp_pause_status_t level, int device) { +#ifdef KMP_STUB + return 1; // just fail +#else + if (device == KMP_HOST_DEVICE) + return __kmpc_pause_resource(level); + else + return tgt_pause_resource(level, device); +#endif +} + +// Compiler will ensure that this is only called from host in sequential region +int FTN_STDCALL FTN_PAUSE_RESOURCE_ALL(kmp_pause_status_t level) { +#ifdef KMP_STUB + return 1; // just fail +#else + int fails = tgt_pause_resource(level, KMP_DEVICE_ALL); // pause devices + fails += __kmpc_pause_resource(level); // pause host + return fails; +#endif +} + #endif // OMP_50_ENABLED // GCC compatibility (versioned symbols) @@ -1345,6 +1369,8 @@ #if OMP_50_ENABLED // OMP_5.0 versioned symbols // KMP_VERSION_SYMBOL(FTN_GET_DEVICE_NUM, 50, "OMP_5.0"); +// KMP_VERSION_SYMBOL(FTN_PAUSE_RESOURCE, 50, "OMP_5.0"); +// KMP_VERSION_SYMBOL(FTN_PAUSE_RESOURCE_ALL, 50, "OMP_5.0"); #endif #endif // KMP_USE_VERSION_SYMBOLS Index: runtime/src/kmp_ftn_os.h =================================================================== --- runtime/src/kmp_ftn_os.h +++ runtime/src/kmp_ftn_os.h @@ -138,6 +138,8 @@ #define FTN_ALLOC omp_alloc #define FTN_FREE omp_free #define FTN_GET_DEVICE_NUM omp_get_device_num +#define FTN_PAUSE_RESOURCE omp_pause_resource +#define FTN_PAUSE_RESOURCE_ALL omp_pause_resource_all #endif #endif /* KMP_FTN_PLAIN */ @@ -263,6 +265,8 @@ #define FTN_ALLOC omp_alloc_ #define FTN_FREE omp_free_ #define FTN_GET_DEVICE_NUM omp_get_device_num_ +#define FTN_PAUSE_RESOURCE omp_pause_resource_ +#define FTN_PAUSE_RESOURCE_ALL omp_pause_resource_all_ #endif #endif /* KMP_FTN_APPEND */ @@ -388,6 +392,8 @@ #define FTN_ALLOC OMP_ALLOC #define FTN_FREE OMP_FREE #define FTN_GET_DEVICE_NUM OMP_GET_DEVICE_NUM +#define FTN_PAUSE_RESOURCE OMP_PAUSE_RESOURCE +#define FTN_PAUSE_RESOURCE_ALL OMP_PAUSE_RESOURCE_ALL #endif #endif /* KMP_FTN_UPPER */ @@ -513,6 +519,8 @@ #define FTN_ALLOC OMP_ALLOC_ #define FTN_FREE OMP_FREE_ #define FTN_GET_DEVICE_NUM OMP_GET_DEVICE_NUM_ +#define FTN_PAUSE_RESOURCE OMP_PAUSE_RESOURCE_ +#define FTN_PAUSE_RESOURCE_ALL OMP_PAUSE_RESOURCE_ALL_ #endif #endif /* KMP_FTN_UAPPEND */ Index: runtime/src/kmp_global.cpp =================================================================== --- runtime/src/kmp_global.cpp +++ runtime/src/kmp_global.cpp @@ -528,5 +528,9 @@ #if OMP_50_ENABLED kmp_target_offload_kind_t __kmp_target_offload = tgt_default; -#endif + +// OMP Pause Resources +kmp_pause_status_t __kmp_pause_status = kmp_not_paused; +#endif // OMP_50_ENABLED + // end of file // Index: runtime/src/kmp_gsupport.cpp =================================================================== --- runtime/src/kmp_gsupport.cpp +++ runtime/src/kmp_gsupport.cpp @@ -120,6 +120,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + // 3rd parameter == FALSE prevents kmp_enter_single from pushing a // workshare when USE_CHECKS is defined. We need to avoid the push, // as there is no corresponding GOMP_single_end() call. @@ -168,6 +172,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + // If this is the first thread to enter, return NULL. The generated code will // then call GOMP_single_copy_end() for this thread only, with the // copyprivate data pointer as an argument. Index: runtime/src/kmp_runtime.cpp =================================================================== --- runtime/src/kmp_runtime.cpp +++ runtime/src/kmp_runtime.cpp @@ -752,6 +752,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + th = __kmp_threads[gtid]; team = th->th.th_team; status = 0; @@ -1176,6 +1180,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + this_thr = __kmp_threads[global_tid]; serial_team = this_thr->th.th_serial_team; @@ -1465,6 +1473,10 @@ if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); +#if OMP_50_ENABLED + __kmp_resume_if_soft_paused(); +#endif + /* setup current data */ master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with // shutdown @@ -5785,7 +5797,6 @@ gtid = thread->th.th_info.ds.ds_gtid; if (!is_root) { - if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { /* Assume the threads are at the fork barrier here */ KA_TRACE( @@ -6854,6 +6865,10 @@ __kmp_do_middle_initialize(); } +#if OMP_50_ENABLED + __kmp_resume_if_hard_paused(); +#endif + /* begin initialization */ KA_TRACE(10, ("__kmp_parallel_initialize: enter\n")); KMP_ASSERT(KMP_UBER_GTID(gtid)); @@ -7785,3 +7800,82 @@ kmp_int32 __kmp_get_reduce_method(void) { return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); } + +#if OMP_50_ENABLED + +// Soft pause sets up threads to ignore blocktime and just go to sleep. +// Spin-wait code checks __kmp_pause_status and reacts accordingly. +void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; } + +// Hard pause shuts down the runtime completely. Resume happens naturally when +// OpenMP is used subsequently. +void __kmp_hard_pause() { + __kmp_pause_status = kmp_hard_paused; + __kmp_internal_end_thread(-1); +} + +// Soft resume sets __kmp_pause_status, and wakes up all threads. +void __kmp_resume_if_soft_paused() { + if (__kmp_pause_status == kmp_soft_paused) { + __kmp_pause_status = kmp_not_paused; + + for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) { + kmp_info_t *thread = __kmp_threads[gtid]; + if (thread) { // Wake it if sleeping + kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread); + if (fl.is_sleeping()) + fl.resume(gtid); + else if (__kmp_try_suspend_mx(thread)) { // got suspend lock + __kmp_unlock_suspend_mx(thread); // unlock it; it won't sleep + } else { // thread holds the lock and may sleep soon + do { // until either the thread sleeps, or we can get the lock + if (fl.is_sleeping()) { + fl.resume(gtid); + break; + } else if (__kmp_try_suspend_mx(thread)) { + __kmp_unlock_suspend_mx(thread); + break; + } + } while (1); + } + } + } + } +} + +// This function is called via __kmpc_pause_resource. Returns 0 if successful. +// TODO: add warning messages +int __kmp_pause_resource(kmp_pause_status_t level) { + if (level == kmp_not_paused) { // requesting resume + if (__kmp_pause_status == kmp_not_paused) { + // error message about runtime not being paused, so can't resume + return 1; + } else { + KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused || + __kmp_pause_status == kmp_hard_paused); + __kmp_pause_status = kmp_not_paused; + return 0; + } + } else if (level == kmp_soft_paused) { // requesting soft pause + if (__kmp_pause_status != kmp_not_paused) { + // error message about already being paused + return 1; + } else { + __kmp_soft_pause(); + return 0; + } + } else if (level == kmp_hard_paused) { // requesting hard pause + if (__kmp_pause_status != kmp_not_paused) { + // error message about already being paused + return 1; + } else { + __kmp_hard_pause(); + return 0; + } + } else { + // error message about invalid level + return 1; + } +} + +#endif // OMP_50_ENABLED Index: runtime/src/kmp_wait_release.h =================================================================== --- runtime/src/kmp_wait_release.h +++ runtime/src/kmp_wait_release.h @@ -272,12 +272,20 @@ // Setup for waiting KMP_INIT_YIELD(spins); - if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { + if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME +#if OMP_50_ENABLED + || __kmp_pause_status == kmp_soft_paused +#endif + ) { #if KMP_USE_MONITOR // The worker threads cannot rely on the team struct existing at this point. // Use the bt values cached in the thread struct instead. #ifdef KMP_ADJUST_BLOCKTIME - if (__kmp_zero_bt && !this_thr->th.th_team_bt_set) + if ( +#if OMP_50_ENABLED + __kmp_pause_status == kmp_soft_paused || +#endif + (__kmp_zero_bt && !this_thr->th.th_team_bt_set)) // Force immediate suspend if not set by user and more threads than // available procs hibernate = 0; @@ -300,7 +308,13 @@ th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate, hibernate - __kmp_global.g.g_time.dt.t_value)); #else - hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals; +#if OMP_50_ENABLED + if (__kmp_pause_status == kmp_soft_paused) { + // Force immediate suspend + hibernate_goal = KMP_NOW(); + } else +#endif + hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals; poll_count = 0; #endif // KMP_USE_MONITOR } @@ -393,7 +407,11 @@ #endif // Don't suspend if KMP_BLOCKTIME is set to "infinite" - if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) + if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME +#if OMP_50_ENABLED + && __kmp_pause_status != kmp_soft_paused +#endif + ) continue; // Don't suspend if there is a likelihood of new tasks being spawned. @@ -409,7 +427,14 @@ continue; #endif +#if OMP_50_ENABLED + if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && + __kmp_pause_status != kmp_soft_paused) + continue; +#endif + KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid)); + #if KMP_OS_UNIX if (final_spin) KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false); Index: runtime/src/z_Linux_util.cpp =================================================================== --- runtime/src/z_Linux_util.cpp +++ runtime/src/z_Linux_util.cpp @@ -1407,6 +1407,21 @@ } } +// return true if lock obtained, false otherwise +int __kmp_try_suspend_mx(kmp_info_t *th) { + return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0); +} + +void __kmp_lock_suspend_mx(kmp_info_t *th) { + int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex); + KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); +} + +void __kmp_unlock_suspend_mx(kmp_info_t *th) { + int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); + KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); +} + /* This routine puts the calling thread to sleep after setting the sleep bit for the indicated flag variable to true. */ template @@ -1430,7 +1445,15 @@ /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread gets called first? */ old_spin = flag->set_sleeping(); - +#if OMP_50_ENABLED + if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && + __kmp_pause_status != kmp_soft_paused) { + flag->unset_sleeping(); + status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); + KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); + return; + } +#endif KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x," " was %x\n", th_gtid, flag->get(), flag->load(), old_spin)); Index: runtime/src/z_Windows_NT_util.cpp =================================================================== --- runtime/src/z_Windows_NT_util.cpp +++ runtime/src/z_Windows_NT_util.cpp @@ -159,6 +159,10 @@ EnterCriticalSection(&mx->cs); } +int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) { + return TryEnterCriticalSection(&mx->cs); +} + void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) { LeaveCriticalSection(&mx->cs); } @@ -300,6 +304,18 @@ } } +int __kmp_try_suspend_mx(kmp_info_t *th) { + return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx); +} + +void __kmp_lock_suspend_mx(kmp_info_t *th) { + __kmp_win32_mutex_lock(&th->th.th_suspend_mx); +} + +void __kmp_unlock_suspend_mx(kmp_info_t *th) { + __kmp_win32_mutex_unlock(&th->th.th_suspend_mx); +} + /* This routine puts the calling thread to sleep after setting the sleep bit for the indicated flag variable to true. */ template @@ -321,6 +337,14 @@ /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread gets called first? */ old_spin = flag->set_sleeping(); +#if OMP_50_ENABLED + if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && + __kmp_pause_status != kmp_soft_paused) { + flag->unset_sleeping(); + __kmp_win32_mutex_unlock(&th->th.th_suspend_mx); + return; + } +#endif KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's" " loc(%p)==%d\n",