Index: openmp/trunk/runtime/src/kmp.h =================================================================== --- openmp/trunk/runtime/src/kmp.h +++ openmp/trunk/runtime/src/kmp.h @@ -3748,9 +3748,6 @@ kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list); -extern void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task); -extern void __kmp_dephash_free_entries(kmp_info_t *thread, kmp_dephash_t *h); -extern void __kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h); extern kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task, bool serialize_immediate); Index: openmp/trunk/runtime/src/kmp_taskdeps.h =================================================================== --- openmp/trunk/runtime/src/kmp_taskdeps.h +++ openmp/trunk/runtime/src/kmp_taskdeps.h @@ -0,0 +1,145 @@ +/* + * kmp_taskdeps.h + */ + + +//===----------------------------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.txt for details. +// +//===----------------------------------------------------------------------===// + + +#ifndef KMP_TASKDEPS_H +#define KMP_TASKDEPS_H + +#include "kmp.h" + +#if OMP_40_ENABLED + +#define KMP_ACQUIRE_DEPNODE(gtid, n) __kmp_acquire_lock(&(n)->dn.lock, (gtid)) +#define KMP_RELEASE_DEPNODE(gtid, n) __kmp_release_lock(&(n)->dn.lock, (gtid)) + +static inline void __kmp_node_deref(kmp_info_t *thread, kmp_depnode_t *node) { + if (!node) + return; + + kmp_int32 n = KMP_ATOMIC_DEC(&node->dn.nrefs) - 1; + if (n == 0) { + KMP_ASSERT(node->dn.nrefs == 0); +#if USE_FAST_MEMORY + __kmp_fast_free(thread, node); +#else + __kmp_thread_free(thread, node); +#endif + } +} + +static inline void __kmp_depnode_list_free(kmp_info_t *thread, + kmp_depnode_list *list) { + kmp_depnode_list *next; + + for (; list; list = next) { + next = list->next; + + __kmp_node_deref(thread, list->node); +#if USE_FAST_MEMORY + __kmp_fast_free(thread, list); +#else + __kmp_thread_free(thread, list); +#endif + } +} + +static inline void __kmp_dephash_free_entries(kmp_info_t *thread, + kmp_dephash_t *h) { + for (size_t i = 0; i < h->size; i++) { + if (h->buckets[i]) { + kmp_dephash_entry_t *next; + for (kmp_dephash_entry_t *entry = h->buckets[i]; entry; entry = next) { + next = entry->next_in_bucket; + __kmp_depnode_list_free(thread, entry->last_ins); + __kmp_node_deref(thread, entry->last_out); +#if USE_FAST_MEMORY + __kmp_fast_free(thread, entry); +#else + __kmp_thread_free(thread, entry); +#endif + } + h->buckets[i] = 0; + } + } +} + +static inline void __kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h) { + __kmp_dephash_free_entries(thread, h); +#if USE_FAST_MEMORY + __kmp_fast_free(thread, h); +#else + __kmp_thread_free(thread, h); +#endif +} + +static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) { + kmp_info_t *thread = __kmp_threads[gtid]; + kmp_depnode_t *node = task->td_depnode; + + if (task->td_dephash) { + KA_TRACE( + 40, ("__kmp_release_deps: T#%d freeing dependencies hash of task %p.\n", + gtid, task)); + __kmp_dephash_free(thread, task->td_dephash); + task->td_dephash = NULL; + } + + if (!node) + return; + + KA_TRACE(20, ("__kmp_release_deps: T#%d notifying successors of task %p.\n", + gtid, task)); + + KMP_ACQUIRE_DEPNODE(gtid, node); + node->dn.task = + NULL; // mark this task as finished, so no new dependencies are generated + KMP_RELEASE_DEPNODE(gtid, node); + + kmp_depnode_list_t *next; + for (kmp_depnode_list_t *p = node->dn.successors; p; p = next) { + kmp_depnode_t *successor = p->node; + kmp_int32 npredecessors = KMP_ATOMIC_DEC(&successor->dn.npredecessors) - 1; + + // successor task can be NULL for wait_depends or because deps are still + // being processed + if (npredecessors == 0) { + KMP_MB(); + if (successor->dn.task) { + KA_TRACE(20, ("__kmp_release_deps: T#%d successor %p of %p scheduled " + "for execution.\n", + gtid, successor->dn.task, task)); + __kmp_omp_task(gtid, successor->dn.task, false); + } + } + + next = p->next; + __kmp_node_deref(thread, p->node); +#if USE_FAST_MEMORY + __kmp_fast_free(thread, p); +#else + __kmp_thread_free(thread, p); +#endif + } + + __kmp_node_deref(thread, node); + + KA_TRACE( + 20, + ("__kmp_release_deps: T#%d all successors of %p notified of completion\n", + gtid, task)); +} + +#endif // OMP_40_ENABLED + +#endif // KMP_TASKDEPS_H Index: openmp/trunk/runtime/src/kmp_taskdeps.cpp =================================================================== --- openmp/trunk/runtime/src/kmp_taskdeps.cpp +++ openmp/trunk/runtime/src/kmp_taskdeps.cpp @@ -16,6 +16,7 @@ #include "kmp.h" #include "kmp_io.h" #include "kmp_wait_release.h" +#include "kmp_taskdeps.h" #if OMPT_SUPPORT #include "ompt-specific.h" #endif @@ -52,26 +53,6 @@ return node; } -static inline void __kmp_node_deref(kmp_info_t *thread, kmp_depnode_t *node) { - if (!node) - return; - - kmp_int32 n = KMP_ATOMIC_DEC(&node->dn.nrefs) - 1; - if (n == 0) { - KMP_ASSERT(node->dn.nrefs == 0); -#if USE_FAST_MEMORY - __kmp_fast_free(thread, node); -#else - __kmp_thread_free(thread, node); -#endif - } -} - -#define KMP_ACQUIRE_DEPNODE(gtid, n) __kmp_acquire_lock(&(n)->dn.lock, (gtid)) -#define KMP_RELEASE_DEPNODE(gtid, n) __kmp_release_lock(&(n)->dn.lock, (gtid)) - -static void __kmp_depnode_list_free(kmp_info_t *thread, kmp_depnode_list *list); - enum { KMP_DEPHASH_OTHER_SIZE = 97, KMP_DEPHASH_MASTER_SIZE = 997 }; static inline kmp_int32 __kmp_dephash_hash(kmp_intptr_t addr, size_t hsize) { @@ -113,34 +94,6 @@ return h; } -void __kmp_dephash_free_entries(kmp_info_t *thread, kmp_dephash_t *h) { - for (size_t i = 0; i < h->size; i++) { - if (h->buckets[i]) { - kmp_dephash_entry_t *next; - for (kmp_dephash_entry_t *entry = h->buckets[i]; entry; entry = next) { - next = entry->next_in_bucket; - __kmp_depnode_list_free(thread, entry->last_ins); - __kmp_node_deref(thread, entry->last_out); -#if USE_FAST_MEMORY - __kmp_fast_free(thread, entry); -#else - __kmp_thread_free(thread, entry); -#endif - } - h->buckets[i] = 0; - } - } -} - -void __kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h) { - __kmp_dephash_free_entries(thread, h); -#if USE_FAST_MEMORY - __kmp_fast_free(thread, h); -#else - __kmp_thread_free(thread, h); -#endif -} - static kmp_dephash_entry * __kmp_dephash_find(kmp_info_t *thread, kmp_dephash_t *h, kmp_intptr_t addr) { kmp_int32 bucket = __kmp_dephash_hash(addr, h->size); @@ -192,22 +145,6 @@ return new_head; } -static void __kmp_depnode_list_free(kmp_info_t *thread, - kmp_depnode_list *list) { - kmp_depnode_list *next; - - for (; list; list = next) { - next = list->next; - - __kmp_node_deref(thread, list->node); -#if USE_FAST_MEMORY - __kmp_fast_free(thread, list); -#else - __kmp_thread_free(thread, list); -#endif - } -} - static inline void __kmp_track_dependence(kmp_depnode_t *source, kmp_depnode_t *sink, kmp_task_t *sink_task) { @@ -386,63 +323,6 @@ return npredecessors > 0 ? true : false; } -void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) { - kmp_info_t *thread = __kmp_threads[gtid]; - kmp_depnode_t *node = task->td_depnode; - - if (task->td_dephash) { - KA_TRACE( - 40, ("__kmp_release_deps: T#%d freeing dependencies hash of task %p.\n", - gtid, task)); - __kmp_dephash_free(thread, task->td_dephash); - task->td_dephash = NULL; - } - - if (!node) - return; - - KA_TRACE(20, ("__kmp_release_deps: T#%d notifying successors of task %p.\n", - gtid, task)); - - KMP_ACQUIRE_DEPNODE(gtid, node); - node->dn.task = - NULL; // mark this task as finished, so no new dependencies are generated - KMP_RELEASE_DEPNODE(gtid, node); - - kmp_depnode_list_t *next; - for (kmp_depnode_list_t *p = node->dn.successors; p; p = next) { - kmp_depnode_t *successor = p->node; - kmp_int32 npredecessors = KMP_ATOMIC_DEC(&successor->dn.npredecessors) - 1; - - // successor task can be NULL for wait_depends or because deps are still - // being processed - if (npredecessors == 0) { - KMP_MB(); - if (successor->dn.task) { - KA_TRACE(20, ("__kmp_release_deps: T#%d successor %p of %p scheduled " - "for execution.\n", - gtid, successor->dn.task, task)); - __kmp_omp_task(gtid, successor->dn.task, false); - } - } - - next = p->next; - __kmp_node_deref(thread, p->node); -#if USE_FAST_MEMORY - __kmp_fast_free(thread, p); -#else - __kmp_thread_free(thread, p); -#endif - } - - __kmp_node_deref(thread, node); - - KA_TRACE( - 20, - ("__kmp_release_deps: T#%d all successors of %p notified of completion\n", - gtid, task)); -} - /*! @ingroup TASKING @param loc_ref location of the original task directive Index: openmp/trunk/runtime/src/kmp_tasking.cpp =================================================================== --- openmp/trunk/runtime/src/kmp_tasking.cpp +++ openmp/trunk/runtime/src/kmp_tasking.cpp @@ -16,6 +16,7 @@ #include "kmp_itt.h" #include "kmp_stats.h" #include "kmp_wait_release.h" +#include "kmp_taskdeps.h" #if OMPT_SUPPORT #include "ompt-specific.h" @@ -764,15 +765,14 @@ #if OMP_40_ENABLED if (taskdata->td_taskgroup) KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); + __kmp_release_deps(gtid, taskdata); #if OMP_45_ENABLED - } - // if we found proxy tasks there could exist a dependency chain - // with the proxy task as origin - if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) || - (task_team && task_team->tt.tt_found_proxy_tasks)) { -#endif + } else if (task_team && task_team->tt.tt_found_proxy_tasks) { + // if we found proxy tasks there could exist a dependency chain + // with the proxy task as origin __kmp_release_deps(gtid, taskdata); -#endif +#endif // OMP_45_ENABLED +#endif // OMP_40_ENABLED } // td_flags.executing must be marked as 0 after __kmp_release_deps has been