This is an archive of the discontinued LLVM Phabricator instance.

Differential D26356

Dynamic affinity dispatch capabilities.
ClosedPublic

Authored by jlpeyton on Nov 7 2016, 10:44 AM.

Details

Reviewers
tlwilmar
AndreyChurbanov
Commits
rG1cdd87adfd37: Introduce dynamic affinity dispatch capabilities
rOMP286890: Introduce dynamic affinity dispatch capabilities
rL286890: Introduce dynamic affinity dispatch capabilities
Summary

This set of changes enables the affinity interface (Either the preexisting native operating system or HWLOC) to be dynamically set at runtime initialization. The point of this change is that we were seeing performance degradations when using HWLOC. This allows the user to use the old affinity mechanisms which on large machines (>64 cores) makes a large difference in initialization time.

These changes mostly move affinity code under a small class hierarchy:

KMPAffinity {
  class Mask {}
};
KMPNativeAffinity : public KMPAffinity {
  class Mask : public KMPAffinity::Mask {}
};
KMPHwlocAffinity {
  class Mask : public KMPAffinity::Mask {}
}

Since all interface functions (for both affinity and the mask implementation) are virtual, the implementation can be chosen at runtime initialization.

Diff Detail

Repository
rL LLVM

Event Timeline

jlpeyton updated this revision to Diff 77057.Nov 7 2016, 10:44 AM
jlpeyton retitled this revision from to Dynamic affinity dispatch capabilities..
jlpeyton updated this object.
jlpeyton added reviewers: AndreyChurbanov, tlwilmar.
jlpeyton set the repository for this revision to rL LLVM.
jlpeyton added a subscriber: openmp-commits.
Herald added subscribers: modocache, mgorny. · View Herald TranscriptNov 7 2016, 10:44 AM
tlwilmar accepted this revision.Nov 10 2016, 1:00 PM
tlwilmar edited edge metadata.

LGTM

This revision is now accepted and ready to land.Nov 10 2016, 1:00 PM
Closed by commit rL286890: Introduce dynamic affinity dispatch capabilities (authored by jlpeyton). · Explain WhyNov 14 2016, 1:18 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
openmp/
trunk/
runtime/
cmake/
1 line
1 line
src/
368 lines
464 lines
98 lines
1 line
18 lines
1 line
12 lines
1 line
37 lines
113 lines
239 lines

Diff 77872

openmp/trunk/runtime/cmake/LibompHandleFlags.cmake

Show All 22 Linesmacro(libomp_setup_flags flags)
endif() endif()
endmacro() endmacro()
# Gets flags common to both the C and C++ compiler # Gets flags common to both the C and C++ compiler
function(libomp_get_c_and_cxxflags_common flags) function(libomp_get_c_and_cxxflags_common flags)
set(flags_local) set(flags_local)
libomp_append(flags_local -std=c++11 LIBOMP_HAVE_STD_CPP11_FLAG) libomp_append(flags_local -std=c++11 LIBOMP_HAVE_STD_CPP11_FLAG)
libomp_append(flags_local -fno-exceptions LIBOMP_HAVE_FNO_EXCEPTIONS_FLAG) libomp_append(flags_local -fno-exceptions LIBOMP_HAVE_FNO_EXCEPTIONS_FLAG)
libomp_append(flags_local -fno-rtti LIBOMP_HAVE_FNO_RTTI_FLAG)
if(${LIBOMP_ENABLE_WERROR}) if(${LIBOMP_ENABLE_WERROR})
libomp_append(flags_local -Werror LIBOMP_HAVE_WERROR_FLAG) libomp_append(flags_local -Werror LIBOMP_HAVE_WERROR_FLAG)
endif() endif()
libomp_append(flags_local -Wno-sign-compare LIBOMP_HAVE_WNO_SIGN_COMPARE_FLAG) libomp_append(flags_local -Wno-sign-compare LIBOMP_HAVE_WNO_SIGN_COMPARE_FLAG)
libomp_append(flags_local -Wno-unused-function LIBOMP_HAVE_WNO_UNUSED_FUNCTION_FLAG) libomp_append(flags_local -Wno-unused-function LIBOMP_HAVE_WNO_UNUSED_FUNCTION_FLAG)
libomp_append(flags_local -Wno-unused-local-typedef LIBOMP_HAVE_WNO_UNUSED_LOCAL_TYPEDEF_FLAG) libomp_append(flags_local -Wno-unused-local-typedef LIBOMP_HAVE_WNO_UNUSED_LOCAL_TYPEDEF_FLAG)
libomp_append(flags_local -Wno-unused-value LIBOMP_HAVE_WNO_UNUSED_VALUE_FLAG) libomp_append(flags_local -Wno-unused-value LIBOMP_HAVE_WNO_UNUSED_VALUE_FLAG)
libomp_append(flags_local -Wno-unused-variable LIBOMP_HAVE_WNO_UNUSED_VARIABLE_FLAG) libomp_append(flags_local -Wno-unused-variable LIBOMP_HAVE_WNO_UNUSED_VARIABLE_FLAG)
▲ Show 20 Lines • Show All 169 Lines • Show Last 20 Lines

openmp/trunk/runtime/cmake/config-ix.cmake

Show First 20 Lines • Show All 43 Lines • ▼ Show 20 Linesfunction(libomp_check_architecture_flag flag retval)
set(CMAKE_REQUIRED_FLAGS "${flag}") set(CMAKE_REQUIRED_FLAGS "${flag}")
check_c_compiler_flag("${flag}" ${retval}) check_c_compiler_flag("${flag}" ${retval})
set(${retval} ${${retval}} PARENT_SCOPE) set(${retval} ${${retval}} PARENT_SCOPE)
endfunction() endfunction()
# Checking C, CXX, Linker Flags # Checking C, CXX, Linker Flags
check_cxx_compiler_flag(-std=c++11 LIBOMP_HAVE_STD_CPP11_FLAG) check_cxx_compiler_flag(-std=c++11 LIBOMP_HAVE_STD_CPP11_FLAG)
check_cxx_compiler_flag(-fno-exceptions LIBOMP_HAVE_FNO_EXCEPTIONS_FLAG) check_cxx_compiler_flag(-fno-exceptions LIBOMP_HAVE_FNO_EXCEPTIONS_FLAG)
check_cxx_compiler_flag(-fno-rtti LIBOMP_HAVE_FNO_RTTI_FLAG)
check_c_compiler_flag("-x c++" LIBOMP_HAVE_X_CPP_FLAG) check_c_compiler_flag("-x c++" LIBOMP_HAVE_X_CPP_FLAG)
check_c_compiler_flag(-Werror LIBOMP_HAVE_WERROR_FLAG) check_c_compiler_flag(-Werror LIBOMP_HAVE_WERROR_FLAG)
check_c_compiler_flag(-Wunused-function LIBOMP_HAVE_WNO_UNUSED_FUNCTION_FLAG) check_c_compiler_flag(-Wunused-function LIBOMP_HAVE_WNO_UNUSED_FUNCTION_FLAG)
check_c_compiler_flag(-Wunused-local-typedef LIBOMP_HAVE_WNO_UNUSED_LOCAL_TYPEDEF_FLAG) check_c_compiler_flag(-Wunused-local-typedef LIBOMP_HAVE_WNO_UNUSED_LOCAL_TYPEDEF_FLAG)
check_c_compiler_flag(-Wunused-value LIBOMP_HAVE_WNO_UNUSED_VALUE_FLAG) check_c_compiler_flag(-Wunused-value LIBOMP_HAVE_WNO_UNUSED_VALUE_FLAG)
check_c_compiler_flag(-Wunused-variable LIBOMP_HAVE_WNO_UNUSED_VARIABLE_FLAG) check_c_compiler_flag(-Wunused-variable LIBOMP_HAVE_WNO_UNUSED_VARIABLE_FLAG)
check_c_compiler_flag(-Wswitch LIBOMP_HAVE_WNO_SWITCH_FLAG) check_c_compiler_flag(-Wswitch LIBOMP_HAVE_WNO_SWITCH_FLAG)
check_c_compiler_flag(-Wcovered-switch-default LIBOMP_HAVE_WNO_COVERED_SWITCH_DEFAULT_FLAG) check_c_compiler_flag(-Wcovered-switch-default LIBOMP_HAVE_WNO_COVERED_SWITCH_DEFAULT_FLAG)
▲ Show 20 Lines • Show All 214 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/kmp.h

Show First 20 Lines • Show All 522 Lines • ▼ Show 20 Lines
/* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */
/* /*
* Only Linux* OS and Windows* OS support thread affinity. * Only Linux* OS and Windows* OS support thread affinity.
*/ */
#if KMP_AFFINITY_SUPPORTED #if KMP_AFFINITY_SUPPORTED
# if KMP_GROUP_AFFINITY
// GROUP_AFFINITY is already defined for _MSC_VER>=1600 (VS2010 and later). // GROUP_AFFINITY is already defined for _MSC_VER>=1600 (VS2010 and later).
# if KMP_OS_WINDOWS
# if _MSC_VER < 1600 # if _MSC_VER < 1600
typedef struct GROUP_AFFINITY { typedef struct GROUP_AFFINITY {
KAFFINITY Mask; KAFFINITY Mask;
WORD Group; WORD Group;
WORD Reserved[3]; WORD Reserved[3];
} GROUP_AFFINITY; } GROUP_AFFINITY;
# endif /* _MSC_VER < 1600 */ # endif /* _MSC_VER < 1600 */
# if KMP_GROUP_AFFINITY
extern int __kmp_num_proc_groups; extern int __kmp_num_proc_groups;
# else
static const int __kmp_num_proc_groups = 1;
# endif /* KMP_GROUP_AFFINITY */
typedef DWORD (*kmp_GetActiveProcessorCount_t)(WORD); typedef DWORD (*kmp_GetActiveProcessorCount_t)(WORD);
extern kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount; extern kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount;
typedef WORD (*kmp_GetActiveProcessorGroupCount_t)(void); typedef WORD (*kmp_GetActiveProcessorGroupCount_t)(void);
extern kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount; extern kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount;
typedef BOOL (*kmp_GetThreadGroupAffinity_t)(HANDLE, GROUP_AFFINITY *); typedef BOOL (*kmp_GetThreadGroupAffinity_t)(HANDLE, GROUP_AFFINITY *);
extern kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity; extern kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity;
typedef BOOL (*kmp_SetThreadGroupAffinity_t)(HANDLE, const GROUP_AFFINITY *, GROUP_AFFINITY *); typedef BOOL (*kmp_SetThreadGroupAffinity_t)(HANDLE, const GROUP_AFFINITY *, GROUP_AFFINITY *);
extern kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity; extern kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity;
# endif /* KMP_GROUP_AFFINITY */ # endif /* KMP_OS_WINDOWS */
extern size_t __kmp_affin_mask_size;
# define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0)
# define KMP_AFFINITY_DISABLE() (__kmp_affin_mask_size = 0)
# define KMP_AFFINITY_ENABLE(mask_size) (__kmp_affin_mask_size = mask_size)
# if !KMP_USE_HWLOC
# define KMP_CPU_SETSIZE (__kmp_affin_mask_size * CHAR_BIT)
# define KMP_CPU_SET_ITERATE(i,mask) \
for(i = 0; (size_t)i < KMP_CPU_SETSIZE; ++i)
# endif
#if KMP_USE_HWLOC # if KMP_USE_HWLOC
extern hwloc_topology_t __kmp_hwloc_topology; extern hwloc_topology_t __kmp_hwloc_topology;
extern int __kmp_hwloc_error; extern int __kmp_hwloc_error;
typedef hwloc_cpuset_t kmp_affin_mask_t; # endif
# define KMP_CPU_SET(i,mask) hwloc_bitmap_set((hwloc_cpuset_t)mask, (unsigned)i)
# define KMP_CPU_ISSET(i,mask) hwloc_bitmap_isset((hwloc_cpuset_t)mask, (unsigned)i)
# define KMP_CPU_CLR(i,mask) hwloc_bitmap_clr((hwloc_cpuset_t)mask, (unsigned)i)
# define KMP_CPU_ZERO(mask) hwloc_bitmap_zero((hwloc_cpuset_t)mask)
# define KMP_CPU_COPY(dest, src) hwloc_bitmap_copy((hwloc_cpuset_t)dest, (hwloc_cpuset_t)src)
# define KMP_CPU_AND(dest, src) hwloc_bitmap_and((hwloc_cpuset_t)dest, (hwloc_cpuset_t)dest, (hwloc_cpuset_t)src)
# define KMP_CPU_COMPLEMENT(max_bit_number, mask) \
{ \
unsigned i; \
for(i=0;i<(unsigned)max_bit_number+1;i++) { \
if(hwloc_bitmap_isset((hwloc_cpuset_t)mask, i)) { \
hwloc_bitmap_clr((hwloc_cpuset_t)mask, i); \
} else { \
hwloc_bitmap_set((hwloc_cpuset_t)mask, i); \
} \
} \
hwloc_bitmap_and((hwloc_cpuset_t)mask, (hwloc_cpuset_t)mask, \
(hwloc_cpuset_t)__kmp_affin_fullMask); \
} \
# define KMP_CPU_UNION(dest, src) hwloc_bitmap_or((hwloc_cpuset_t)dest, (hwloc_cpuset_t)dest, (hwloc_cpuset_t)src) extern size_t __kmp_affin_mask_size;
# define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0)
# define KMP_AFFINITY_DISABLE() (__kmp_affin_mask_size = 0)
# define KMP_AFFINITY_ENABLE(mask_size) (__kmp_affin_mask_size = mask_size)
# define KMP_CPU_SET_ITERATE(i,mask) \ # define KMP_CPU_SET_ITERATE(i,mask) \
for(i = hwloc_bitmap_first((hwloc_cpuset_t)mask); (int)i != -1; i = hwloc_bitmap_next((hwloc_cpuset_t)mask, i)) for (i = (mask)->begin(); i != (mask)->end() ; i = (mask)->next(i))
# define KMP_CPU_SET(i,mask) (mask)->set(i)
# define KMP_CPU_ALLOC(ptr) ptr = (kmp_affin_mask_t*)hwloc_bitmap_alloc() # define KMP_CPU_ISSET(i,mask) (mask)->is_set(i)
# define KMP_CPU_FREE(ptr) hwloc_bitmap_free((hwloc_bitmap_t)ptr); # define KMP_CPU_CLR(i,mask) (mask)->clear(i)
# define KMP_CPU_ZERO(mask) (mask)->zero()
# define KMP_CPU_COPY(dest, src) (dest)->copy(src)
# define KMP_CPU_AND(dest, src) (dest)->bitwise_and(src)
# define KMP_CPU_COMPLEMENT(max_bit_number, mask) (mask)->bitwise_not()
# define KMP_CPU_UNION(dest, src) (dest)->bitwise_or(src)
# define KMP_CPU_ALLOC(ptr) (ptr = __kmp_affinity_dispatch->allocate_mask())
# define KMP_CPU_FREE(ptr) __kmp_affinity_dispatch->deallocate_mask(ptr)
# define KMP_CPU_ALLOC_ON_STACK(ptr) KMP_CPU_ALLOC(ptr) # define KMP_CPU_ALLOC_ON_STACK(ptr) KMP_CPU_ALLOC(ptr)
# define KMP_CPU_FREE_FROM_STACK(ptr) KMP_CPU_FREE(ptr) # define KMP_CPU_FREE_FROM_STACK(ptr) KMP_CPU_FREE(ptr)
# define KMP_CPU_INTERNAL_ALLOC(ptr) KMP_CPU_ALLOC(ptr) # define KMP_CPU_INTERNAL_ALLOC(ptr) KMP_CPU_ALLOC(ptr)
# define KMP_CPU_INTERNAL_FREE(ptr) KMP_CPU_FREE(ptr) # define KMP_CPU_INTERNAL_FREE(ptr) KMP_CPU_FREE(ptr)
# define KMP_CPU_INDEX(arr,i) __kmp_affinity_dispatch->index_mask_array(arr, i)
// # define KMP_CPU_ALLOC_ARRAY(arr, n) (arr = __kmp_affinity_dispatch->allocate_mask_array(n))
// The following macro should be used to index an array of masks. # define KMP_CPU_FREE_ARRAY(arr, n) __kmp_affinity_dispatch->deallocate_mask_array(arr)
// The array should be declared as "kmp_affinity_t *" and allocated with # define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) KMP_CPU_ALLOC_ARRAY(arr, n)
// size "__kmp_affinity_mask_size * len". The macro takes care of the fact # define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) KMP_CPU_FREE_ARRAY(arr, n)
// that on Windows* OS, sizeof(kmp_affin_t) is really the size of the mask, but # define __kmp_get_system_affinity(mask, abort_bool) (mask)->get_system_affinity(abort_bool)
// on Linux* OS, sizeof(kmp_affin_t) is 1. # define __kmp_set_system_affinity(mask, abort_bool) (mask)->set_system_affinity(abort_bool)
// # define __kmp_get_proc_group(mask) (mask)->get_proc_group()
# define KMP_CPU_INDEX(array,i) ((kmp_affin_mask_t*)(array[i]))
# define KMP_CPU_ALLOC_ARRAY(arr, n) { \ class KMPAffinity {
arr = (kmp_affin_mask_t *)__kmp_allocate(n*sizeof(kmp_affin_mask_t)); \ public:
unsigned i; \ class Mask {
for(i=0;i<(unsigned)n;i++) { \ public:
arr[i] = hwloc_bitmap_alloc(); \ void* operator new(size_t n);
} \ void operator delete(void* p);
} void* operator new[](size_t n);
# define KMP_CPU_FREE_ARRAY(arr, n) { \ void operator delete[](void* p);
unsigned i; \ virtual ~Mask() {}
for(i=0;i<(unsigned)n;i++) { \ // Set bit i to 1
hwloc_bitmap_free(arr[i]); \ virtual void set(int i) {}
} \ // Return bit i
__kmp_free(arr); \ virtual bool is_set(int i) const { return false; }
} // Set bit i to 0
# define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) { \ virtual void clear(int i) {}
arr = (kmp_affin_mask_t *)KMP_INTERNAL_MALLOC(n*sizeof(kmp_affin_mask_t)); \ // Zero out entire mask
unsigned i; \ virtual void zero() {}
for(i=0;i<(unsigned)n;i++) { \ // Copy src into this mask
arr[i] = hwloc_bitmap_alloc(); \ virtual void copy(const Mask* src) {}
} \ // this &= rhs
} virtual void bitwise_and(const Mask* rhs) {}
# define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) { \ // this |= rhs
unsigned i; \ virtual void bitwise_or(const Mask* rhs) {}
for(i=0;i<(unsigned)n;i++) { \ // this = ~this
hwloc_bitmap_free(arr[i]); \ virtual void bitwise_not() {}
} \ // API for iterating over an affinity mask
KMP_INTERNAL_FREE(arr); \ // for (int i = mask->begin(); i != mask->end(); i = mask->next(i))
} virtual int begin() const { return 0; }
virtual int end() const { return 0; }
#else /* KMP_USE_HWLOC */ virtual int next(int previous) const { return 0; }
# if KMP_OS_LINUX // Set the system's affinity to this affinity mask's value
// virtual int set_system_affinity(bool abort_on_error) const { return -1; }
// On Linux* OS, the mask is actually a vector of length __kmp_affin_mask_size // Set this affinity mask to the current system affinity
// (in bytes). It should be allocated on a word boundary. virtual int get_system_affinity(bool abort_on_error) { return -1; }
// // Only 1 DWORD in the mask should have any procs set.
// WARNING!!! We have made the base type of the affinity mask unsigned char, // Return the appropriate index, or -1 for an invalid mask.
// in order to eliminate a lot of checks that the true system mask size is virtual int get_proc_group() const { return -1; }
// really a multiple of 4 bytes (on Linux* OS). };
// void* operator new(size_t n);
// THESE MACROS WON'T WORK PROPERLY ON BIG ENDIAN MACHINES!!! void operator delete(void* p);
// // Determine if affinity is capable
virtual void determine_capable(const char* env_var) {}
typedef unsigned char kmp_affin_mask_t; // Bind the current thread to os proc
virtual void bind_thread(int proc) {}
# define _KMP_CPU_SET(i,mask) (mask[i/CHAR_BIT] |= (((kmp_affin_mask_t)1) << (i % CHAR_BIT))) // Factory functions to allocate/deallocate a mask
# define KMP_CPU_SET(i,mask) _KMP_CPU_SET((i), ((kmp_affin_mask_t *)(mask))) virtual Mask* allocate_mask() { return nullptr; }
# define _KMP_CPU_ISSET(i,mask) (!!(mask[i/CHAR_BIT] & (((kmp_affin_mask_t)1) << (i % CHAR_BIT)))) virtual void deallocate_mask(Mask* m) { }
# define KMP_CPU_ISSET(i,mask) _KMP_CPU_ISSET((i), ((kmp_affin_mask_t *)(mask))) virtual Mask* allocate_mask_array(int num) { return nullptr; }
# define _KMP_CPU_CLR(i,mask) (mask[i/CHAR_BIT] &= ~(((kmp_affin_mask_t)1) << (i % CHAR_BIT))) virtual void deallocate_mask_array(Mask* m) { }
# define KMP_CPU_CLR(i,mask) _KMP_CPU_CLR((i), ((kmp_affin_mask_t *)(mask))) virtual Mask* index_mask_array(Mask* m, int index) { return nullptr; }
static void pick_api();
# define KMP_CPU_ZERO(mask) \ static void destroy_api();
{ \ enum api_type {
size_t __i; \ NATIVE_OS
for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \ #if KMP_USE_HWLOC
((kmp_affin_mask_t *)(mask))[__i] = 0; \ , HWLOC
} \
}
# define KMP_CPU_COPY(dest, src) \
{ \
size_t __i; \
for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \
((kmp_affin_mask_t *)(dest))[__i] \
= ((kmp_affin_mask_t *)(src))[__i]; \
} \
}
# define KMP_CPU_AND(dest, src) \
{ \
size_t __i; \
for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \
((kmp_affin_mask_t *)(dest))[__i] \
&= ((kmp_affin_mask_t *)(src))[__i]; \
} \
}
# define KMP_CPU_COMPLEMENT(max_bit_number, mask) \
{ \
size_t __i; \
for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \
((kmp_affin_mask_t *)(mask))[__i] \
= ~((kmp_affin_mask_t *)(mask))[__i]; \
} \
KMP_CPU_AND(mask, __kmp_affin_fullMask); \
}
# define KMP_CPU_UNION(dest, src) \
{ \
size_t __i; \
for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \
((kmp_affin_mask_t *)(dest))[__i] \
|= ((kmp_affin_mask_t *)(src))[__i]; \
} \
}
# endif /* KMP_OS_LINUX */
# if KMP_OS_WINDOWS
//
// On Windows* OS, the mask size is 4 bytes for IA-32 architecture, and on
// Intel(R) 64 it is 8 bytes times the number of processor groups.
//
# if KMP_GROUP_AFFINITY
typedef DWORD_PTR kmp_affin_mask_t;
# define _KMP_CPU_SET(i,mask) \
(mask[i/(CHAR_BIT * sizeof(kmp_affin_mask_t))] |= \
(((kmp_affin_mask_t)1) << (i % (CHAR_BIT * sizeof(kmp_affin_mask_t)))))
# define KMP_CPU_SET(i,mask) \
_KMP_CPU_SET((i), ((kmp_affin_mask_t *)(mask)))
# define _KMP_CPU_ISSET(i,mask) \
(!!(mask[i/(CHAR_BIT * sizeof(kmp_affin_mask_t))] & \
(((kmp_affin_mask_t)1) << (i % (CHAR_BIT * sizeof(kmp_affin_mask_t))))))
# define KMP_CPU_ISSET(i,mask) \
_KMP_CPU_ISSET((i), ((kmp_affin_mask_t *)(mask)))
# define _KMP_CPU_CLR(i,mask) \
(mask[i/(CHAR_BIT * sizeof(kmp_affin_mask_t))] &= \
~(((kmp_affin_mask_t)1) << (i % (CHAR_BIT * sizeof(kmp_affin_mask_t)))))
# define KMP_CPU_CLR(i,mask) \
_KMP_CPU_CLR((i), ((kmp_affin_mask_t *)(mask)))
# define KMP_CPU_ZERO(mask) \
{ \
int __i; \
for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \
((kmp_affin_mask_t *)(mask))[__i] = 0; \
} \
}
# define KMP_CPU_COPY(dest, src) \
{ \
int __i; \
for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \
((kmp_affin_mask_t *)(dest))[__i] \
= ((kmp_affin_mask_t *)(src))[__i]; \
} \
}
# define KMP_CPU_AND(dest, src) \
{ \
int __i; \
for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \
((kmp_affin_mask_t *)(dest))[__i] \
&= ((kmp_affin_mask_t *)(src))[__i]; \
} \
}
# define KMP_CPU_COMPLEMENT(max_bit_number, mask) \
{ \
int __i; \
for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \
((kmp_affin_mask_t *)(mask))[__i] \
= ~((kmp_affin_mask_t *)(mask))[__i]; \
} \
KMP_CPU_AND(mask, __kmp_affin_fullMask); \
}
# define KMP_CPU_UNION(dest, src) \
{ \
int __i; \
for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \
((kmp_affin_mask_t *)(dest))[__i] \
|= ((kmp_affin_mask_t *)(src))[__i]; \
} \
}
# else /* KMP_GROUP_AFFINITY */
typedef DWORD kmp_affin_mask_t; /* for compatibility with older winbase.h */
# define KMP_CPU_SET(i,mask) (*(mask) |= (((kmp_affin_mask_t)1) << (i)))
# define KMP_CPU_ISSET(i,mask) (!!(*(mask) & (((kmp_affin_mask_t)1) << (i))))
# define KMP_CPU_CLR(i,mask) (*(mask) &= ~(((kmp_affin_mask_t)1) << (i)))
# define KMP_CPU_ZERO(mask) (*(mask) = 0)
# define KMP_CPU_COPY(dest, src) (*(dest) = *(src))
# define KMP_CPU_AND(dest, src) (*(dest) &= *(src))
# define KMP_CPU_COMPLEMENT(max_bit_number, mask) (*(mask) = ~*(mask)); KMP_CPU_AND(mask, __kmp_affin_fullMask)
# define KMP_CPU_UNION(dest, src) (*(dest) |= *(src))
# endif /* KMP_GROUP_AFFINITY */
# endif /* KMP_OS_WINDOWS */
//
// __kmp_allocate() will return memory allocated on a 4-bytes boundary.
// after zeroing it - it takes care of those assumptions stated above.
//
# define KMP_CPU_ALLOC(ptr) \
(ptr = ((kmp_affin_mask_t *)__kmp_allocate(__kmp_affin_mask_size)))
# define KMP_CPU_FREE(ptr) __kmp_free(ptr)
# define KMP_CPU_ALLOC_ON_STACK(ptr) (ptr = ((kmp_affin_mask_t *)KMP_ALLOCA(__kmp_affin_mask_size)))
# define KMP_CPU_FREE_FROM_STACK(ptr) /* Nothing */
# define KMP_CPU_INTERNAL_ALLOC(ptr) (ptr = ((kmp_affin_mask_t *)KMP_INTERNAL_MALLOC(__kmp_affin_mask_size)))
# define KMP_CPU_INTERNAL_FREE(ptr) KMP_INTERNAL_FREE(ptr)
//
// The following macro should be used to index an array of masks.
// The array should be declared as "kmp_affinity_t *" and allocated with
// size "__kmp_affinity_mask_size * len". The macro takes care of the fact
// that on Windows* OS, sizeof(kmp_affin_t) is really the size of the mask, but
// on Linux* OS, sizeof(kmp_affin_t) is 1.
//
# define KMP_CPU_INDEX(array,i) \
((kmp_affin_mask_t *)(((char *)(array)) + (i) * __kmp_affin_mask_size))
# define KMP_CPU_ALLOC_ARRAY(arr, n) arr = (kmp_affin_mask_t *)__kmp_allocate(n * __kmp_affin_mask_size)
# define KMP_CPU_FREE_ARRAY(arr, n) __kmp_free(arr);
# define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) arr = (kmp_affin_mask_t *)KMP_INTERNAL_MALLOC(n * __kmp_affin_mask_size)
# define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) KMP_INTERNAL_FREE(arr);
#endif /* KMP_USE_HWLOC */
// prototype after typedef of kmp_affin_mask_t
#if KMP_GROUP_AFFINITY
extern int __kmp_get_proc_group(kmp_affin_mask_t const *mask);
#endif #endif
};
virtual api_type get_api_type() const { KMP_ASSERT(0); return NATIVE_OS; };
private:
static bool picked_api;
};
typedef KMPAffinity::Mask kmp_affin_mask_t;
extern KMPAffinity* __kmp_affinity_dispatch;
// //
// Declare local char buffers with this size for printing debug and info // Declare local char buffers with this size for printing debug and info
// messages, using __kmp_affinity_print_mask(). // messages, using __kmp_affinity_print_mask().
// //
#define KMP_AFFIN_MASK_PRINT_LEN 1024 #define KMP_AFFIN_MASK_PRINT_LEN 1024
enum affinity_type { enum affinity_type {
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines
extern int __kmp_affinity_compact; /* Affinity 'compact' value */ extern int __kmp_affinity_compact; /* Affinity 'compact' value */
extern int __kmp_affinity_offset; /* Affinity offset value */ extern int __kmp_affinity_offset; /* Affinity offset value */
extern int __kmp_affinity_verbose; /* Was verbose specified for KMP_AFFINITY? */ extern int __kmp_affinity_verbose; /* Was verbose specified for KMP_AFFINITY? */
extern int __kmp_affinity_warnings; /* KMP_AFFINITY warnings enabled ? */ extern int __kmp_affinity_warnings; /* KMP_AFFINITY warnings enabled ? */
extern int __kmp_affinity_respect_mask; /* Respect process' initial affinity mask? */ extern int __kmp_affinity_respect_mask; /* Respect process' initial affinity mask? */
extern char * __kmp_affinity_proclist; /* proc ID list */ extern char * __kmp_affinity_proclist; /* proc ID list */
extern kmp_affin_mask_t *__kmp_affinity_masks; extern kmp_affin_mask_t *__kmp_affinity_masks;
extern unsigned __kmp_affinity_num_masks; extern unsigned __kmp_affinity_num_masks;
extern int __kmp_get_system_affinity(kmp_affin_mask_t *mask, int abort_on_error);
extern int __kmp_set_system_affinity(kmp_affin_mask_t const *mask, int abort_on_error);
extern void __kmp_affinity_bind_thread(int which); extern void __kmp_affinity_bind_thread(int which);
extern kmp_affin_mask_t *__kmp_affin_fullMask; extern kmp_affin_mask_t *__kmp_affin_fullMask;
extern char const * __kmp_cpuinfo_file; extern char const * __kmp_cpuinfo_file;
#endif /* KMP_AFFINITY_SUPPORTED */ #endif /* KMP_AFFINITY_SUPPORTED */
#if OMP_40_ENABLED #if OMP_40_ENABLED
▲ Show 20 Lines • Show All 1,693 Lines • ▼ Show 20 Lines#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
kmp_r_sched_t t_sched; // run-time schedule for the team kmp_r_sched_t t_sched; // run-time schedule for the team
#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
int t_first_place; // first & last place in parent thread's partition. int t_first_place; // first & last place in parent thread's partition.
int t_last_place; // Restore these values to master after par region. int t_last_place; // Restore these values to master after par region.
#endif // OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED #endif // OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via omp_set_num_threads() call int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via omp_set_num_threads() call
// Read/write by workers as well ----------------------------------------------------------------------- // Read/write by workers as well -----------------------------------------------------------------------
#if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && !KMP_USE_HWLOC #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
// Using CACHE_LINE=64 reduces memory footprint, but causes a big perf regression of epcc 'parallel' // Using CACHE_LINE=64 reduces memory footprint, but causes a big perf regression of epcc 'parallel'
// and 'barrier' on fxe256lin01. This extra padding serves to fix the performance of epcc 'parallel' // and 'barrier' on fxe256lin01. This extra padding serves to fix the performance of epcc 'parallel'
// and 'barrier' when CACHE_LINE=64. TODO: investigate more and get rid if this padding. // and 'barrier' when CACHE_LINE=64. TODO: investigate more and get rid if this padding.
char dummy_padding[1024]; char dummy_padding[1024];
#endif #endif
KMP_ALIGN_CACHE kmp_internal_control_t *t_control_stack_top; // internal control stack for additional nested teams. KMP_ALIGN_CACHE kmp_internal_control_t *t_control_stack_top; // internal control stack for additional nested teams.
// for SERIALIZED teams nested 2 or more levels deep // for SERIALIZED teams nested 2 or more levels deep
#if OMP_40_ENABLED #if OMP_40_ENABLED
▲ Show 20 Lines • Show All 1,101 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/kmp_affinity.h

Show All 9 Lines
// This file is dual licensed under the MIT and the University of Illinois Open // This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details. // Source Licenses. See LICENSE.txt for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#ifndef KMP_AFFINITY_H #ifndef KMP_AFFINITY_H
#define KMP_AFFINITY_H #define KMP_AFFINITY_H
#include "kmp_os.h"
#include "kmp.h"
#if KMP_AFFINITY_SUPPORTED
#if KMP_USE_HWLOC
class KMPHwlocAffinity: public KMPAffinity {
public:
class Mask : public KMPAffinity::Mask {
hwloc_cpuset_t mask;
public:
Mask() { mask = hwloc_bitmap_alloc(); this->zero(); }
~Mask() { hwloc_bitmap_free(mask); }
void set(int i) override { hwloc_bitmap_set(mask, i); }
bool is_set(int i) const override { return hwloc_bitmap_isset(mask, i); }
void clear(int i) override { hwloc_bitmap_clr(mask, i); }
void zero() override { hwloc_bitmap_zero(mask); }
void copy(const KMPAffinity::Mask* src) override {
const Mask* convert = static_cast<const Mask*>(src);
hwloc_bitmap_copy(mask, convert->mask);
}
void bitwise_and(const KMPAffinity::Mask* rhs) override {
const Mask* convert = static_cast<const Mask*>(rhs);
hwloc_bitmap_and(mask, mask, convert->mask);
}
void bitwise_or(const KMPAffinity::Mask * rhs) override {
const Mask* convert = static_cast<const Mask*>(rhs);
hwloc_bitmap_or(mask, mask, convert->mask);
}
void bitwise_not() override { hwloc_bitmap_not(mask, mask); }
int begin() const override { return hwloc_bitmap_first(mask); }
int end() const override { return -1; }
int next(int previous) const override { return hwloc_bitmap_next(mask, previous); }
int get_system_affinity(bool abort_on_error) override {
KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
"Illegal get affinity operation when not capable");
int retval = hwloc_get_cpubind(__kmp_hwloc_topology, mask, HWLOC_CPUBIND_THREAD);
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG( FatalSysError ), KMP_ERR( error ), __kmp_msg_null);
}
return error;
}
int set_system_affinity(bool abort_on_error) const override {
KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
"Illegal get affinity operation when not capable");
int retval = hwloc_set_cpubind(__kmp_hwloc_topology, mask, HWLOC_CPUBIND_THREAD);
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG( FatalSysError ), KMP_ERR( error ), __kmp_msg_null);
}
return error;
}
int get_proc_group() const override {
int i;
int group = -1;
# if KMP_OS_WINDOWS
if (__kmp_num_proc_groups == 1) {
return 1;
}
for (i = 0; i < __kmp_num_proc_groups; i++) {
// On windows, the long type is always 32 bits
unsigned long first_32_bits = hwloc_bitmap_to_ith_ulong(mask, i*2);
unsigned long second_32_bits = hwloc_bitmap_to_ith_ulong(mask, i*2+1);
if (first_32_bits == 0 && second_32_bits == 0) {
continue;
}
if (group >= 0) {
return -1;
}
group = i;
}
# endif /* KMP_OS_WINDOWS */
return group;
}
};
void determine_capable(const char* var) override {
const hwloc_topology_support* topology_support;
if(__kmp_hwloc_topology == NULL) {
if(hwloc_topology_init(&__kmp_hwloc_topology) < 0) {
__kmp_hwloc_error = TRUE;
if(__kmp_affinity_verbose)
KMP_WARNING(AffHwlocErrorOccurred, var, "hwloc_topology_init()");
}
if(hwloc_topology_load(__kmp_hwloc_topology) < 0) {
__kmp_hwloc_error = TRUE;
if(__kmp_affinity_verbose)
KMP_WARNING(AffHwlocErrorOccurred, var, "hwloc_topology_load()");
}
}
topology_support = hwloc_topology_get_support(__kmp_hwloc_topology);
// Is the system capable of setting/getting this thread's affinity?
// also, is topology discovery possible? (pu indicates ability to discover processing units)
// and finally, were there no errors when calling any hwloc_* API functions?
if(topology_support && topology_support->cpubind->set_thisthread_cpubind &&
topology_support->cpubind->get_thisthread_cpubind &&
topology_support->discovery->pu &&
!__kmp_hwloc_error)
{
// enables affinity according to KMP_AFFINITY_CAPABLE() macro
KMP_AFFINITY_ENABLE(TRUE);
} else {
// indicate that hwloc didn't work and disable affinity
__kmp_hwloc_error = TRUE;
KMP_AFFINITY_DISABLE();
}
}
void bind_thread(int which) override {
KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
"Illegal set affinity operation when not capable");
KMPAffinity::Mask *mask;
KMP_CPU_ALLOC_ON_STACK(mask);
KMP_CPU_ZERO(mask);
KMP_CPU_SET(which, mask);
__kmp_set_system_affinity(mask, TRUE);
KMP_CPU_FREE_FROM_STACK(mask);
}
KMPAffinity::Mask* allocate_mask() override { return new Mask(); }
void deallocate_mask(KMPAffinity::Mask* m) override { delete m; }
KMPAffinity::Mask* allocate_mask_array(int num) override { return new Mask[num]; }
void deallocate_mask_array(KMPAffinity::Mask* array) override {
Mask* hwloc_array = static_cast<Mask*>(array);
delete[] hwloc_array;
}
KMPAffinity::Mask* index_mask_array(KMPAffinity::Mask* array, int index) override {
Mask* hwloc_array = static_cast<Mask*>(array);
return &(hwloc_array[index]);
}
api_type get_api_type() const override { return HWLOC; }
};
#endif /* KMP_USE_HWLOC */
#if KMP_OS_LINUX
/*
* On some of the older OS's that we build on, these constants aren't present
* in <asm/unistd.h> #included from <sys.syscall.h>. They must be the same on
* all systems of the same arch where they are defined, and they cannot change.
* stone forever.
*/
#include <sys/syscall.h>
# if KMP_ARCH_X86 || KMP_ARCH_ARM
# ifndef __NR_sched_setaffinity
# define __NR_sched_setaffinity 241
# elif __NR_sched_setaffinity != 241
# error Wrong code for setaffinity system call.
# endif /* __NR_sched_setaffinity */
# ifndef __NR_sched_getaffinity
# define __NR_sched_getaffinity 242
# elif __NR_sched_getaffinity != 242
# error Wrong code for getaffinity system call.
# endif /* __NR_sched_getaffinity */
# elif KMP_ARCH_AARCH64
# ifndef __NR_sched_setaffinity
# define __NR_sched_setaffinity 122
# elif __NR_sched_setaffinity != 122
# error Wrong code for setaffinity system call.
# endif /* __NR_sched_setaffinity */
# ifndef __NR_sched_getaffinity
# define __NR_sched_getaffinity 123
# elif __NR_sched_getaffinity != 123
# error Wrong code for getaffinity system call.
# endif /* __NR_sched_getaffinity */
# elif KMP_ARCH_X86_64
# ifndef __NR_sched_setaffinity
# define __NR_sched_setaffinity 203
# elif __NR_sched_setaffinity != 203
# error Wrong code for setaffinity system call.
# endif /* __NR_sched_setaffinity */
# ifndef __NR_sched_getaffinity
# define __NR_sched_getaffinity 204
# elif __NR_sched_getaffinity != 204
# error Wrong code for getaffinity system call.
# endif /* __NR_sched_getaffinity */
# elif KMP_ARCH_PPC64
# ifndef __NR_sched_setaffinity
# define __NR_sched_setaffinity 222
# elif __NR_sched_setaffinity != 222
# error Wrong code for setaffinity system call.
# endif /* __NR_sched_setaffinity */
# ifndef __NR_sched_getaffinity
# define __NR_sched_getaffinity 223
# elif __NR_sched_getaffinity != 223
# error Wrong code for getaffinity system call.
# endif /* __NR_sched_getaffinity */
# else
# error Unknown or unsupported architecture
# endif /* KMP_ARCH_* */
class KMPNativeAffinity : public KMPAffinity {
class Mask : public KMPAffinity::Mask {
typedef unsigned char mask_t;
static const int BITS_PER_MASK_T = sizeof(mask_t)*CHAR_BIT;
public:
mask_t* mask;
Mask() { mask = (mask_t*)__kmp_allocate(__kmp_affin_mask_size); }
~Mask() { if (mask) __kmp_free(mask); }
void set(int i) override { mask[i/BITS_PER_MASK_T] |= ((mask_t)1 << (i % BITS_PER_MASK_T)); }
bool is_set(int i) const override { return (mask[i/BITS_PER_MASK_T] & ((mask_t)1 << (i % BITS_PER_MASK_T))); }
void clear(int i) override { mask[i/BITS_PER_MASK_T] &= ~((mask_t)1 << (i % BITS_PER_MASK_T)); }
void zero() override {
for (size_t i=0; i<__kmp_affin_mask_size; ++i)
mask[i] = 0;
}
void copy(const KMPAffinity::Mask* src) override {
const Mask * convert = static_cast<const Mask*>(src);
for (size_t i=0; i<__kmp_affin_mask_size; ++i)
mask[i] = convert->mask[i];
}
void bitwise_and(const KMPAffinity::Mask* rhs) override {
const Mask * convert = static_cast<const Mask*>(rhs);
for (size_t i=0; i<__kmp_affin_mask_size; ++i)
mask[i] &= convert->mask[i];
}
void bitwise_or(const KMPAffinity::Mask* rhs) override {
const Mask * convert = static_cast<const Mask*>(rhs);
for (size_t i=0; i<__kmp_affin_mask_size; ++i)
mask[i] |= convert->mask[i];
}
void bitwise_not() override {
for (size_t i=0; i<__kmp_affin_mask_size; ++i)
mask[i] = ~(mask[i]);
}
int begin() const override {
int retval = 0;
while (retval < end() && !is_set(retval))
++retval;
return retval;
}
int end() const override { return __kmp_affin_mask_size*BITS_PER_MASK_T; }
int next(int previous) const override {
int retval = previous+1;
while (retval < end() && !is_set(retval))
++retval;
return retval;
}
int get_system_affinity(bool abort_on_error) override {
KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
"Illegal get affinity operation when not capable");
int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG( FatalSysError ), KMP_ERR( error ), __kmp_msg_null);
}
return error;
}
int set_system_affinity(bool abort_on_error) const override {
KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
"Illegal get affinity operation when not capable");
int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG( FatalSysError ), KMP_ERR( error ), __kmp_msg_null);
}
return error;
}
};
void determine_capable(const char* env_var) override {
__kmp_affinity_determine_capable(env_var);
}
void bind_thread(int which) override {
__kmp_affinity_bind_thread(which);
}
KMPAffinity::Mask* allocate_mask() override {
KMPNativeAffinity::Mask* retval = new Mask();
return retval;
}
void deallocate_mask(KMPAffinity::Mask* m) override {
KMPNativeAffinity::Mask* native_mask = static_cast<KMPNativeAffinity::Mask*>(m);
delete m;
}
KMPAffinity::Mask* allocate_mask_array(int num) override { return new Mask[num]; }
void deallocate_mask_array(KMPAffinity::Mask* array) override {
Mask* linux_array = static_cast<Mask*>(array);
delete[] linux_array;
}
KMPAffinity::Mask* index_mask_array(KMPAffinity::Mask* array, int index) override {
Mask* linux_array = static_cast<Mask*>(array);
return &(linux_array[index]);
}
api_type get_api_type() const override { return NATIVE_OS; }
};
#endif /* KMP_OS_LINUX */
#if KMP_OS_WINDOWS
class KMPNativeAffinity : public KMPAffinity {
class Mask : public KMPAffinity::Mask {
typedef ULONG_PTR mask_t;
static const int BITS_PER_MASK_T = sizeof(mask_t)*CHAR_BIT;
mask_t* mask;
public:
Mask() { mask = (mask_t*)__kmp_allocate(sizeof(mask_t)*__kmp_num_proc_groups); }
~Mask() { if (mask) __kmp_free(mask); }
void set(int i) override { mask[i/BITS_PER_MASK_T] |= ((mask_t)1 << (i % BITS_PER_MASK_T)); }
bool is_set(int i) const override { return (mask[i/BITS_PER_MASK_T] & ((mask_t)1 << (i % BITS_PER_MASK_T))); }
void clear(int i) override { mask[i/BITS_PER_MASK_T] &= ~((mask_t)1 << (i % BITS_PER_MASK_T)); }
void zero() override {
for (size_t i=0; i<__kmp_num_proc_groups; ++i)
mask[i] = 0;
}
void copy(const KMPAffinity::Mask* src) override {
const Mask * convert = static_cast<const Mask*>(src);
for (size_t i=0; i<__kmp_num_proc_groups; ++i)
mask[i] = convert->mask[i];
}
void bitwise_and(const KMPAffinity::Mask* rhs) override {
const Mask * convert = static_cast<const Mask*>(rhs);
for (size_t i=0; i<__kmp_num_proc_groups; ++i)
mask[i] &= convert->mask[i];
}
void bitwise_or(const KMPAffinity::Mask* rhs) override {
const Mask * convert = static_cast<const Mask*>(rhs);
for (size_t i=0; i<__kmp_num_proc_groups; ++i)
mask[i] |= convert->mask[i];
}
void bitwise_not() override {
for (size_t i=0; i<__kmp_num_proc_groups; ++i)
mask[i] = ~(mask[i]);
}
int begin() const override {
int retval = 0;
while (retval < end() && !is_set(retval))
++retval;
return retval;
}
int end() const override { return __kmp_num_proc_groups*BITS_PER_MASK_T; }
int next(int previous) const override {
int retval = previous+1;
while (retval < end() && !is_set(retval))
++retval;
return retval;
}
int set_system_affinity(bool abort_on_error) const override {
if (__kmp_num_proc_groups > 1) {
// Check for a valid mask.
GROUP_AFFINITY ga;
int group = get_proc_group();
if (group < 0) {
if (abort_on_error) {
KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
}
return -1;
}
// Transform the bit vector into a GROUP_AFFINITY struct
// and make the system call to set affinity.
ga.Group = group;
ga.Mask = mask[group];
ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetThreadAffMask ),
KMP_ERR( error ), __kmp_msg_null);
}
return error;
}
} else {
if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetThreadAffMask ),
KMP_ERR( error ), __kmp_msg_null);
}
return error;
}
}
return 0;
}
int get_system_affinity(bool abort_on_error) override {
if (__kmp_num_proc_groups > 1) {
this->zero();
GROUP_AFFINITY ga;
KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);
if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "GetThreadGroupAffinity()"),
KMP_ERR(error), __kmp_msg_null);
}
return error;
}
if ((ga.Group < 0) || (ga.Group > __kmp_num_proc_groups) || (ga.Mask == 0)) {
return -1;
}
mask[ga.Group] = ga.Mask;
} else {
mask_t newMask, sysMask, retval;
if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "GetProcessAffinityMask()"),
KMP_ERR(error), __kmp_msg_null);
}
return error;
}
retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
if (! retval) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
KMP_ERR(error), __kmp_msg_null);
}
return error;
}
newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
if (! newMask) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
KMP_ERR(error), __kmp_msg_null);
}
}
*mask = retval;
}
return 0;
}
int get_proc_group() const override {
int group = -1;
if (__kmp_num_proc_groups == 1) {
return 1;
}
for (int i = 0; i < __kmp_num_proc_groups; i++) {
if (mask[i] == 0)
continue;
if (group >= 0)
return -1;
group = i;
}
return group;
}
};
void determine_capable(const char* env_var) override {
__kmp_affinity_determine_capable(env_var);
}
void bind_thread(int which) override {
__kmp_affinity_bind_thread(which);
}
KMPAffinity::Mask* allocate_mask() override { return new Mask(); }
void deallocate_mask(KMPAffinity::Mask* m) override { delete m; }
KMPAffinity::Mask* allocate_mask_array(int num) override { return new Mask[num]; }
void deallocate_mask_array(KMPAffinity::Mask* array) override {
Mask* windows_array = static_cast<Mask*>(array);
delete[] windows_array;
}
KMPAffinity::Mask* index_mask_array(KMPAffinity::Mask* array, int index) override {
Mask* windows_array = static_cast<Mask*>(array);
return &(windows_array[index]);
}
api_type get_api_type() const override { return NATIVE_OS; }
};
#endif /* KMP_OS_WINDOWS */
#endif /* KMP_AFFINITY_SUPPORTED */
class Address { class Address {
public: public:
static const unsigned maxDepth = 32; static const unsigned maxDepth = 32;
unsigned labels[maxDepth]; unsigned labels[maxDepth];
unsigned childNums[maxDepth]; unsigned childNums[maxDepth];
unsigned depth; unsigned depth;
unsigned leader; unsigned leader;
Address(unsigned _depth) Address(unsigned _depth)
▲ Show 20 Lines • Show All 266 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/kmp_affinity.cpp

Show First 20 Lines • Show All 41 Lines • ▼ Show 20 Linesvoid __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar) {
thr_bar->depth = depth; thr_bar->depth = depth;
thr_bar->base_leaf_kids = (kmp_uint8)machine_hierarchy.numPerLevel[0]-1; thr_bar->base_leaf_kids = (kmp_uint8)machine_hierarchy.numPerLevel[0]-1;
thr_bar->skip_per_level = machine_hierarchy.skipPerLevel; thr_bar->skip_per_level = machine_hierarchy.skipPerLevel;
} }
#if KMP_AFFINITY_SUPPORTED #if KMP_AFFINITY_SUPPORTED
// bool KMPAffinity::picked_api = false;
// Print the affinity mask to the character array in a pretty format.
// void* KMPAffinity::Mask::operator new(size_t n) { return __kmp_allocate(n); }
void* KMPAffinity::Mask::operator new[](size_t n) { return __kmp_allocate(n); }
void KMPAffinity::Mask::operator delete(void* p) { __kmp_free(p); }
void KMPAffinity::Mask::operator delete[](void* p) { __kmp_free(p); }
void* KMPAffinity::operator new(size_t n) { return __kmp_allocate(n); }
void KMPAffinity::operator delete(void* p) { __kmp_free(p); }
void KMPAffinity::pick_api() {
KMPAffinity* affinity_dispatch;
if (picked_api)
return;
#if KMP_USE_HWLOC #if KMP_USE_HWLOC
char * if (__kmp_affinity_top_method == affinity_top_method_hwloc) {
__kmp_affinity_print_mask(char *buf, int buf_len, kmp_affin_mask_t *mask) affinity_dispatch = new KMPHwlocAffinity();
} else
#endif
{ {
int num_chars_to_write, num_chars_written; affinity_dispatch = new KMPNativeAffinity();
char* scan; }
KMP_ASSERT(buf_len >= 40); __kmp_affinity_dispatch = affinity_dispatch;
picked_api = true;
// bufsize of 0 just retrieves the needed buffer size. }
num_chars_to_write = hwloc_bitmap_list_snprintf(buf, 0, (hwloc_bitmap_t)mask);
// need '{', "xxxxxxxx...xx", '}', '\0' = num_chars_to_write + 3 bytes void KMPAffinity::destroy_api() {
// * num_chars_to_write returned by hwloc_bitmap_list_snprintf does not if (__kmp_affinity_dispatch != NULL) {
// take into account the '\0' character. delete __kmp_affinity_dispatch;
if(hwloc_bitmap_iszero((hwloc_bitmap_t)mask)) { __kmp_affinity_dispatch = NULL;
KMP_SNPRINTF(buf, buf_len, "{<empty>}"); picked_api = false;
} else if(num_chars_to_write < buf_len - 3) {
// no problem fitting the mask into buf_len number of characters
buf[0] = '{';
// use buf_len-3 because we have the three characters: '{' '}' '\0' to add to the buffer
num_chars_written = hwloc_bitmap_list_snprintf(buf+1, buf_len-3, (hwloc_bitmap_t)mask);
buf[num_chars_written+1] = '}';
buf[num_chars_written+2] = '\0';
} else {
// Need to truncate the affinity mask string and add ellipsis.
// To do this, we first write out the '{' + str(mask)
buf[0] = '{';
hwloc_bitmap_list_snprintf(buf+1, buf_len-1, (hwloc_bitmap_t)mask);
// then, what we do here is go to the 7th to last character, then go backwards until we are NOT
// on a digit then write "...}\0". This way it is a clean ellipsis addition and we don't
// overwrite part of an affinity number. i.e., we avoid something like { 45, 67, 8...} and get
// { 45, 67,...} instead.
scan = buf + buf_len - 7;
while(*scan >= '0' && *scan <= '9' && scan >= buf)
scan--;
*(scan+1) = '.';
*(scan+2) = '.';
*(scan+3) = '.';
*(scan+4) = '}';
*(scan+5) = '\0';
} }
return buf;
} }
#else
//
// Print the affinity mask to the character array in a pretty format.
//
char * char *
__kmp_affinity_print_mask(char *buf, int buf_len, kmp_affin_mask_t *mask) __kmp_affinity_print_mask(char *buf, int buf_len, kmp_affin_mask_t *mask)
{ {
KMP_ASSERT(buf_len >= 40); KMP_ASSERT(buf_len >= 40);
char *scan = buf; char *scan = buf;
char *end = buf + buf_len - 1; char *end = buf + buf_len - 1;
// //
// Find first element / check for empty set. // Find first element / check for empty set.
// //
size_t i; size_t i;
for (i = 0; i < KMP_CPU_SETSIZE; i++) { i = mask->begin();
if (KMP_CPU_ISSET(i, mask)) { if (i == mask->end()) {
break;
}
}
if (i == KMP_CPU_SETSIZE) {
KMP_SNPRINTF(scan, end-scan+1, "{<empty>}"); KMP_SNPRINTF(scan, end-scan+1, "{<empty>}");
while (*scan != '\0') scan++; while (*scan != '\0') scan++;
KMP_ASSERT(scan <= end); KMP_ASSERT(scan <= end);
return buf; return buf;
} }
KMP_SNPRINTF(scan, end-scan+1, "{%ld", (long)i); KMP_SNPRINTF(scan, end-scan+1, "{%ld", (long)i);
while (*scan != '\0') scan++; while (*scan != '\0') scan++;
i++; i++;
for (; i < KMP_CPU_SETSIZE; i++) { for (; i != mask->end(); i = mask->next(i)) {
if (! KMP_CPU_ISSET(i, mask)) { if (! KMP_CPU_ISSET(i, mask)) {
continue; continue;
} }
// //
// Check for buffer overflow. A string of the form ",<n>" will have // Check for buffer overflow. A string of the form ",<n>" will have
// at most 10 characters, plus we want to leave room to print ",...}" // at most 10 characters, plus we want to leave room to print ",...}"
// if the set is too large to print for a total of 15 characters. // if the set is too large to print for a total of 15 characters.
// We already left room for '\0' in setting end. // We already left room for '\0' in setting end.
// //
if (end - scan < 15) { if (end - scan < 15) {
break; break;
} }
KMP_SNPRINTF(scan, end-scan+1, ",%-ld", (long)i); KMP_SNPRINTF(scan, end-scan+1, ",%-ld", (long)i);
while (*scan != '\0') scan++; while (*scan != '\0') scan++;
} }
if (i < KMP_CPU_SETSIZE) { if (i != mask->end()) {
KMP_SNPRINTF(scan, end-scan+1, ",..."); KMP_SNPRINTF(scan, end-scan+1, ",...");
while (*scan != '\0') scan++; while (*scan != '\0') scan++;
} }
KMP_SNPRINTF(scan, end-scan+1, "}"); KMP_SNPRINTF(scan, end-scan+1, "}");
while (*scan != '\0') scan++; while (*scan != '\0') scan++;
KMP_ASSERT(scan <= end); KMP_ASSERT(scan <= end);
return buf; return buf;
} }
#endif // KMP_USE_HWLOC
void void
__kmp_affinity_entire_machine_mask(kmp_affin_mask_t *mask) __kmp_affinity_entire_machine_mask(kmp_affin_mask_t *mask)
{ {
KMP_CPU_ZERO(mask); KMP_CPU_ZERO(mask);
# if KMP_GROUP_AFFINITY # if KMP_GROUP_AFFINITY
▲ Show 20 Lines • Show All 514 Lines • ▼ Show 20 Lines if (__kmp_affinity_verbose) {
KMP_INFORM(Uniform, "KMP_AFFINITY"); KMP_INFORM(Uniform, "KMP_AFFINITY");
KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg, KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
__kmp_nThreadsPerCore, __kmp_ncores); __kmp_nThreadsPerCore, __kmp_ncores);
} }
KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL); KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
__kmp_pu_os_idx = (int*)__kmp_allocate(sizeof(int) * __kmp_avail_proc); __kmp_pu_os_idx = (int*)__kmp_allocate(sizeof(int) * __kmp_avail_proc);
if (__kmp_affinity_type == affinity_none) { if (__kmp_affinity_type == affinity_none) {
int avail_ct = 0; int avail_ct = 0;
unsigned int i; int i;
KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) { KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
if (! KMP_CPU_ISSET(i, __kmp_affin_fullMask)) if (! KMP_CPU_ISSET(i, __kmp_affin_fullMask))
continue; continue;
__kmp_pu_os_idx[avail_ct++] = i; // suppose indices are flat __kmp_pu_os_idx[avail_ct++] = i; // suppose indices are flat
} }
return 0; return 0;
} }
▲ Show 20 Lines • Show All 337 Lines • ▼ Show 20 Lines KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
// //
// Skip this proc if it is not included in the machine model. // Skip this proc if it is not included in the machine model.
// //
if (! KMP_CPU_ISSET(i, __kmp_affin_fullMask)) { if (! KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
continue; continue;
} }
KMP_DEBUG_ASSERT((int)nApics < __kmp_avail_proc); KMP_DEBUG_ASSERT((int)nApics < __kmp_avail_proc);
__kmp_affinity_bind_thread(i); __kmp_affinity_dispatch->bind_thread(i);
threadInfo[nApics].osId = i; threadInfo[nApics].osId = i;
// //
// The apic id and max threads per pkg come from cpuid(1). // The apic id and max threads per pkg come from cpuid(1).
// //
__kmp_x86_cpuid(1, 0, &buf); __kmp_x86_cpuid(1, 0, &buf);
if (! (buf.edx >> 9) & 1) { if (! (buf.edx >> 9) & 1) {
__kmp_set_system_affinity(oldMask, TRUE); __kmp_set_system_affinity(oldMask, TRUE);
▲ Show 20 Lines • Show All 499 Lines • ▼ Show 20 Lines KMP_CPU_SET_ITERATE(proc, __kmp_affin_fullMask) {
// //
// Skip this proc if it is not included in the machine model. // Skip this proc if it is not included in the machine model.
// //
if (! KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) { if (! KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
continue; continue;
} }
KMP_DEBUG_ASSERT(nApics < __kmp_avail_proc); KMP_DEBUG_ASSERT(nApics < __kmp_avail_proc);
__kmp_affinity_bind_thread(proc); __kmp_affinity_dispatch->bind_thread(proc);
// //
// Extrach the labels for each level in the machine topology map // Extrach the labels for each level in the machine topology map
// from the Apic ID. // from the Apic ID.
// //
Address addr(depth); Address addr(depth);
int prev_shift = 0; int prev_shift = 0;
▲ Show 20 Lines • Show All 2,141 Lines • ▼ Show 20 Lines if (__kmp_affinity_top_method == affinity_top_method_all) {
// //
// In the default code path, errors are not fatal - we just try using // In the default code path, errors are not fatal - we just try using
// another method. We only emit a warning message if affinity is on, // another method. We only emit a warning message if affinity is on,
// or the verbose flag is set, an the nowarnings flag was not set. // or the verbose flag is set, an the nowarnings flag was not set.
// //
const char *file_name = NULL; const char *file_name = NULL;
int line = 0; int line = 0;
# if KMP_USE_HWLOC # if KMP_USE_HWLOC
if (depth < 0) { if (depth < 0 && __kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC) {
if (__kmp_affinity_verbose) { if (__kmp_affinity_verbose) {
KMP_INFORM(AffUsingHwloc, "KMP_AFFINITY"); KMP_INFORM(AffUsingHwloc, "KMP_AFFINITY");
} }
if(!__kmp_hwloc_error) { if(!__kmp_hwloc_error) {
depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id); depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id);
if (depth == 0) { if (depth == 0) {
KMP_EXIT_AFF_NONE; KMP_EXIT_AFF_NONE;
} else if(depth < 0 && __kmp_affinity_verbose) { } else if(depth < 0 && __kmp_affinity_verbose) {
▲ Show 20 Lines • Show All 225 Lines • ▼ Show 20 Lines else if (__kmp_affinity_top_method == affinity_top_method_flat) {
} }
// should not fail // should not fail
KMP_ASSERT(depth > 0); KMP_ASSERT(depth > 0);
KMP_ASSERT(address2os != NULL); KMP_ASSERT(address2os != NULL);
} }
# if KMP_USE_HWLOC # if KMP_USE_HWLOC
else if (__kmp_affinity_top_method == affinity_top_method_hwloc) { else if (__kmp_affinity_top_method == affinity_top_method_hwloc) {
KMP_ASSERT(__kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC);
if (__kmp_affinity_verbose) { if (__kmp_affinity_verbose) {
KMP_INFORM(AffUsingHwloc, "KMP_AFFINITY"); KMP_INFORM(AffUsingHwloc, "KMP_AFFINITY");
} }
depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id); depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id);
if (depth == 0) { if (depth == 0) {
KMP_EXIT_AFF_NONE; KMP_EXIT_AFF_NONE;
} }
} }
▲ Show 20 Lines • Show All 270 Lines • ▼ Show 20 Lines if( procarr != NULL ) {
procarr = NULL; procarr = NULL;
} }
# if KMP_USE_HWLOC # if KMP_USE_HWLOC
if (__kmp_hwloc_topology != NULL) { if (__kmp_hwloc_topology != NULL) {
hwloc_topology_destroy(__kmp_hwloc_topology); hwloc_topology_destroy(__kmp_hwloc_topology);
__kmp_hwloc_topology = NULL; __kmp_hwloc_topology = NULL;
} }
# endif # endif
KMPAffinity::destroy_api();
} }
void void
__kmp_affinity_set_init_mask(int gtid, int isa_root) __kmp_affinity_set_init_mask(int gtid, int isa_root)
{ {
if (! KMP_AFFINITY_CAPABLE()) { if (! KMP_AFFINITY_CAPABLE()) {
return; return;
▲ Show 20 Lines • Show All 688 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/kmp_ftn_cdecl.c

/* /*
* kmp_ftn_cdecl.c -- Fortran __cdecl linkage support for OpenMP. * kmp_ftn_cdecl.c -- Fortran __cdecl linkage support for OpenMP.
*/ */
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
// This file is dual licensed under the MIT and the University of Illinois Open // This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details. // Source Licenses. See LICENSE.txt for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "kmp.h" #include "kmp.h"
#include "kmp_affinity.h"
#if KMP_OS_WINDOWS #if KMP_OS_WINDOWS
# if defined KMP_WIN_CDECL || !defined KMP_DYNAMIC_LIB # if defined KMP_WIN_CDECL || !defined KMP_DYNAMIC_LIB
# define KMP_FTN_ENTRIES KMP_FTN_UPPER # define KMP_FTN_ENTRIES KMP_FTN_UPPER
# endif # endif
#elif KMP_OS_UNIX #elif KMP_OS_UNIX
# define KMP_FTN_ENTRIES KMP_FTN_PLAIN # define KMP_FTN_ENTRIES KMP_FTN_PLAIN
#endif #endif
Show All 11 Lines

openmp/trunk/runtime/src/kmp_ftn_entry.h

Show First 20 Lines • Show All 273 Lines • ▼ Show 20 Lines
FTN_CREATE_AFFINITY_MASK( void **mask ) FTN_CREATE_AFFINITY_MASK( void **mask )
{ {
#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
*mask = NULL; *mask = NULL;
#else #else
// //
// We really only NEED serial initialization here. // We really only NEED serial initialization here.
// //
kmp_affin_mask_t* mask_internals;
if ( ! TCR_4(__kmp_init_middle) ) { if ( ! TCR_4(__kmp_init_middle) ) {
__kmp_middle_initialize(); __kmp_middle_initialize();
} }
# if KMP_USE_HWLOC mask_internals = __kmp_affinity_dispatch->allocate_mask();
*mask = (hwloc_cpuset_t)hwloc_bitmap_alloc(); KMP_CPU_ZERO( mask_internals );
# else *mask = mask_internals;
*mask = kmpc_malloc( __kmp_affin_mask_size );
# endif
KMP_CPU_ZERO( (kmp_affin_mask_t *)(*mask) );
#endif #endif
} }
void FTN_STDCALL void FTN_STDCALL
FTN_DESTROY_AFFINITY_MASK( void **mask ) FTN_DESTROY_AFFINITY_MASK( void **mask )
{ {
#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
// Nothing // Nothing
#else #else
// //
// We really only NEED serial initialization here. // We really only NEED serial initialization here.
// //
kmp_affin_mask_t* mask_internals;
if ( ! TCR_4(__kmp_init_middle) ) { if ( ! TCR_4(__kmp_init_middle) ) {
__kmp_middle_initialize(); __kmp_middle_initialize();
} }
if ( __kmp_env_consistency_check ) { if ( __kmp_env_consistency_check ) {
if ( *mask == NULL ) { if ( *mask == NULL ) {
KMP_FATAL( AffinityInvalidMask, "kmp_destroy_affinity_mask" ); KMP_FATAL( AffinityInvalidMask, "kmp_destroy_affinity_mask" );
} }
} }
# if KMP_USE_HWLOC mask_internals = (kmp_affin_mask_t*)(*mask);
hwloc_bitmap_free((hwloc_cpuset_t)(*mask)); __kmp_affinity_dispatch->deallocate_mask(mask_internals);
# else
kmpc_free( *mask );
# endif
*mask = NULL; *mask = NULL;
#endif #endif
} }
int FTN_STDCALL int FTN_STDCALL
FTN_SET_AFFINITY_MASK_PROC( int KMP_DEREF proc, void **mask ) FTN_SET_AFFINITY_MASK_PROC( int KMP_DEREF proc, void **mask )
{ {
#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
▲ Show 20 Lines • Show All 1,173 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/kmp_ftn_extra.c

/* /*
* kmp_ftn_extra.c -- Fortran 'extra' linkage support for OpenMP. * kmp_ftn_extra.c -- Fortran 'extra' linkage support for OpenMP.
*/ */
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
// This file is dual licensed under the MIT and the University of Illinois Open // This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details. // Source Licenses. See LICENSE.txt for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "kmp.h" #include "kmp.h"
#include "kmp_affinity.h"
#if KMP_OS_WINDOWS #if KMP_OS_WINDOWS
# define KMP_FTN_ENTRIES KMP_FTN_PLAIN # define KMP_FTN_ENTRIES KMP_FTN_PLAIN
#elif KMP_OS_UNIX #elif KMP_OS_UNIX
# define KMP_FTN_ENTRIES KMP_FTN_APPEND # define KMP_FTN_ENTRIES KMP_FTN_APPEND
#endif #endif
// Note: This string is not printed when KMP_VERSION=1. // Note: This string is not printed when KMP_VERSION=1.
Show All 9 Lines

openmp/trunk/runtime/src/kmp_global.c

/* /*
* kmp_global.c -- KPTS global variables for runtime support library * kmp_global.c -- KPTS global variables for runtime support library
*/ */
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
// This file is dual licensed under the MIT and the University of Illinois Open // This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details. // Source Licenses. See LICENSE.txt for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "kmp.h" #include "kmp.h"
#include "kmp_affinity.h"
kmp_key_t __kmp_gtid_threadprivate_key; kmp_key_t __kmp_gtid_threadprivate_key;
#if KMP_ARCH_X86 || KMP_ARCH_X86_64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
kmp_cpuinfo_t __kmp_cpuinfo = { 0 }; // Not initialized kmp_cpuinfo_t __kmp_cpuinfo = { 0 }; // Not initialized
#endif #endif
#if KMP_STATS_ENABLED #if KMP_STATS_ENABLED
▲ Show 20 Lines • Show All 192 Lines • ▼ Show 20 Lines
#endif /* KMP_OS_LINUX */ #endif /* KMP_OS_LINUX */
#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS) #if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
enum mic_type __kmp_mic_type = non_mic; enum mic_type __kmp_mic_type = non_mic;
#endif #endif
#if KMP_AFFINITY_SUPPORTED #if KMP_AFFINITY_SUPPORTED
KMPAffinity* __kmp_affinity_dispatch = NULL;
# if KMP_USE_HWLOC # if KMP_USE_HWLOC
int __kmp_hwloc_error = FALSE; int __kmp_hwloc_error = FALSE;
hwloc_topology_t __kmp_hwloc_topology = NULL; hwloc_topology_t __kmp_hwloc_topology = NULL;
# endif # endif
# if KMP_OS_WINDOWS
# if KMP_GROUP_AFFINITY # if KMP_GROUP_AFFINITY
int __kmp_num_proc_groups = 1; int __kmp_num_proc_groups = 1;
# endif /* KMP_GROUP_AFFINITY */
kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount = NULL; kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount = NULL;
kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount = NULL; kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount = NULL;
kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity = NULL; kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity = NULL;
kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity = NULL; kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity = NULL;
# endif /* KMP_OS_WINDOWS */
# endif /* KMP_GROUP_AFFINITY */
size_t __kmp_affin_mask_size = 0; size_t __kmp_affin_mask_size = 0;
enum affinity_type __kmp_affinity_type = affinity_default; enum affinity_type __kmp_affinity_type = affinity_default;
enum affinity_gran __kmp_affinity_gran = affinity_gran_default; enum affinity_gran __kmp_affinity_gran = affinity_gran_default;
int __kmp_affinity_gran_levels = -1; int __kmp_affinity_gran_levels = -1;
int __kmp_affinity_dups = TRUE; int __kmp_affinity_dups = TRUE;
enum affinity_top_method __kmp_affinity_top_method = affinity_top_method_default; enum affinity_top_method __kmp_affinity_top_method = affinity_top_method_default;
int __kmp_affinity_compact = 0; int __kmp_affinity_compact = 0;
▲ Show 20 Lines • Show All 248 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/kmp_runtime.c

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show All 19 Lines
#include "kmp_itt.h" #include "kmp_itt.h"
#include "kmp_str.h" #include "kmp_str.h"
#include "kmp_settings.h" #include "kmp_settings.h"
#include "kmp_i18n.h" #include "kmp_i18n.h"
#include "kmp_io.h" #include "kmp_io.h"
#include "kmp_error.h" #include "kmp_error.h"
#include "kmp_stats.h" #include "kmp_stats.h"
#include "kmp_wait_release.h" #include "kmp_wait_release.h"
#include "kmp_affinity.h"
#if OMPT_SUPPORT #if OMPT_SUPPORT
#include "ompt-specific.h" #include "ompt-specific.h"
#endif #endif
/* these are temporary issues to be dealt with */ /* these are temporary issues to be dealt with */
#define KMP_USE_PRCTL 0 #define KMP_USE_PRCTL 0
▲ Show 20 Lines • Show All 7,644 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/kmp_settings.c

Show All 17 Lines
#include "kmp_environment.h" #include "kmp_environment.h"
#include "kmp_atomic.h" #include "kmp_atomic.h"
#include "kmp_itt.h" #include "kmp_itt.h"
#include "kmp_str.h" #include "kmp_str.h"
#include "kmp_settings.h" #include "kmp_settings.h"
#include "kmp_i18n.h" #include "kmp_i18n.h"
#include "kmp_lock.h" #include "kmp_lock.h"
#include "kmp_io.h" #include "kmp_io.h"
#include "kmp_affinity.h"
static int __kmp_env_toPrint( char const * name, int flag ); static int __kmp_env_toPrint( char const * name, int flag );
bool __kmp_env_format = 0; // 0 - old format; 1 - new format bool __kmp_env_format = 0; // 0 - old format; 1 - new format
// ------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------
// Helper string functions. Subject to move to kmp_str. // Helper string functions. Subject to move to kmp_str.
// ------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------
▲ Show 20 Lines • Show All 5,300 Lines • ▼ Show 20 Lines
#if KMP_AFFINITY_SUPPORTED #if KMP_AFFINITY_SUPPORTED
if ( ! TCR_4(__kmp_init_middle) ) { if ( ! TCR_4(__kmp_init_middle) ) {
// //
// Determine if the machine/OS is actually capable of supporting // Determine if the machine/OS is actually capable of supporting
// affinity. // affinity.
// //
const char *var = "KMP_AFFINITY"; const char *var = "KMP_AFFINITY";
# if KMP_USE_HWLOC KMPAffinity::pick_api();
if(__kmp_hwloc_topology == NULL) {
if(hwloc_topology_init(&__kmp_hwloc_topology) < 0) {
__kmp_hwloc_error = TRUE;
if(__kmp_affinity_verbose)
KMP_WARNING(AffHwlocErrorOccurred, var, "hwloc_topology_init()");
}
if(hwloc_topology_load(__kmp_hwloc_topology) < 0) {
__kmp_hwloc_error = TRUE;
if(__kmp_affinity_verbose)
KMP_WARNING(AffHwlocErrorOccurred, var, "hwloc_topology_load()");
}
}
# endif
if ( __kmp_affinity_type == affinity_disabled ) { if ( __kmp_affinity_type == affinity_disabled ) {
KMP_AFFINITY_DISABLE(); KMP_AFFINITY_DISABLE();
} }
else if ( ! KMP_AFFINITY_CAPABLE() ) { else if ( ! KMP_AFFINITY_CAPABLE() ) {
# if KMP_USE_HWLOC __kmp_affinity_dispatch->determine_capable(var);
const hwloc_topology_support* topology_support = hwloc_topology_get_support(__kmp_hwloc_topology);
// Is the system capable of setting/getting this thread's affinity?
// also, is topology discovery possible? (pu indicates ability to discover processing units)
// and finally, were there no errors when calling any hwloc_* API functions?
if(topology_support && topology_support->cpubind->set_thisthread_cpubind &&
topology_support->cpubind->get_thisthread_cpubind &&
topology_support->discovery->pu &&
!__kmp_hwloc_error)
{
// enables affinity according to KMP_AFFINITY_CAPABLE() macro
KMP_AFFINITY_ENABLE(TRUE);
} else {
// indicate that hwloc didn't work and disable affinity
__kmp_hwloc_error = TRUE;
KMP_AFFINITY_DISABLE();
}
# else
__kmp_affinity_determine_capable( var );
# endif // KMP_USE_HWLOC
if ( ! KMP_AFFINITY_CAPABLE() ) { if ( ! KMP_AFFINITY_CAPABLE() ) {
if ( __kmp_affinity_verbose || ( __kmp_affinity_warnings if ( __kmp_affinity_verbose || ( __kmp_affinity_warnings
&& ( __kmp_affinity_type != affinity_default ) && ( __kmp_affinity_type != affinity_default )
&& ( __kmp_affinity_type != affinity_none ) && ( __kmp_affinity_type != affinity_none )
&& ( __kmp_affinity_type != affinity_disabled ) ) ) { && ( __kmp_affinity_type != affinity_disabled ) ) ) {
KMP_WARNING( AffNotSupported, var ); KMP_WARNING( AffNotSupported, var );
} }
__kmp_affinity_type = affinity_disabled; __kmp_affinity_type = affinity_disabled;
▲ Show 20 Lines • Show All 275 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/z_Linux_util.c

Show All 16 Lines
#include "kmp_wrapper_getpid.h" #include "kmp_wrapper_getpid.h"
#include "kmp_itt.h" #include "kmp_itt.h"
#include "kmp_str.h" #include "kmp_str.h"
#include "kmp_i18n.h" #include "kmp_i18n.h"
#include "kmp_lock.h" #include "kmp_lock.h"
#include "kmp_io.h" #include "kmp_io.h"
#include "kmp_stats.h" #include "kmp_stats.h"
#include "kmp_wait_release.h" #include "kmp_wait_release.h"
#include "kmp_affinity.h"
#if !KMP_OS_FREEBSD && !KMP_OS_NETBSD #if !KMP_OS_FREEBSD && !KMP_OS_NETBSD
# include <alloca.h> # include <alloca.h>
#endif #endif
#include <unistd.h> #include <unistd.h>
#include <math.h> // HUGE_VAL. #include <math.h> // HUGE_VAL.
#include <sys/time.h> #include <sys/time.h>
#include <sys/times.h> #include <sys/times.h>
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines
/* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */
#if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED) #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
/* /*
* Affinity support * Affinity support
*/ */
/*
* On some of the older OS's that we build on, these constants aren't present
* in <asm/unistd.h> #included from <sys.syscall.h>. They must be the same on
* all systems of the same arch where they are defined, and they cannot change.
* stone forever.
*/
# if KMP_ARCH_X86 || KMP_ARCH_ARM
# ifndef __NR_sched_setaffinity
# define __NR_sched_setaffinity 241
# elif __NR_sched_setaffinity != 241
# error Wrong code for setaffinity system call.
# endif /* __NR_sched_setaffinity */
# ifndef __NR_sched_getaffinity
# define __NR_sched_getaffinity 242
# elif __NR_sched_getaffinity != 242
# error Wrong code for getaffinity system call.
# endif /* __NR_sched_getaffinity */
# elif KMP_ARCH_AARCH64
# ifndef __NR_sched_setaffinity
# define __NR_sched_setaffinity 122
# elif __NR_sched_setaffinity != 122
# error Wrong code for setaffinity system call.
# endif /* __NR_sched_setaffinity */
# ifndef __NR_sched_getaffinity
# define __NR_sched_getaffinity 123
# elif __NR_sched_getaffinity != 123
# error Wrong code for getaffinity system call.
# endif /* __NR_sched_getaffinity */
# elif KMP_ARCH_X86_64
# ifndef __NR_sched_setaffinity
# define __NR_sched_setaffinity 203
# elif __NR_sched_setaffinity != 203
# error Wrong code for setaffinity system call.
# endif /* __NR_sched_setaffinity */
# ifndef __NR_sched_getaffinity
# define __NR_sched_getaffinity 204
# elif __NR_sched_getaffinity != 204
# error Wrong code for getaffinity system call.
# endif /* __NR_sched_getaffinity */
# elif KMP_ARCH_PPC64
# ifndef __NR_sched_setaffinity
# define __NR_sched_setaffinity 222
# elif __NR_sched_setaffinity != 222
# error Wrong code for setaffinity system call.
# endif /* __NR_sched_setaffinity */
# ifndef __NR_sched_getaffinity
# define __NR_sched_getaffinity 223
# elif __NR_sched_getaffinity != 223
# error Wrong code for getaffinity system call.
# endif /* __NR_sched_getaffinity */
# else
# error Unknown or unsupported architecture
# endif /* KMP_ARCH_* */
int
__kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
{
KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
"Illegal set affinity operation when not capable");
#if KMP_USE_HWLOC
int retval = hwloc_set_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
#else
int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
#endif
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FatalSysError ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
}
int
__kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
{
KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
"Illegal get affinity operation when not capable");
#if KMP_USE_HWLOC
int retval = hwloc_get_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
#else
int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
#endif
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FatalSysError ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
}
void void
__kmp_affinity_bind_thread( int which ) __kmp_affinity_bind_thread( int which )
{ {
KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
"Illegal set affinity operation when not capable"); "Illegal set affinity operation when not capable");
kmp_affin_mask_t *mask; kmp_affin_mask_t *mask;
KMP_CPU_ALLOC_ON_STACK(mask); KMP_CPU_ALLOC_ON_STACK(mask);
▲ Show 20 Lines • Show All 2,458 Lines • Show Last 20 Lines

openmp/trunk/runtime/src/z_Windows_NT_util.c

Show All 12 Lines
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "kmp.h" #include "kmp.h"
#include "kmp_itt.h" #include "kmp_itt.h"
#include "kmp_i18n.h" #include "kmp_i18n.h"
#include "kmp_io.h" #include "kmp_io.h"
#include "kmp_wait_release.h" #include "kmp_wait_release.h"
#include "kmp_affinity.h"
/* This code is related to NtQuerySystemInformation() function. This function /* This code is related to NtQuerySystemInformation() function. This function
is used in the Load balance algorithm for OMP_DYNAMIC=true to find the is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
number of running threads in the system. */ number of running threads in the system. */
#include <ntstatus.h> #include <ntstatus.h>
#include <ntsecapi.h> // UNICODE_STRING #include <ntsecapi.h> // UNICODE_STRING
▲ Show 20 Lines • Show All 93 Lines • ▼ Show 20 Lines
typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG ); typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
NtQuerySystemInformation_t NtQuerySystemInformation = NULL; NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
HMODULE ntdll = NULL; HMODULE ntdll = NULL;
/* End of NtQuerySystemInformation()-related code */ /* End of NtQuerySystemInformation()-related code */
#if KMP_GROUP_AFFINITY
static HMODULE kernel32 = NULL; static HMODULE kernel32 = NULL;
#endif /* KMP_GROUP_AFFINITY */
/* ----------------------------------------------------------------------------------- */ /* ----------------------------------------------------------------------------------- */
/* ----------------------------------------------------------------------------------- */ /* ----------------------------------------------------------------------------------- */
#if KMP_HANDLE_SIGNALS #if KMP_HANDLE_SIGNALS
typedef void (* sig_func_t )( int ); typedef void (* sig_func_t )( int );
static sig_func_t __kmp_sighldrs[ NSIG ]; static sig_func_t __kmp_sighldrs[ NSIG ];
static int __kmp_siginstalled[ NSIG ]; static int __kmp_siginstalled[ NSIG ];
▲ Show 20 Lines • Show All 396 Lines • ▼ Show 20 Lines__kmp_gtid_get_specific()
KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n", KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
__kmp_gtid_threadprivate_key, gtid )); __kmp_gtid_threadprivate_key, gtid ));
return gtid; return gtid;
} }
/* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */
#if KMP_GROUP_AFFINITY
//
// Only 1 DWORD in the mask should have any procs set.
// Return the appropriate index, or -1 for an invalid mask.
//
int
__kmp_get_proc_group( kmp_affin_mask_t const *mask )
{
int i;
int group = -1;
for (i = 0; i < __kmp_num_proc_groups; i++) {
#if KMP_USE_HWLOC
// On windows, the long type is always 32 bits
unsigned long first_32_bits = hwloc_bitmap_to_ith_ulong((hwloc_const_bitmap_t)mask, i*2);
unsigned long second_32_bits = hwloc_bitmap_to_ith_ulong((hwloc_const_bitmap_t)mask, i*2+1);
if (first_32_bits == 0 && second_32_bits == 0) {
continue;
}
#else
if (mask[i] == 0) {
continue;
}
#endif
if (group >= 0) {
return -1;
}
group = i;
}
return group;
}
#endif /* KMP_GROUP_AFFINITY */
int
__kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
{
#if KMP_USE_HWLOC
int retval = hwloc_set_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FatalSysError ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
#else
# if KMP_GROUP_AFFINITY
if (__kmp_num_proc_groups > 1) {
//
// Check for a valid mask.
//
GROUP_AFFINITY ga;
int group = __kmp_get_proc_group( mask );
if (group < 0) {
if (abort_on_error) {
KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
}
return -1;
}
//
// Transform the bit vector into a GROUP_AFFINITY struct
// and make the system call to set affinity.
//
ga.Group = group;
ga.Mask = mask[group];
ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantSetThreadAffMask ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
}
}
else
# endif /* KMP_GROUP_AFFINITY */
{
if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantSetThreadAffMask ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
}
}
#endif /* KMP_USE_HWLOC */
return 0;
}
int
__kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
{
#if KMP_USE_HWLOC
int retval = hwloc_get_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FatalSysError ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
#else /* KMP_USE_HWLOC */
# if KMP_GROUP_AFFINITY
if (__kmp_num_proc_groups > 1) {
KMP_CPU_ZERO(mask);
GROUP_AFFINITY ga;
KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);
if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG(FunctionError, "GetThreadGroupAffinity()"),
KMP_ERR(error),
__kmp_msg_null
);
}
return error;
}
if ((ga.Group < 0) || (ga.Group > __kmp_num_proc_groups)
|| (ga.Mask == 0)) {
return -1;
}
mask[ga.Group] = ga.Mask;
}
else
# endif /* KMP_GROUP_AFFINITY */
{
kmp_affin_mask_t newMask, sysMask, retval;
if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG(FunctionError, "GetProcessAffinityMask()"),
KMP_ERR(error),
__kmp_msg_null
);
}
return error;
}
retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
if (! retval) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
KMP_ERR(error),
__kmp_msg_null
);
}
return error;
}
newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
if (! newMask) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
KMP_ERR(error),
__kmp_msg_null
);
}
}
*mask = retval;
}
#endif /* KMP_USE_HWLOC */
return 0;
}
void void
__kmp_affinity_bind_thread( int proc ) __kmp_affinity_bind_thread( int proc )
{ {
#if KMP_USE_HWLOC
kmp_affin_mask_t *mask;
KMP_CPU_ALLOC_ON_STACK(mask);
KMP_CPU_ZERO(mask);
KMP_CPU_SET(proc, mask);
__kmp_set_system_affinity(mask, TRUE);
KMP_CPU_FREE_FROM_STACK(mask);
#else /* KMP_USE_HWLOC */
# if KMP_GROUP_AFFINITY
if (__kmp_num_proc_groups > 1) { if (__kmp_num_proc_groups > 1) {
// //
// Form the GROUP_AFFINITY struct directly, rather than filling // Form the GROUP_AFFINITY struct directly, rather than filling
// out a bit vector and calling __kmp_set_system_affinity(). // out a bit vector and calling __kmp_set_system_affinity().
// //
GROUP_AFFINITY ga; GROUP_AFFINITY ga;
KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
* CHAR_BIT * sizeof(DWORD_PTR)))); * CHAR_BIT * sizeof(DWORD_PTR))));
ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR)); ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR))); ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0; ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL); KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) { if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
DWORD error = GetLastError(); DWORD error = GetLastError();
if (__kmp_affinity_verbose) { // AC: continue silently if not verbose if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
__kmp_msg( __kmp_msg(
kmp_ms_warning, kmp_ms_warning,
KMP_MSG( CantSetThreadAffMask ), KMP_MSG( CantSetThreadAffMask ),
KMP_ERR( error ), KMP_ERR( error ),
__kmp_msg_null __kmp_msg_null
); );
} }
} }
} else {
kmp_affin_mask_t *mask;
KMP_CPU_ALLOC_ON_STACK(mask);
KMP_CPU_ZERO(mask);
KMP_CPU_SET(proc, mask);
__kmp_set_system_affinity(mask, TRUE);
KMP_CPU_FREE_FROM_STACK(mask);
} }
else
# endif /* KMP_GROUP_AFFINITY */
{
kmp_affin_mask_t mask;
KMP_CPU_ZERO(&mask);
KMP_CPU_SET(proc, &mask);
__kmp_set_system_affinity(&mask, TRUE);
}
#endif /* KMP_USE_HWLOC */
} }
void void
__kmp_affinity_determine_capable( const char *env_var ) __kmp_affinity_determine_capable( const char *env_var )
{ {
// //
// All versions of Windows* OS (since Win '95) support SetThreadAffinityMask(). // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
// //
▲ Show 20 Lines • Show All 1,182 Lines • Show Last 20 Lines