Page MenuHomePhabricator

Please use GitHub pull requests for new patches. Phabricator shutdown timeline

Differential D101173 Diff 344545 openmp/runtime/src/kmp_os.h

Changeset View

Standalone View

View Options

openmp/runtime/src/kmp_os.h

Show First 20 Lines • Show All 133 Lines • ▼ Show 20 Lines
typedef struct kmp_struct64 kmp_int64; typedef struct kmp_struct64 kmp_int64;
typedef struct kmp_struct64 kmp_uint64; typedef struct kmp_struct64 kmp_uint64;
/* Not sure what to use for KMP_[U]INT64_SPEC here */ /* Not sure what to use for KMP_[U]INT64_SPEC here */
#endif #endif
#if KMP_ARCH_X86 && KMP_MSVC_COMPAT #if KMP_ARCH_X86 && KMP_MSVC_COMPAT
#undef KMP_USE_X87CONTROL #undef KMP_USE_X87CONTROL
#define KMP_USE_X87CONTROL 1 #define KMP_USE_X87CONTROL 1
#endif #endif
#if KMP_ARCH_X86_64 #if KMP_ARCH_X86_64 || KMP_ARCH_AARCH64
#define KMP_INTPTR 1 #define KMP_INTPTR 1
typedef __int64 kmp_intptr_t; typedef __int64 kmp_intptr_t;
typedef unsigned __int64 kmp_uintptr_t; typedef unsigned __int64 kmp_uintptr_t;
#define KMP_INTPTR_SPEC "I64d" #define KMP_INTPTR_SPEC "I64d"
#define KMP_UINTPTR_SPEC "I64u" #define KMP_UINTPTR_SPEC "I64u"
#endif #endif
#endif /* KMP_OS_WINDOWS */ #endif /* KMP_OS_WINDOWS */
▲ Show 20 Lines • Show All 307 Lines • ▼ Show 20 Lines
#define KMP_XCHG_FIXED64(p, v) \ #define KMP_XCHG_FIXED64(p, v) \
InterlockedExchange64((volatile kmp_int64 *)(p), (kmp_int64)(v)) InterlockedExchange64((volatile kmp_int64 *)(p), (kmp_int64)(v))
inline kmp_real32 KMP_XCHG_REAL32(volatile kmp_real32 *p, kmp_real32 v) { inline kmp_real32 KMP_XCHG_REAL32(volatile kmp_real32 *p, kmp_real32 v) {
kmp_int32 tmp = InterlockedExchange((volatile long *)p, *(long *)&v); kmp_int32 tmp = InterlockedExchange((volatile long *)p, *(long *)&v);
return *(kmp_real32 *)&tmp; return *(kmp_real32 *)&tmp;
} }
// Routines that we still need to implement in assembly. #define KMP_TEST_THEN_OR8(p, v) __kmp_test_then_or8((p), (v))
extern kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 v); #define KMP_TEST_THEN_AND8(p, v) __kmp_test_then_and8((p), (v))
#define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32((p), (v))
#define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32((p), (v))
#define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64((p), (v))
#define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64((p), (v))
extern kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 v); extern kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 v);
extern kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 v); extern kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 v);
extern kmp_int32 __kmp_test_then_add32(volatile kmp_int32 *p, kmp_int32 v); extern kmp_int32 __kmp_test_then_add32(volatile kmp_int32 *p, kmp_int32 v);
extern kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 v); extern kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 v);
extern kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 v); extern kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 v);
extern kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 v); extern kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 v);
extern kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 v); extern kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 v);
extern kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 v); extern kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 v);
#if KMP_ARCH_AARCH64 && KMP_COMPILER_MSVC && !KMP_COMPILER_CLANG
#define KMP_TEST_THEN_INC64(p) _InterlockedExchangeAdd64((p), 1LL)
#define KMP_TEST_THEN_INC_ACQ64(p) _InterlockedExchangeAdd64_acq((p), 1LL)
#define KMP_TEST_THEN_ADD4_64(p) _InterlockedExchangeAdd64((p), 4LL)
// #define KMP_TEST_THEN_ADD4_ACQ64(p) _InterlockedExchangeAdd64_acq((p), 4LL)
// #define KMP_TEST_THEN_DEC64(p) _InterlockedExchangeAdd64((p), -1LL)
// #define KMP_TEST_THEN_DEC_ACQ64(p) _InterlockedExchangeAdd64_acq((p), -1LL)
// #define KMP_TEST_THEN_ADD8(p, v) _InterlockedExchangeAdd8((p), (v))
#define KMP_TEST_THEN_ADD64(p, v) _InterlockedExchangeAdd64((p), (v))
#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \
__kmp_compare_and_store_acq8((p), (cv), (sv))
#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \
__kmp_compare_and_store_rel8((p), (cv), (sv))
#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \
__kmp_compare_and_store_acq16((p), (cv), (sv))
// #define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \
// __kmp_compare_and_store_rel16((p), (cv), (sv))
#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) \
__kmp_compare_and_store_acq32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \
(kmp_int32)(sv))
#define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) \
__kmp_compare_and_store_rel32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \
(kmp_int32)(sv))
#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \
__kmp_compare_and_store_acq64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \
(kmp_int64)(sv))
#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \
__kmp_compare_and_store_rel64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \
(kmp_int64)(sv))
#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \
__kmp_compare_and_store_ptr((void *volatile *)(p), (void *)(cv), (void *)(sv))
// KMP_COMPARE_AND_STORE expects this order: pointer, compare, exchange
// _InterlockedCompareExchange expects this order: pointer, exchange, compare
// KMP_COMPARE_AND_STORE also returns a bool indicating a successful write. A
// write is successful if the return value of _InterlockedCompareExchange is the
// same as the compare value.
inline kmp_int8 __kmp_compare_and_store_acq8(volatile kmp_int8 *p, kmp_int8 cv,
kmp_int8 sv) {
return _InterlockedCompareExchange8_acq(p, sv, cv) == cv;
}
inline kmp_int8 __kmp_compare_and_store_rel8(volatile kmp_int8 *p, kmp_int8 cv,
kmp_int8 sv) {
return _InterlockedCompareExchange8_rel(p, sv, cv) == cv;
}
inline kmp_int16 __kmp_compare_and_store_acq16(volatile kmp_int16 *p,
kmp_int16 cv, kmp_int16 sv) {
return _InterlockedCompareExchange16_acq(p, sv, cv) == cv;
}
inline kmp_int16 __kmp_compare_and_store_rel16(volatile kmp_int16 *p,
kmp_int16 cv, kmp_int16 sv) {
return _InterlockedCompareExchange16_rel(p, sv, cv) == cv;
}
inline kmp_int32 __kmp_compare_and_store_acq32(volatile kmp_int32 *p,
kmp_int32 cv, kmp_int32 sv) {
return _InterlockedCompareExchange_acq((volatile long *)p, sv, cv) == cv;
}
inline kmp_int32 __kmp_compare_and_store_rel32(volatile kmp_int32 *p,
kmp_int32 cv, kmp_int32 sv) {
return _InterlockedCompareExchange_rel((volatile long *)p, sv, cv) == cv;
}
inline kmp_int32 __kmp_compare_and_store_acq64(volatile kmp_int64 *p,
kmp_int64 cv, kmp_int64 sv) {
return _InterlockedCompareExchange64_acq(p, sv, cv) == cv;
}
inline kmp_int32 __kmp_compare_and_store_rel64(volatile kmp_int64 *p,
kmp_int64 cv, kmp_int64 sv) {
return _InterlockedCompareExchange64_rel(p, sv, cv) == cv;
}
inline kmp_int32 __kmp_compare_and_store_ptr(void *volatile *p, void *cv,
void *sv) {
return _InterlockedCompareExchangePointer(p, sv, cv) == cv;
}
// The _RET versions return the value instead of a bool
// #define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) \
// _InterlockedCompareExchange8((p), (sv), (cv))
// #define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) \
// _InterlockedCompareExchange16((p), (sv), (cv))
#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \
_InterlockedCompareExchange64((volatile kmp_int64 *)(p), (kmp_int64)(sv), \
(kmp_int64)(cv))
// #define KMP_XCHG_FIXED8(p, v) \
// _InterlockedExchange8((volatile kmp_int8 *)(p), (kmp_int8)(v));
// #define KMP_XCHG_FIXED16(p, v) _InterlockedExchange16((p), (v));
// #define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64((p), (v)));
// inline kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v) {
// kmp_int64 tmp = _InterlockedExchange64((volatile kmp_int64 *)p, *(kmp_int64
// *)&v); return *(kmp_real64 *)&tmp;
// }
#else // !KMP_ARCH_AARCH64
// Routines that we still need to implement in assembly.
extern kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 v);
extern kmp_int8 __kmp_compare_and_store8(volatile kmp_int8 *p, kmp_int8 cv, extern kmp_int8 __kmp_compare_and_store8(volatile kmp_int8 *p, kmp_int8 cv,
kmp_int8 sv); kmp_int8 sv);
extern kmp_int16 __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, extern kmp_int16 __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv,
kmp_int16 sv); kmp_int16 sv);
extern kmp_int32 __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, extern kmp_int32 __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv,
kmp_int32 sv); kmp_int32 sv);
extern kmp_int32 __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, extern kmp_int32 __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv,
kmp_int64 sv); kmp_int64 sv);
Show All 24 Lines
//#define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32((p), -1) //#define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32((p), -1)
//#define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32((p), -1) //#define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32((p), -1)
#define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64((p), -1LL) #define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64((p), -1LL)
#define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64((p), -1LL) #define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64((p), -1LL)
//#define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32((p), (v)) //#define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32((p), (v))
#define KMP_TEST_THEN_ADD8(p, v) __kmp_test_then_add8((p), (v)) #define KMP_TEST_THEN_ADD8(p, v) __kmp_test_then_add8((p), (v))
#define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64((p), (v)) #define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64((p), (v))
#define KMP_TEST_THEN_OR8(p, v) __kmp_test_then_or8((p), (v))
#define KMP_TEST_THEN_AND8(p, v) __kmp_test_then_and8((p), (v))
#define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32((p), (v))
#define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32((p), (v))
#define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64((p), (v))
#define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64((p), (v))
#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \ #define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \
__kmp_compare_and_store8((p), (cv), (sv)) __kmp_compare_and_store8((p), (cv), (sv))
#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \ #define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \
__kmp_compare_and_store8((p), (cv), (sv)) __kmp_compare_and_store8((p), (cv), (sv))
#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \ #define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \
__kmp_compare_and_store16((p), (cv), (sv)) __kmp_compare_and_store16((p), (cv), (sv))
#define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \ #define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \
Show All 31 Lines
#define KMP_XCHG_FIXED8(p, v) \ #define KMP_XCHG_FIXED8(p, v) \
__kmp_xchg_fixed8((volatile kmp_int8 *)(p), (kmp_int8)(v)); __kmp_xchg_fixed8((volatile kmp_int8 *)(p), (kmp_int8)(v));
#define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16((p), (v)); #define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16((p), (v));
//#define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32((p), (v)); //#define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32((p), (v));
//#define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64((p), (v)); //#define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64((p), (v));
//#define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32((p), (v)); //#define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32((p), (v));
#define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64((p), (v)); #define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64((p), (v));
#endif
#elif (KMP_ASM_INTRINS && KMP_OS_UNIX) || !(KMP_ARCH_X86 || KMP_ARCH_X86_64) #elif (KMP_ASM_INTRINS && KMP_OS_UNIX) || !(KMP_ARCH_X86 || KMP_ARCH_X86_64)
/* cast p to correct type so that proper intrinsic will be used */ /* cast p to correct type so that proper intrinsic will be used */
#define KMP_TEST_THEN_INC32(p) \ #define KMP_TEST_THEN_INC32(p) \
__sync_fetch_and_add((volatile kmp_int32 *)(p), 1) __sync_fetch_and_add((volatile kmp_int32 *)(p), 1)
#define KMP_TEST_THEN_INC_ACQ32(p) \ #define KMP_TEST_THEN_INC_ACQ32(p) \
__sync_fetch_and_add((volatile kmp_int32 *)(p), 1) __sync_fetch_and_add((volatile kmp_int32 *)(p), 1)
▲ Show 20 Lines • Show All 302 Lines • ▼ Show 20 Lines
#else #else
#define KMP_MB() /* _asm{ nop } */ #define KMP_MB() /* _asm{ nop } */
#define KMP_IMB() /* _asm{ nop } */ #define KMP_IMB() /* _asm{ nop } */
#endif #endif
#endif /* KMP_OS_WINDOWS */ #endif /* KMP_OS_WINDOWS */
#if KMP_ARCH_PPC64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || \ #if KMP_ARCH_PPC64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || \
KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64
#if KMP_OS_WINDOWS
#undef KMP_MB
#define KMP_MB() std::atomic_thread_fence(std::memory_order_seq_cst)
#else /* !KMP_OS_WINDOWS */
#define KMP_MB() __sync_synchronize() #define KMP_MB() __sync_synchronize()
#endif #endif
#endif
#ifndef KMP_MB #ifndef KMP_MB
#define KMP_MB() /* nothing to do */ #define KMP_MB() /* nothing to do */
#endif #endif
#ifndef KMP_IMB #ifndef KMP_IMB
#define KMP_IMB() /* nothing to do */ #define KMP_IMB() /* nothing to do */
#endif #endif
▲ Show 20 Lines • Show All 198 Lines • Show Last 20 Lines