This is an archive of the discontinued LLVM Phabricator instance.

Differential D91105

[OpenMP] Use explicit type casting in kmp_atomic.cpp
ClosedPublic

Authored by hbae on Nov 9 2020, 3:00 PM.

Details

Reviewers
AndreyChurbanov
jlpeyton
jdoerfert
Commits
rG44a11c342caa: [OpenMP] Use explicit type casting in kmp_atomic.cpp
Summary

Changed macros in kmp_atomic.cpp so that they use explicit type casting.

Diff Detail

Event Timeline

hbae created this revision.Nov 9 2020, 3:00 PM
Herald added subscribers: jfb, guansong, yaxunl. · View Herald TranscriptNov 9 2020, 3:00 PM
hbae requested review of this revision.Nov 9 2020, 3:00 PM
Herald added a reviewer: jdoerfert. · View Herald TranscriptNov 9 2020, 3:00 PM
Herald added a subscriber: sstefan1. · View Herald Transcript
Harbormaster completed remote builds in B78196: Diff 303993.Nov 9 2020, 3:47 PM
AndreyChurbanov accepted this revision.Nov 17 2020, 12:12 PM

LGTM

This revision is now accepted and ready to land.Nov 17 2020, 12:12 PM
Closed by commit rG44a11c342caa: [OpenMP] Use explicit type casting in kmp_atomic.cpp (authored by hbae). · Explain WhyNov 17 2020, 12:31 PM
This revision was automatically updated to reflect the committed changes.
hbae added a commit: rG44a11c342caa: [OpenMP] Use explicit type casting in kmp_atomic.cpp.

Revision Contents

PathSize
openmp/
runtime/
src/
247 lines
2 lines

Diff 305882

openmp/runtime/src/kmp_atomic.cpp

Show First 20 Lines • Show All 600 Lines • ▼ Show 20 Lines/* 2007-03-02:
Without "volatile" specifier in OP_CMPXCHG and MIN_MAX_CMPXCHG we have a bug Without "volatile" specifier in OP_CMPXCHG and MIN_MAX_CMPXCHG we have a bug
on *_32 and *_32e. This is just a temporary workaround for the problem. It on *_32 and *_32e. This is just a temporary workaround for the problem. It
seems the right solution is writing OP_CMPXCHG and MIN_MAX_CMPXCHG routines seems the right solution is writing OP_CMPXCHG and MIN_MAX_CMPXCHG routines
in assembler language. */ in assembler language. */
#define KMP_ATOMIC_VOLATILE volatile #define KMP_ATOMIC_VOLATILE volatile
#if (KMP_ARCH_X86) && KMP_HAVE_QUAD #if (KMP_ARCH_X86) && KMP_HAVE_QUAD
static inline void operator+=(Quad_a4_t &lhs, Quad_a4_t &rhs) { static inline Quad_a4_t operator+(Quad_a4_t &lhs, Quad_a4_t &rhs) {
lhs.q += rhs.q; return lhs.q + rhs.q;
} }
static inline void operator-=(Quad_a4_t &lhs, Quad_a4_t &rhs) { static inline Quad_a4_t operator-(Quad_a4_t &lhs, Quad_a4_t &rhs) {
lhs.q -= rhs.q; return lhs.q - rhs.q;
} }
static inline void operator*=(Quad_a4_t &lhs, Quad_a4_t &rhs) { static inline Quad_a4_t operator*(Quad_a4_t &lhs, Quad_a4_t &rhs) {
lhs.q *= rhs.q; return lhs.q * rhs.q;
} }
static inline void operator/=(Quad_a4_t &lhs, Quad_a4_t &rhs) { static inline Quad_a4_t operator/(Quad_a4_t &lhs, Quad_a4_t &rhs) {
lhs.q /= rhs.q; return lhs.q / rhs.q;
} }
static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) { static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
return lhs.q < rhs.q; return lhs.q < rhs.q;
} }
static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) { static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
return lhs.q > rhs.q; return lhs.q > rhs.q;
} }
static inline void operator+=(Quad_a16_t &lhs, Quad_a16_t &rhs) { static inline Quad_a16_t operator+(Quad_a16_t &lhs, Quad_a16_t &rhs) {
lhs.q += rhs.q; return lhs.q + rhs.q;
} }
static inline void operator-=(Quad_a16_t &lhs, Quad_a16_t &rhs) { static inline Quad_a16_t operator-(Quad_a16_t &lhs, Quad_a16_t &rhs) {
lhs.q -= rhs.q; return lhs.q - rhs.q;
} }
static inline void operator*=(Quad_a16_t &lhs, Quad_a16_t &rhs) { static inline Quad_a16_t operator*(Quad_a16_t &lhs, Quad_a16_t &rhs) {
lhs.q *= rhs.q; return lhs.q * rhs.q;
} }
static inline void operator/=(Quad_a16_t &lhs, Quad_a16_t &rhs) { static inline Quad_a16_t operator/(Quad_a16_t &lhs, Quad_a16_t &rhs) {
lhs.q /= rhs.q; return lhs.q / rhs.q;
} }
static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) { static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
return lhs.q < rhs.q; return lhs.q < rhs.q;
} }
static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) { static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
return lhs.q > rhs.q; return lhs.q > rhs.q;
} }
static inline void operator+=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) { static inline kmp_cmplx128_a4_t operator+(kmp_cmplx128_a4_t &lhs,
lhs.q += rhs.q; kmp_cmplx128_a4_t &rhs) {
return lhs.q + rhs.q;
} }
static inline void operator-=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) { static inline kmp_cmplx128_a4_t operator-(kmp_cmplx128_a4_t &lhs,
lhs.q -= rhs.q; kmp_cmplx128_a4_t &rhs) {
return lhs.q - rhs.q;
} }
static inline void operator*=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) { static inline kmp_cmplx128_a4_t operator*(kmp_cmplx128_a4_t &lhs,
lhs.q *= rhs.q; kmp_cmplx128_a4_t &rhs) {
return lhs.q * rhs.q;
} }
static inline void operator/=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) { static inline kmp_cmplx128_a4_t operator/(kmp_cmplx128_a4_t &lhs,
lhs.q /= rhs.q; kmp_cmplx128_a4_t &rhs) {
return lhs.q / rhs.q;
} }
static inline void operator+=(kmp_cmplx128_a16_t &lhs, static inline kmp_cmplx128_a16_t operator+(kmp_cmplx128_a16_t &lhs,
kmp_cmplx128_a16_t &rhs) { kmp_cmplx128_a16_t &rhs) {
lhs.q += rhs.q; return lhs.q + rhs.q;
} }
static inline void operator-=(kmp_cmplx128_a16_t &lhs, static inline kmp_cmplx128_a16_t operator-(kmp_cmplx128_a16_t &lhs,
kmp_cmplx128_a16_t &rhs) { kmp_cmplx128_a16_t &rhs) {
lhs.q -= rhs.q; return lhs.q - rhs.q;
} }
static inline void operator*=(kmp_cmplx128_a16_t &lhs, static inline kmp_cmplx128_a16_t operator*(kmp_cmplx128_a16_t &lhs,
kmp_cmplx128_a16_t &rhs) { kmp_cmplx128_a16_t &rhs) {
lhs.q *= rhs.q; return lhs.q * rhs.q;
} }
static inline void operator/=(kmp_cmplx128_a16_t &lhs, static inline kmp_cmplx128_a16_t operator/(kmp_cmplx128_a16_t &lhs,
kmp_cmplx128_a16_t &rhs) { kmp_cmplx128_a16_t &rhs) {
lhs.q /= rhs.q; return lhs.q / rhs.q;
} }
#endif // (KMP_ARCH_X86) && KMP_HAVE_QUAD #endif // (KMP_ARCH_X86) && KMP_HAVE_QUAD
// ATOMIC implementation routines ----------------------------------------- // ATOMIC implementation routines -----------------------------------------
// One routine for each operation and operand type. // One routine for each operation and operand type.
// All routines declarations looks like // All routines declarations looks like
// void __kmpc_atomic_RTYPE_OP( ident_t*, int, TYPE *lhs, TYPE rhs ); // void __kmpc_atomic_RTYPE_OP( ident_t*, int, TYPE *lhs, TYPE rhs );
Show All 38 Lines
// 1, 2-byte - expect valid parameter, other - check before this macro // 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL(OP, LCK_ID) \ #define OP_CRITICAL(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\ \
(*lhs) OP(rhs); \ (*lhs) OP(rhs); \
\ \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
#define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
(*lhs) = (TYPE)((*lhs)OP(rhs)); \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// For GNU compatibility, we may need to use a critical section, // For GNU compatibility, we may need to use a critical section,
// even though it is not required by the ISA. // even though it is not required by the ISA.
// //
// On IA-32 architecture, all atomic operations except for fixed 4 byte add, // On IA-32 architecture, all atomic operations except for fixed 4 byte add,
// sub, and bitwise logical ops, and 1 & 2 byte logical ops use a common // sub, and bitwise logical ops, and 1 & 2 byte logical ops use a common
// critical section. On Intel(R) 64, all atomic operations are done with fetch // critical section. On Intel(R) 64, all atomic operations are done with fetch
// and add or compare and exchange. Therefore, the FLAG parameter to this // and add or compare and exchange. Therefore, the FLAG parameter to this
Show All 13 Lines
#ifdef KMP_GOMP_COMPAT #ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL(OP, FLAG) \ #define OP_GOMP_CRITICAL(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \ if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL(OP, 0); \ OP_CRITICAL(OP, 0); \
return; \ return; \
} }
#define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_UPDATE_CRITICAL(TYPE, OP, 0); \
return; \
}
#else #else
#define OP_GOMP_CRITICAL(OP, FLAG) #define OP_GOMP_CRITICAL(OP, FLAG)
#define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG)
#endif /* KMP_GOMP_COMPAT */ #endif /* KMP_GOMP_COMPAT */
#if KMP_MIC #if KMP_MIC
#define KMP_DO_PAUSE _mm_delay_32(1) #define KMP_DO_PAUSE _mm_delay_32(1)
#else #else
#define KMP_DO_PAUSE KMP_CPU_PAUSE() #define KMP_DO_PAUSE KMP_CPU_PAUSE()
#endif /* KMP_MIC */ #endif /* KMP_MIC */
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store" routine // Operation on *lhs, rhs using "compare_and_store" routine
// TYPE - operands' type // TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls // BITS - size in bits, used to distinguish low level calls
// OP - operator // OP - operator
#define OP_CMPXCHG(TYPE, BITS, OP) \ #define OP_CMPXCHG(TYPE, BITS, OP) \
{ \ { \
TYPE old_value, new_value; \ TYPE old_value, new_value; \
old_value = *(TYPE volatile *)lhs; \ old_value = *(TYPE volatile *)lhs; \
new_value = old_value OP rhs; \ new_value = (TYPE)(old_value OP rhs); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
KMP_DO_PAUSE; \ KMP_DO_PAUSE; \
\ \
old_value = *(TYPE volatile *)lhs; \ old_value = *(TYPE volatile *)lhs; \
new_value = old_value OP rhs; \ new_value = (TYPE)(old_value OP rhs); \
} \ } \
} }
#if USE_CMPXCHG_FIX #if USE_CMPXCHG_FIX
// 2007-06-25: // 2007-06-25:
// workaround for C78287 (complex(kind=4) data type). lin_32, lin_32e, win_32 // workaround for C78287 (complex(kind=4) data type). lin_32, lin_32e, win_32
// and win_32e are affected (I verified the asm). Compiler ignores the volatile // and win_32e are affected (I verified the asm). Compiler ignores the volatile
// qualifier of the temp_val in the OP_CMPXCHG macro. This is a problem of the // qualifier of the temp_val in the OP_CMPXCHG macro. This is a problem of the
// compiler. Related tracker is C76005, targeted to 11.0. I verified the asm of // compiler. Related tracker is C76005, targeted to 11.0. I verified the asm of
// the workaround. // the workaround.
#define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
{ \ { \
struct _sss { \ struct _sss { \
TYPE cmp; \ TYPE cmp; \
kmp_int##BITS *vvv; \ kmp_int##BITS *vvv; \
}; \ }; \
struct _sss old_value, new_value; \ struct _sss old_value, new_value; \
old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \ old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \ new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
*old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
new_value.cmp = old_value.cmp OP rhs; \ new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \ (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
*VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \ *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
KMP_DO_PAUSE; \ KMP_DO_PAUSE; \
\ \
*old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
new_value.cmp = old_value.cmp OP rhs; \ new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
} \ } \
} }
// end of the first part of the workaround for C78287 // end of the first part of the workaround for C78287
#endif // USE_CMPXCHG_FIX #endif // USE_CMPXCHG_FIX
#if KMP_ARCH_X86 || KMP_ARCH_X86_64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ==================================== // X86 or X86_64: no alignment problems ====================================
#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \ GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
/* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \ /* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \ GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG(TYPE, BITS, OP) \ OP_CMPXCHG(TYPE, BITS, OP) \
} }
#if USE_CMPXCHG_FIX #if USE_CMPXCHG_FIX
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// workaround for C78287 (complex(kind=4) data type) // workaround for C78287 (complex(kind=4) data type)
#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
MASK, GOMP_FLAG) \ MASK, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
} }
// end of the second part of the workaround for C78287 // end of the second part of the workaround for C78287
#endif // USE_CMPXCHG_FIX #endif // USE_CMPXCHG_FIX
#else #else
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses. // Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \ GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
/* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \ /* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
} else { \ } else { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL(OP## =, LCK_ID) /* unaligned address - use critical */ \ OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \ } \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \ GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \ OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \ } else { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL(OP## =, LCK_ID) /* unaligned address - use critical */ \ OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \ } \
} }
#if USE_CMPXCHG_FIX #if USE_CMPXCHG_FIX
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// workaround for C78287 (complex(kind=4) data type) // workaround for C78287 (complex(kind=4) data type)
#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
MASK, GOMP_FLAG) \ MASK, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \ OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \ } else { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL(OP## =, LCK_ID) /* unaligned address - use critical */ \ OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \ } \
} }
// end of the second part of the workaround for C78287 // end of the second part of the workaround for C78287
#endif // USE_CMPXCHG_FIX #endif // USE_CMPXCHG_FIX
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
// Routines for ATOMIC 4-byte operands addition and subtraction // Routines for ATOMIC 4-byte operands addition and subtraction
ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3, ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
▲ Show 20 Lines • Show All 300 Lines • ▼ Show 20 LinesMIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1) // __kmpc_atomic_float16_min_a16 1) // __kmpc_atomic_float16_min_a16
#endif // (KMP_ARCH_X86) #endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD #endif // KMP_HAVE_QUAD
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Need separate macros for .EQV. because of the need of complement (~) // Need separate macros for .EQV. because of the need of complement (~)
// OP ignored for critical sections, ^=~ used instead // OP ignored for critical sections, ^=~ used instead
#define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) /* send assignment */ \ OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL(^= ~, LCK_ID) /* send assignment and complement */ \ OP_CRITICAL(^= (TYPE) ~, LCK_ID) /* send assignment and complement */ \
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
#if KMP_ARCH_X86 || KMP_ARCH_X86_64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// X86 or X86_64: no alignment problems =================================== // X86 or X86_64: no alignment problems ===================================
#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \ GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) /* send assignment */ \ OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) /* send assignment */ \
OP_CMPXCHG(TYPE, BITS, OP) \ OP_CMPXCHG(TYPE, BITS, OP) \
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
#else #else
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses. // Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \ GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \ OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \ OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \ } else { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL(^= ~, LCK_ID) /* unaligned address - use critical */ \ OP_CRITICAL(^= (TYPE) ~, LCK_ID) /* unaligned address - use critical */ \
} \ } \
} }
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0, ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_neqv KMP_ARCH_X86) // __kmpc_atomic_fixed1_neqv
ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1, ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_neqv KMP_ARCH_X86) // __kmpc_atomic_fixed2_neqv
Show All 13 Lines
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use // Routines for Extended types: long double, _Quad, complex flavours (use
// critical section) // critical section)
// TYPE_ID, OP_ID, TYPE - detailed above // TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator // OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable // LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) /* send assignment */ \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL(OP## =, LCK_ID) /* send assignment */ \ OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) /* send assignment */ \
} }
/* ------------------------------------------------------------------------- */ /* ------------------------------------------------------------------------- */
// routines for long double type // routines for long double type
ATOMIC_CRITICAL(float10, add, long double, +, 10r, ATOMIC_CRITICAL(float10, add, long double, +, 10r,
1) // __kmpc_atomic_float10_add 1) // __kmpc_atomic_float10_add
ATOMIC_CRITICAL(float10, sub, long double, -, 10r, ATOMIC_CRITICAL(float10, sub, long double, -, 10r,
1) // __kmpc_atomic_float10_sub 1) // __kmpc_atomic_float10_sub
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines
#if KMP_ARCH_X86 || KMP_ARCH_X86_64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Operation on *lhs, rhs bound by critical section // Operation on *lhs, rhs bound by critical section
// OP - operator (it's supposed to contain an assignment) // OP - operator (it's supposed to contain an assignment)
// LCK_ID - lock identifier // LCK_ID - lock identifier
// Note: don't check gtid as it should always be valid // Note: don't check gtid as it should always be valid
// 1, 2-byte - expect valid parameter, other - check before this macro // 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL_REV(OP, LCK_ID) \ #define OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\ \
(*lhs) = (rhs)OP(*lhs); \ (*lhs) = (TYPE)((rhs)OP(*lhs)); \
\ \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
#ifdef KMP_GOMP_COMPAT #ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_REV(OP, FLAG) \ #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \ if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL_REV(OP, 0); \ OP_CRITICAL_REV(TYPE, OP, 0); \
return; \ return; \
} }
#else #else
#define OP_GOMP_CRITICAL_REV(OP, FLAG) #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG)
#endif /* KMP_GOMP_COMPAT */ #endif /* KMP_GOMP_COMPAT */
// Beginning of a definition (provides name, parameters, gebug trace) // Beginning of a definition (provides name, parameters, gebug trace)
// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned // TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned
// fixed) // fixed)
// OP_ID - operation identifier (add, sub, mul, ...) // OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operands' type // TYPE - operands' type
#define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ #define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
Show All 10 Lines
// Note: temp_val introduced in order to force the compiler to read // Note: temp_val introduced in order to force the compiler to read
// *lhs only once (w/o it the compiler reads *lhs twice) // *lhs only once (w/o it the compiler reads *lhs twice)
#define OP_CMPXCHG_REV(TYPE, BITS, OP) \ #define OP_CMPXCHG_REV(TYPE, BITS, OP) \
{ \ { \
TYPE KMP_ATOMIC_VOLATILE temp_val; \ TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value, new_value; \ TYPE old_value, new_value; \
temp_val = *lhs; \ temp_val = *lhs; \
old_value = temp_val; \ old_value = temp_val; \
new_value = rhs OP old_value; \ new_value = (TYPE)(rhs OP old_value); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
KMP_DO_PAUSE; \ KMP_DO_PAUSE; \
\ \
temp_val = *lhs; \ temp_val = *lhs; \
old_value = temp_val; \ old_value = temp_val; \
new_value = rhs OP old_value; \ new_value = (TYPE)(rhs OP old_value); \
} \ } \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \ #define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_REV(TYPE, BITS, OP) \ OP_CMPXCHG_REV(TYPE, BITS, OP) \
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Entries definition for integer operands // Entries definition for integer operands
// TYPE_ID - operands type and size (fixed4, float4) // TYPE_ID - operands type and size (fixed4, float4)
// OP_ID - operation identifier (add, sub, mul, ...) // OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operand type // TYPE - operand type
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use // Routines for Extended types: long double, _Quad, complex flavours (use
// critical section) // critical section)
// TYPE_ID, OP_ID, TYPE - detailed above // TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator // OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable // LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ #define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
OP_CRITICAL_REV(OP, LCK_ID) \ OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
} }
/* ------------------------------------------------------------------------- */ /* ------------------------------------------------------------------------- */
// routines for long double type // routines for long double type
ATOMIC_CRITICAL_REV(float10, sub, long double, -, 10r, ATOMIC_CRITICAL_REV(float10, sub, long double, -, 10r,
1) // __kmpc_atomic_float10_sub_rev 1) // __kmpc_atomic_float10_sub_rev
ATOMIC_CRITICAL_REV(float10, div, long double, /, 10r, ATOMIC_CRITICAL_REV(float10, div, long double, /, 10r,
1) // __kmpc_atomic_float10_div_rev 1) // __kmpc_atomic_float10_div_rev
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
KA_TRACE(100, \ KA_TRACE(100, \
("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
gtid)); gtid));
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \ #define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \
GOMP_FLAG) \ GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) /* send assignment */ \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL(OP## =, LCK_ID) /* send assignment */ \ OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) /* send assignment */ \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#if KMP_ARCH_X86 || KMP_ARCH_X86_64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ==================================== // X86 or X86_64: no alignment problems ====================================
#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \ LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG(TYPE, BITS, OP) \ OP_CMPXCHG(TYPE, BITS, OP) \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#else #else
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses. // Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \ LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \ OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \ } else { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL(OP## =, LCK_ID) /* unaligned address - use critical */ \ OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \ } \
} }
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#if KMP_ARCH_X86 || KMP_ARCH_X86_64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
RTYPE, LCK_ID, MASK, GOMP_FLAG) \ RTYPE, LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_REV(TYPE, BITS, OP) \ OP_CMPXCHG_REV(TYPE, BITS, OP) \
} }
#define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ #define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
LCK_ID, GOMP_FLAG) \ LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
OP_CRITICAL_REV(OP, LCK_ID) \ OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
} }
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
// RHS=float8 // RHS=float8
ATOMIC_CMPXCHG_MIX(fixed1, char, mul, 8, *, float8, kmp_real64, 1i, 0, ATOMIC_CMPXCHG_MIX(fixed1, char, mul, 8, *, float8, kmp_real64, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_mul_float8 KMP_ARCH_X86) // __kmpc_atomic_fixed1_mul_float8
ATOMIC_CMPXCHG_MIX(fixed1, char, div, 8, /, float8, kmp_real64, 1i, 0, ATOMIC_CMPXCHG_MIX(fixed1, char, div, 8, /, float8, kmp_real64, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_float8 KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_float8
▲ Show 20 Lines • Show All 175 Lines • ▼ Show 20 Lines
#if KMP_ARCH_X86 || KMP_ARCH_X86_64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ==================================== // X86 or X86_64: no alignment problems ====================================
#if USE_CMPXCHG_FIX #if USE_CMPXCHG_FIX
// workaround for C78287 (complex(kind=4) data type) // workaround for C78287 (complex(kind=4) data type)
#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \ LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
} }
// end of the second part of the workaround for C78287 // end of the second part of the workaround for C78287
#else #else
#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \ LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG(TYPE, BITS, OP) \ OP_CMPXCHG(TYPE, BITS, OP) \
} }
#endif // USE_CMPXCHG_FIX #endif // USE_CMPXCHG_FIX
#else #else
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses. // Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \ LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \ OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \ } else { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL(OP## =, LCK_ID) /* unaligned address - use critical */ \ OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \ } \
} }
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c, ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
7, KMP_ARCH_X86) // __kmpc_atomic_cmplx4_add_cmplx8 7, KMP_ARCH_X86) // __kmpc_atomic_cmplx4_add_cmplx8
ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c, ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
7, KMP_ARCH_X86) // __kmpc_atomic_cmplx4_sub_cmplx8 7, KMP_ARCH_X86) // __kmpc_atomic_cmplx4_sub_cmplx8
▲ Show 20 Lines • Show All 320 Lines • ▼ Show 20 Lines#define OP_CRITICAL_CPT(OP, LCK_ID) \
} else { \ } else { \
new_value = (*lhs); \ new_value = (*lhs); \
(*lhs) OP rhs; \ (*lhs) OP rhs; \
} \ } \
\ \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return new_value; return new_value;
#define OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (flag) { \
(*lhs) = (TYPE)((*lhs)OP rhs); \
new_value = (*lhs); \
} else { \
new_value = (*lhs); \
(*lhs) = (TYPE)((*lhs)OP rhs); \
} \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return new_value;
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT #ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_CPT(OP, FLAG) \ #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \ if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL_CPT(OP## =, 0); \ OP_UPDATE_CRITICAL_CPT(TYPE, OP, 0); \
} }
#else #else
#define OP_GOMP_CRITICAL_CPT(OP, FLAG) #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG)
#endif /* KMP_GOMP_COMPAT */ #endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store" routine // Operation on *lhs, rhs using "compare_and_store" routine
// TYPE - operands' type // TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls // BITS - size in bits, used to distinguish low level calls
// OP - operator // OP - operator
// Note: temp_val introduced in order to force the compiler to read // Note: temp_val introduced in order to force the compiler to read
// *lhs only once (w/o it the compiler reads *lhs twice) // *lhs only once (w/o it the compiler reads *lhs twice)
#define OP_CMPXCHG_CPT(TYPE, BITS, OP) \ #define OP_CMPXCHG_CPT(TYPE, BITS, OP) \
{ \ { \
TYPE KMP_ATOMIC_VOLATILE temp_val; \ TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value, new_value; \ TYPE old_value, new_value; \
temp_val = *lhs; \ temp_val = *lhs; \
old_value = temp_val; \ old_value = temp_val; \
new_value = old_value OP rhs; \ new_value = (TYPE)(old_value OP rhs); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
KMP_CPU_PAUSE(); \ KMP_CPU_PAUSE(); \
\ \
temp_val = *lhs; \ temp_val = *lhs; \
old_value = temp_val; \ old_value = temp_val; \
new_value = old_value OP rhs; \ new_value = (TYPE)(old_value OP rhs); \
} \ } \
if (flag) { \ if (flag) { \
return new_value; \ return new_value; \
} else \ } else \
return old_value; \ return old_value; \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ #define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \ TYPE new_value; \
OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_CPT(TYPE, BITS, OP) \ OP_CMPXCHG_CPT(TYPE, BITS, OP) \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ #define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE old_value, new_value; \ TYPE old_value, new_value; \
OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
/* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \ /* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
if (flag) { \ if (flag) { \
return old_value OP rhs; \ return old_value OP rhs; \
} else \ } else \
return old_value; \ return old_value; \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines KA_TRACE(100, \
("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
gtid)); gtid));
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
RTYPE, LCK_ID, MASK, GOMP_FLAG) \ RTYPE, LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE new_value; \ TYPE new_value; \
OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_CPT(TYPE, BITS, OP) \ OP_CMPXCHG_CPT(TYPE, BITS, OP) \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
LCK_ID, GOMP_FLAG) \ LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE new_value; \ TYPE new_value; \
OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) /* send assignment */ \ OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL_CPT(OP## =, LCK_ID) /* send assignment */ \ OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) /* send assignment */ \
} }
ATOMIC_CMPXCHG_CPT_MIX(fixed1, char, add_cpt, 8, +, fp, _Quad, 1i, 0, ATOMIC_CMPXCHG_CPT_MIX(fixed1, char, add_cpt, 8, +, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_add_cpt_fp KMP_ARCH_X86) // __kmpc_atomic_fixed1_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0, ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_add_cpt_fp KMP_ARCH_X86) // __kmpc_atomic_fixed1u_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1, char, sub_cpt, 8, -, fp, _Quad, 1i, 0, ATOMIC_CMPXCHG_CPT_MIX(fixed1, char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_cpt_fp KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_cpt_fp
▲ Show 20 Lines • Show All 271 Lines • ▼ Show 20 Lines#define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \
} }
#else #else
#define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */ #endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
#define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ #define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \ TYPE new_value; \
OP_GOMP_CRITICAL_EQV_CPT(^= ~, GOMP_FLAG) /* send assignment */ \ OP_GOMP_CRITICAL_EQV_CPT(^= (TYPE) ~, GOMP_FLAG) /* send assignment */ \
OP_CMPXCHG_CPT(TYPE, BITS, OP) \ OP_CMPXCHG_CPT(TYPE, BITS, OP) \
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^, ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_neqv_cpt KMP_ARCH_X86) // __kmpc_atomic_fixed1_neqv_cpt
ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^, ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
Show All 15 Lines
// Routines for Extended types: long double, _Quad, complex flavours (use // Routines for Extended types: long double, _Quad, complex flavours (use
// critical section) // critical section)
// TYPE_ID, OP_ID, TYPE - detailed above // TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator // OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable // LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ #define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \ TYPE new_value; \
OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) /* send assignment */ \ OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL_CPT(OP## =, LCK_ID) /* send assignment */ \ OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) /* send assignment */ \
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Workaround for cmplx4. Regular routines with return value don't work // Workaround for cmplx4. Regular routines with return value don't work
// on Win_32e. Let's return captured values through the additional parameter. // on Win_32e. Let's return captured values through the additional parameter.
#define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \ #define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\ \
▲ Show 20 Lines • Show All 120 Lines • ▼ Show 20 Lines
// Supported only on IA-32 architecture and Intel(R) 64 // Supported only on IA-32 architecture and Intel(R) 64
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// Operation on *lhs, rhs bound by critical section // Operation on *lhs, rhs bound by critical section
// OP - operator (it's supposed to contain an assignment) // OP - operator (it's supposed to contain an assignment)
// LCK_ID - lock identifier // LCK_ID - lock identifier
// Note: don't check gtid as it should always be valid // Note: don't check gtid as it should always be valid
// 1, 2-byte - expect valid parameter, other - check before this macro // 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL_CPT_REV(OP, LCK_ID) \ #define OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\ \
if (flag) { \ if (flag) { \
/*temp_val = (*lhs);*/ \ /*temp_val = (*lhs);*/ \
(*lhs) = (rhs)OP(*lhs); \ (*lhs) = (TYPE)((rhs)OP(*lhs)); \
new_value = (*lhs); \ new_value = (*lhs); \
} else { \ } else { \
new_value = (*lhs); \ new_value = (*lhs); \
(*lhs) = (rhs)OP(*lhs); \ (*lhs) = (TYPE)((rhs)OP(*lhs)); \
} \ } \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return new_value; return new_value;
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT #ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG) \ #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \ if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \ KMP_CHECK_GTID; \
OP_CRITICAL_CPT_REV(OP, 0); \ OP_CRITICAL_CPT_REV(TYPE, OP, 0); \
} }
#else #else
#define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG) #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG)
#endif /* KMP_GOMP_COMPAT */ #endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store" routine // Operation on *lhs, rhs using "compare_and_store" routine
// TYPE - operands' type // TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls // BITS - size in bits, used to distinguish low level calls
// OP - operator // OP - operator
// Note: temp_val introduced in order to force the compiler to read // Note: temp_val introduced in order to force the compiler to read
// *lhs only once (w/o it the compiler reads *lhs twice) // *lhs only once (w/o it the compiler reads *lhs twice)
#define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ #define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
{ \ { \
TYPE KMP_ATOMIC_VOLATILE temp_val; \ TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value, new_value; \ TYPE old_value, new_value; \
temp_val = *lhs; \ temp_val = *lhs; \
old_value = temp_val; \ old_value = temp_val; \
new_value = rhs OP old_value; \ new_value = (TYPE)(rhs OP old_value); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
KMP_CPU_PAUSE(); \ KMP_CPU_PAUSE(); \
\ \
temp_val = *lhs; \ temp_val = *lhs; \
old_value = temp_val; \ old_value = temp_val; \
new_value = rhs OP old_value; \ new_value = (TYPE)(rhs OP old_value); \
} \ } \
if (flag) { \ if (flag) { \
return new_value; \ return new_value; \
} else \ } else \
return old_value; \ return old_value; \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \ TYPE new_value; \
OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
} }
ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /, ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_cpt_rev KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /, ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_cpt_rev KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<, ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines
// critical section) // critical section)
// TYPE_ID, OP_ID, TYPE - detailed above // TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator // OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable // LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \ TYPE new_value; \
/*printf("__kmp_atomic_mode = %d\n", __kmp_atomic_mode);*/ \ /*printf("__kmp_atomic_mode = %d\n", __kmp_atomic_mode);*/ \
OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
OP_CRITICAL_CPT_REV(OP, LCK_ID) \ OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
} }
/* ------------------------------------------------------------------------- */ /* ------------------------------------------------------------------------- */
// routines for long double type // routines for long double type
ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev, long double, -, 10r, ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev, long double, -, 10r,
1) // __kmpc_atomic_float10_sub_cpt_rev 1) // __kmpc_atomic_float10_sub_cpt_rev
ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev, long double, /, 10r, ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev, long double, /, 10r,
1) // __kmpc_atomic_float10_div_cpt_rev 1) // __kmpc_atomic_float10_div_cpt_rev
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines
// fixed) // fixed)
// OP_ID - operation identifier (add, sub, mul, ...) // OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operands' type // TYPE - operands' type
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
RTYPE, LCK_ID, MASK, GOMP_FLAG) \ RTYPE, LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE new_value; \ TYPE new_value; \
OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
#define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
LCK_ID, GOMP_FLAG) \ LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE new_value; \ TYPE new_value; \
OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) /* send assignment */ \ OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL_CPT_REV(OP, LCK_ID) /* send assignment */ \ OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) /* send assignment */ \
} }
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1, char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0, ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1, char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_cpt_rev_fp KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0, ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_sub_cpt_rev_fp KMP_ARCH_X86) // __kmpc_atomic_fixed1u_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1, char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0, ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1, char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_cpt_rev_fp KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_cpt_rev_fp
▲ Show 20 Lines • Show All 525 Lines • Show Last 20 Lines

openmp/runtime/src/kmp_stats.h

Show First 20 Lines • Show All 416 Lines • ▼ Show 20 Linespublic:
double getMax() const { return maxVal; } double getMax() const { return maxVal; }
uint64_t getCount() const { return sampleCount; } uint64_t getCount() const { return sampleCount; }
double getSD() const { return sqrt(m2 / sampleCount); } double getSD() const { return sqrt(m2 / sampleCount); }
double getTotal() const { return sampleCount * meanVal; } double getTotal() const { return sampleCount * meanVal; }
logHistogram const *getHist() const { return &hist; } logHistogram const *getHist() const { return &hist; }
void setOffset(double d) { offset = d; } void setOffset(double d) { offset = d; }
void reset() { void reset() {
minVal = std::numeric_limits<double>::max(); minVal = (std::numeric_limits<double>::max)();
maxVal = -minVal; maxVal = -minVal;
meanVal = 0.0; meanVal = 0.0;
m2 = 0.0; m2 = 0.0;
sampleCount = 0; sampleCount = 0;
offset = 0.0; offset = 0.0;
hist.reset(); hist.reset();
} }
void addSample(double sample); void addSample(double sample);
▲ Show 20 Lines • Show All 577 Lines • Show Last 20 Lines