Skip to content

Commit 70ebeab

Browse files
EricWFEricWF
committedApr 24, 2019
Rewrite cxa guard implementation.
This patch does three main things: (1) It re-writes the cxa guard implementation to make it testable. (2) Adds support for recursive init detection on non-apple platforms. (3) It adds a futex based implementation. The futex based implementation locks and notifies on a per-object basis, unlike the current implementation which uses a global lock for all objects. Once this patch settles I'll turn it on by default when supported. llvm-svn: 359060
1 parent 0b09875 commit 70ebeab

File tree

5 files changed

+1127
-263
lines changed

5 files changed

+1127
-263
lines changed
 

‎libcxxabi/src/cxa_guard.cpp

+14-262
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@
77
//===----------------------------------------------------------------------===//
88

99
#include "__cxxabi_config.h"
10+
#include "cxxabi.h"
1011

11-
#include "abort_message.h"
12-
#include <__threading_support>
13-
14-
#include <stdint.h>
15-
#include <string.h>
12+
// Tell the implementation that we're building the actual implementation
13+
// (and not testing it)
14+
#define BUILDING_CXA_GUARD
15+
#include "cxa_guard_impl.h"
1616

1717
/*
1818
This implementation must be careful to not call code external to this file
@@ -24,278 +24,30 @@
2424
to not be a problem.
2525
*/
2626

27-
namespace __cxxabiv1
28-
{
29-
30-
namespace
31-
{
32-
33-
enum InitializationResult {
34-
INIT_COMPLETE,
35-
INIT_NOT_COMPLETE,
36-
};
27+
namespace __cxxabiv1 {
3728

3829
#if defined(_LIBCXXABI_GUARD_ABI_ARM)
39-
// A 32-bit, 4-byte-aligned static data value. The least significant 2 bits must
40-
// be statically initialized to 0.
41-
typedef uint32_t guard_type;
30+
using guard_type = uint32_t;
4231
#else
43-
typedef uint64_t guard_type;
44-
#endif
45-
46-
#if !defined(_LIBCXXABI_HAS_NO_THREADS) && defined(__APPLE__) && \
47-
!defined(_LIBCXXABI_GUARD_ABI_ARM)
48-
// This is a special-case pthread dependency for Mac. We can't pull this
49-
// out into libcxx's threading API (__threading_support) because not all
50-
// supported Mac environments provide this function (in pthread.h). To
51-
// make it possible to build/use libcxx in those environments, we have to
52-
// keep this pthread dependency local to libcxxabi. If there is some
53-
// convenient way to detect precisely when pthread_mach_thread_np is
54-
// available in a given Mac environment, it might still be possible to
55-
// bury this dependency in __threading_support.
56-
#ifndef _LIBCPP_HAS_THREAD_API_PTHREAD
57-
#error "How do I pthread_mach_thread_np()?"
58-
#endif
59-
#define LIBCXXABI_HAS_DEADLOCK_DETECTION
60-
#define LOCK_ID_FOR_THREAD() pthread_mach_thread_np(std::__libcpp_thread_get_current_id())
61-
typedef uint32_t lock_type;
62-
#else
63-
#define LOCK_ID_FOR_THREAD() true
64-
typedef bool lock_type;
65-
#endif
66-
67-
enum class OnRelease : char { UNLOCK, UNLOCK_AND_BROADCAST };
68-
69-
struct GlobalMutexGuard {
70-
explicit GlobalMutexGuard(const char* calling_func, OnRelease on_release)
71-
: calling_func(calling_func), on_release(on_release) {
72-
#ifndef _LIBCXXABI_HAS_NO_THREADS
73-
if (std::__libcpp_mutex_lock(&guard_mut))
74-
abort_message("%s failed to acquire mutex", calling_func);
75-
#endif
76-
}
77-
78-
~GlobalMutexGuard() {
79-
#ifndef _LIBCXXABI_HAS_NO_THREADS
80-
if (std::__libcpp_mutex_unlock(&guard_mut))
81-
abort_message("%s failed to release mutex", calling_func);
82-
if (on_release == OnRelease::UNLOCK_AND_BROADCAST) {
83-
if (std::__libcpp_condvar_broadcast(&guard_cv))
84-
abort_message("%s failed to broadcast condition variable",
85-
calling_func);
86-
}
87-
#endif
88-
}
89-
90-
void wait_for_signal() {
91-
#ifndef _LIBCXXABI_HAS_NO_THREADS
92-
if (std::__libcpp_condvar_wait(&guard_cv, &guard_mut))
93-
abort_message("%s condition variable wait failed", calling_func);
94-
#endif
95-
}
96-
97-
private:
98-
GlobalMutexGuard(GlobalMutexGuard const&) = delete;
99-
GlobalMutexGuard& operator=(GlobalMutexGuard const&) = delete;
100-
101-
const char* const calling_func;
102-
OnRelease on_release;
103-
104-
#ifndef _LIBCXXABI_HAS_NO_THREADS
105-
static std::__libcpp_mutex_t guard_mut;
106-
static std::__libcpp_condvar_t guard_cv;
32+
using guard_type = uint64_t;
10733
#endif
108-
};
109-
110-
#ifndef _LIBCXXABI_HAS_NO_THREADS
111-
std::__libcpp_mutex_t GlobalMutexGuard::guard_mut = _LIBCPP_MUTEX_INITIALIZER;
112-
std::__libcpp_condvar_t GlobalMutexGuard::guard_cv =
113-
_LIBCPP_CONDVAR_INITIALIZER;
114-
#endif
115-
116-
struct GuardObject;
117-
118-
/// GuardValue - An abstraction for accessing the various fields and bits of
119-
/// the guard object.
120-
struct GuardValue {
121-
private:
122-
explicit GuardValue(guard_type v) : value(v) {}
123-
friend struct GuardObject;
124-
125-
public:
126-
/// Functions returning the values used to represent the uninitialized,
127-
/// initialized, and initialization pending states.
128-
static GuardValue ZERO();
129-
static GuardValue INIT_COMPLETE();
130-
static GuardValue INIT_PENDING();
131-
132-
/// Returns true if the guard value represents that the initialization is
133-
/// complete.
134-
bool is_initialization_complete() const;
135-
136-
/// Returns true if the guard value represents that the initialization is
137-
/// currently pending.
138-
bool is_initialization_pending() const;
139-
140-
/// Returns the lock value for the current guard value.
141-
lock_type get_lock_value() const;
142-
143-
private:
144-
// Returns a guard object corresponding to the specified lock value.
145-
static guard_type guard_value_from_lock(lock_type l);
146-
147-
// Returns the lock value represented by the specified guard object.
148-
static lock_type lock_value_from_guard(guard_type g);
149-
150-
private:
151-
guard_type value;
152-
};
153-
154-
/// GuardObject - Manages correctly reading and writing to the guard object.
155-
struct GuardObject {
156-
explicit GuardObject(guard_type *g) : guard(g) {}
157-
158-
// Read the current value of the guard object.
159-
// TODO: Make this read atomic.
160-
GuardValue read() const;
161-
162-
// Write the specified value to the guard object.
163-
// TODO: Make this atomic
164-
void write(GuardValue new_val);
165-
166-
private:
167-
GuardObject(const GuardObject&) = delete;
168-
GuardObject& operator=(const GuardObject&) = delete;
169-
170-
guard_type *guard;
171-
};
172-
173-
} // unnamed namespace
17434

17535
extern "C"
17636
{
177-
17837
_LIBCXXABI_FUNC_VIS int __cxa_guard_acquire(guard_type* raw_guard_object) {
179-
GlobalMutexGuard gmutex("__cxa_guard_acquire", OnRelease::UNLOCK);
180-
GuardObject guard(raw_guard_object);
181-
GuardValue current_value = guard.read();
182-
183-
if (current_value.is_initialization_complete())
184-
return INIT_COMPLETE;
185-
186-
const GuardValue LOCK_ID = GuardValue::INIT_PENDING();
187-
#ifdef LIBCXXABI_HAS_DEADLOCK_DETECTION
188-
if (current_value.is_initialization_pending() &&
189-
current_value.get_lock_value() == LOCK_ID.get_lock_value()) {
190-
abort_message("__cxa_guard_acquire detected deadlock");
191-
}
192-
#endif
193-
while (current_value.is_initialization_pending()) {
194-
gmutex.wait_for_signal();
195-
current_value = guard.read();
196-
}
197-
if (current_value.is_initialization_complete())
198-
return INIT_COMPLETE;
199-
200-
guard.write(LOCK_ID);
201-
return INIT_NOT_COMPLETE;
38+
SelectedImplementation imp(raw_guard_object);
39+
return static_cast<int>(imp.cxa_guard_acquire());
20240
}
20341

20442
_LIBCXXABI_FUNC_VIS void __cxa_guard_release(guard_type *raw_guard_object) {
205-
GlobalMutexGuard gmutex("__cxa_guard_release",
206-
OnRelease::UNLOCK_AND_BROADCAST);
207-
GuardObject guard(raw_guard_object);
208-
guard.write(GuardValue::ZERO());
209-
guard.write(GuardValue::INIT_COMPLETE());
43+
SelectedImplementation imp(raw_guard_object);
44+
imp.cxa_guard_release();
21045
}
21146

21247
_LIBCXXABI_FUNC_VIS void __cxa_guard_abort(guard_type *raw_guard_object) {
213-
GlobalMutexGuard gmutex("__cxa_guard_abort", OnRelease::UNLOCK_AND_BROADCAST);
214-
GuardObject guard(raw_guard_object);
215-
guard.write(GuardValue::ZERO());
48+
SelectedImplementation imp(raw_guard_object);
49+
imp.cxa_guard_abort();
21650
}
21751
} // extern "C"
21852

219-
//===----------------------------------------------------------------------===//
220-
// GuardObject Definitions
221-
//===----------------------------------------------------------------------===//
222-
223-
GuardValue GuardObject::read() const {
224-
// FIXME: Make this atomic
225-
guard_type val = *guard;
226-
return GuardValue(val);
227-
}
228-
229-
void GuardObject::write(GuardValue new_val) {
230-
// FIXME: make this atomic
231-
*guard = new_val.value;
232-
}
233-
234-
//===----------------------------------------------------------------------===//
235-
// GuardValue Definitions
236-
//===----------------------------------------------------------------------===//
237-
238-
GuardValue GuardValue::ZERO() { return GuardValue(0); }
239-
240-
GuardValue GuardValue::INIT_COMPLETE() {
241-
guard_type value = {0};
242-
#if defined(_LIBCXXABI_GUARD_ABI_ARM)
243-
value |= 1;
244-
#else
245-
char* init_bit = (char*)&value;
246-
*init_bit = 1;
247-
#endif
248-
return GuardValue(value);
249-
}
250-
251-
GuardValue GuardValue::INIT_PENDING() {
252-
return GuardValue(guard_value_from_lock(LOCK_ID_FOR_THREAD()));
253-
}
254-
255-
bool GuardValue::is_initialization_complete() const {
256-
#if defined(_LIBCXXABI_GUARD_ABI_ARM)
257-
return value & 1;
258-
#else
259-
const char* init_bit = (const char*)&value;
260-
return *init_bit;
261-
#endif
262-
}
263-
264-
bool GuardValue::is_initialization_pending() const {
265-
return lock_value_from_guard(value) != 0;
266-
}
267-
268-
lock_type GuardValue::get_lock_value() const {
269-
return lock_value_from_guard(value);
270-
}
271-
272-
// Create a guard object with the lock set to the specified value.
273-
guard_type GuardValue::guard_value_from_lock(lock_type l) {
274-
#if defined(__APPLE__) && !defined(_LIBCXXABI_GUARD_ABI_ARM)
275-
#if __LITTLE_ENDIAN__
276-
return static_cast<guard_type>(l) << 32;
277-
#else
278-
return static_cast<guard_type>(l);
279-
#endif
280-
#else // defined(__APPLE__) && !defined(_LIBCXXABI_GUARD_ABI_ARM)
281-
guard_type f = {0};
282-
memcpy(static_cast<char*>(static_cast<void*>(&f)) + 1, &l, sizeof(lock_type));
283-
return f;
284-
#endif // defined(__APPLE__) && !defined(_LIBCXXABI_GUARD_ABI_ARM)
285-
}
286-
287-
lock_type GuardValue::lock_value_from_guard(guard_type g) {
288-
#if defined(__APPLE__) && !defined(_LIBCXXABI_GUARD_ABI_ARM)
289-
#if __LITTLE_ENDIAN__
290-
return static_cast<lock_type>(g >> 32);
291-
#else
292-
return static_cast<lock_type>(g);
293-
#endif
294-
#else // defined(__APPLE__) && !defined(_LIBCXXABI_GUARD_ABI_ARM)
295-
uint8_t guard_bytes[sizeof(guard_type)];
296-
memcpy(&guard_bytes, &g, sizeof(guard_type));
297-
return guard_bytes[1] != 0;
298-
#endif // defined(__APPLE__) && !defined(_LIBCXXABI_GUARD_ABI_ARM)
299-
}
300-
30153
} // __cxxabiv1

‎libcxxabi/src/cxa_guard_impl.h

+550
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,550 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
#ifndef LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
9+
#define LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
10+
11+
/* cxa_guard_impl.h - Implements the C++ runtime support for function local
12+
* static guards.
13+
* The layout of the guard object is the same across ARM and Itanium.
14+
*
15+
* The first "guard byte" (which is checked by the compiler) is set only upon
16+
* the completion of cxa release.
17+
*
18+
* The second "init byte" does the rest of the bookkeeping. It tracks if
19+
* initialization is complete or pending, and if there are waiting threads.
20+
*
21+
* If the guard variable is 64-bits and the platforms supplies a 32-bit thread
22+
* identifier, it is used to detect recursive initialization. The thread ID of
23+
* the thread currently performing initialization is stored in the second word.
24+
*
25+
* Guard Object Layout:
26+
* -------------------------------------------------------------------------
27+
* |a: guard byte | a+1: init byte | a+2 : unused ... | a+4: thread-id ... |
28+
* ------------------------------------------------------------------------
29+
*
30+
* Access Protocol:
31+
* For each implementation the guard byte is checked and set before accessing
32+
* the init byte.
33+
*
34+
* Overall Design:
35+
* The implementation was designed to allow each implementation to be tested
36+
* independent of the C++ runtime or platform support.
37+
*
38+
*/
39+
40+
#include "__cxxabi_config.h"
41+
#include "include/atomic_support.h"
42+
#include <unistd.h>
43+
#include <sys/types.h>
44+
#if defined(__has_include)
45+
# if __has_include(<sys/syscall.h>)
46+
# include <sys/syscall.h>
47+
# endif
48+
#endif
49+
50+
#include <stdlib.h>
51+
#include <__threading_support>
52+
53+
// To make testing possible, this header is included from both cxa_guard.cpp
54+
// and a number of tests.
55+
//
56+
// For this reason we place everything in an anonymous namespace -- even though
57+
// we're in a header. We want the actual implementation and the tests to have
58+
// unique definitions of the types in this header (since the tests may depend
59+
// on function local statics).
60+
//
61+
// To enforce this either `BUILDING_CXA_GUARD` or `TESTING_CXA_GUARD` must be
62+
// defined when including this file. Only `src/cxa_guard.cpp` should define
63+
// the former.
64+
#ifdef BUILDING_CXA_GUARD
65+
# include "abort_message.h"
66+
# define ABORT_WITH_MESSAGE(...) ::abort_message(__VA_ARGS__)
67+
#elif defined(TESTING_CXA_GUARD)
68+
# define ABORT_WITH_MESSAGE(...) ::abort()
69+
#else
70+
# error "Either BUILDING_CXA_GUARD or TESTING_CXA_GUARD must be defined"
71+
#endif
72+
73+
74+
namespace __cxxabiv1 {
75+
// Use an anonymous namespace to ensure that the tests and actual implementation
76+
// have unique definitions of these symbols.
77+
namespace {
78+
79+
//===----------------------------------------------------------------------===//
80+
// Misc Utilities
81+
//===----------------------------------------------------------------------===//
82+
83+
template <class T, T(*Init)()>
84+
struct LazyValue {
85+
LazyValue() : is_init(false) {}
86+
87+
T& get() {
88+
if (!is_init) {
89+
value = Init();
90+
is_init = true;
91+
}
92+
return value;
93+
}
94+
private:
95+
T value;
96+
bool is_init = false;
97+
};
98+
99+
//===----------------------------------------------------------------------===//
100+
// PlatformGetThreadID
101+
//===----------------------------------------------------------------------===//
102+
103+
#if defined(__APPLE__) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
104+
uint32_t PlatformThreadID() {
105+
static_assert(sizeof(mach_port_t) == sizeof(uint32_t), "");
106+
return static_cast<uint32_t>(
107+
pthread_mach_thread_np(std::__libcpp_thread_get_current_id()));
108+
}
109+
#elif defined(SYS_gettid) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
110+
uint32_t PlatformThreadID() {
111+
static_assert(sizeof(pid_t) == sizeof(uint32_t), "");
112+
return static_cast<uint32_t>(syscall(SYS_gettid));
113+
}
114+
#else
115+
constexpr uint32_t (*PlatformThreadID)() = nullptr;
116+
#endif
117+
118+
119+
constexpr bool DoesPlatformSupportThreadID() {
120+
#ifdef __clang__
121+
#pragma clang diagnostic push
122+
#pragma clang diagnostic ignored "-Wtautological-pointer-compare"
123+
#endif
124+
return +PlatformThreadID != nullptr;
125+
#ifdef __clang__
126+
#pragma clang diagnostic pop
127+
#endif
128+
}
129+
130+
//===----------------------------------------------------------------------===//
131+
// GuardBase
132+
//===----------------------------------------------------------------------===//
133+
134+
enum class AcquireResult {
135+
INIT_IS_DONE,
136+
INIT_IS_PENDING,
137+
};
138+
constexpr AcquireResult INIT_IS_DONE = AcquireResult::INIT_IS_DONE;
139+
constexpr AcquireResult INIT_IS_PENDING = AcquireResult::INIT_IS_PENDING;
140+
141+
static constexpr uint8_t UNSET = 0;
142+
static constexpr uint8_t COMPLETE_BIT = (1 << 0);
143+
static constexpr uint8_t PENDING_BIT = (1 << 1);
144+
static constexpr uint8_t WAITING_BIT = (1 << 2);
145+
146+
template <class Derived>
147+
struct GuardObject {
148+
GuardObject() = delete;
149+
GuardObject(GuardObject const&) = delete;
150+
GuardObject& operator=(GuardObject const&) = delete;
151+
152+
explicit GuardObject(uint32_t* g)
153+
: base_address(g), guard_byte_address(reinterpret_cast<uint8_t*>(g)),
154+
init_byte_address(reinterpret_cast<uint8_t*>(g) + 1),
155+
thread_id_address(nullptr) {}
156+
157+
explicit GuardObject(uint64_t* g)
158+
: base_address(g), guard_byte_address(reinterpret_cast<uint8_t*>(g)),
159+
init_byte_address(reinterpret_cast<uint8_t*>(g) + 1),
160+
thread_id_address(reinterpret_cast<uint32_t*>(g) + 1) {}
161+
162+
public:
163+
/// Implements __cxa_guard_acquire
164+
AcquireResult cxa_guard_acquire() {
165+
AtomicInt<uint8_t> guard_byte(guard_byte_address);
166+
if (guard_byte.load(std::_AO_Acquire) == COMPLETE_BIT)
167+
return INIT_IS_DONE;
168+
return derived()->acquire_init_byte();
169+
}
170+
171+
/// Implements __cxa_guard_release
172+
void cxa_guard_release() {
173+
AtomicInt<uint8_t> guard_byte(guard_byte_address);
174+
// Store complete first, so that when release wakes other folks, they see
175+
// it as having been completed.
176+
guard_byte.store(COMPLETE_BIT, std::_AO_Release);
177+
derived()->release_init_byte();
178+
}
179+
180+
/// Implements __cxa_guard_abort
181+
void cxa_guard_abort() { derived()->abort_init_byte(); }
182+
183+
public:
184+
/// base_address - the address of the original guard object.
185+
void* const base_address;
186+
/// The address of the guord byte at offset 0.
187+
uint8_t* const guard_byte_address;
188+
/// The address of the byte used by the implementation during initialization.
189+
uint8_t* const init_byte_address;
190+
/// An optional address storing an identifier for the thread performing initialization.
191+
/// It's used to detect recursive initialization.
192+
uint32_t* const thread_id_address;
193+
194+
private:
195+
Derived* derived() { return static_cast<Derived*>(this); }
196+
};
197+
198+
//===----------------------------------------------------------------------===//
199+
// Single Threaded Implementation
200+
//===----------------------------------------------------------------------===//
201+
202+
struct InitByteNoThreads : GuardObject<InitByteNoThreads> {
203+
using GuardObject::GuardObject;
204+
205+
AcquireResult acquire_init_byte() {
206+
if (*init_byte_address == COMPLETE_BIT)
207+
return INIT_IS_DONE;
208+
if (*init_byte_address & PENDING_BIT)
209+
ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
210+
*init_byte_address = PENDING_BIT;
211+
return INIT_IS_PENDING;
212+
}
213+
214+
void release_init_byte() { *init_byte_address = COMPLETE_BIT; }
215+
void abort_init_byte() { *init_byte_address = UNSET; }
216+
};
217+
218+
219+
//===----------------------------------------------------------------------===//
220+
// Global Mutex Implementation
221+
//===----------------------------------------------------------------------===//
222+
223+
struct LibcppMutex;
224+
struct LibcppCondVar;
225+
226+
#ifndef _LIBCXXABI_HAS_NO_THREADS
227+
struct LibcppMutex {
228+
LibcppMutex() = default;
229+
LibcppMutex(LibcppMutex const&) = delete;
230+
LibcppMutex& operator=(LibcppMutex const&) = delete;
231+
232+
bool lock() { return std::__libcpp_mutex_lock(&mutex); }
233+
bool unlock() { return std::__libcpp_mutex_unlock(&mutex); }
234+
235+
private:
236+
friend struct LibcppCondVar;
237+
std::__libcpp_mutex_t mutex = _LIBCPP_MUTEX_INITIALIZER;
238+
};
239+
240+
struct LibcppCondVar {
241+
LibcppCondVar() = default;
242+
LibcppCondVar(LibcppCondVar const&) = delete;
243+
LibcppCondVar& operator=(LibcppCondVar const&) = delete;
244+
245+
bool wait(LibcppMutex& mut) {
246+
return std::__libcpp_condvar_wait(&cond, &mut.mutex);
247+
}
248+
bool broadcast() { return std::__libcpp_condvar_broadcast(&cond); }
249+
250+
private:
251+
std::__libcpp_condvar_t cond = _LIBCPP_CONDVAR_INITIALIZER;
252+
};
253+
#endif // !defined(_LIBCXXABI_HAS_NO_THREADS)
254+
255+
256+
template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
257+
uint32_t (*GetThreadID)() = PlatformThreadID>
258+
struct InitByteGlobalMutex
259+
: GuardObject<InitByteGlobalMutex<Mutex, CondVar, global_mutex, global_cond,
260+
GetThreadID>> {
261+
262+
using BaseT = typename InitByteGlobalMutex::GuardObject;
263+
using BaseT::BaseT;
264+
265+
explicit InitByteGlobalMutex(uint32_t *g)
266+
: BaseT(g), has_thread_id_support(false) {}
267+
explicit InitByteGlobalMutex(uint64_t *g)
268+
: BaseT(g), has_thread_id_support(DoesPlatformSupportThreadID()) {}
269+
270+
public:
271+
AcquireResult acquire_init_byte() {
272+
LockGuard g("__cxa_guard_acquire");
273+
// Check for possible recursive initialization.
274+
if (has_thread_id_support && (*init_byte_address & PENDING_BIT)) {
275+
if (*thread_id_address == current_thread_id.get())
276+
ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
277+
}
278+
279+
// Wait until the pending bit is not set.
280+
while (*init_byte_address & PENDING_BIT) {
281+
*init_byte_address |= WAITING_BIT;
282+
global_cond.wait(global_mutex);
283+
}
284+
285+
if (*init_byte_address == COMPLETE_BIT)
286+
return INIT_IS_DONE;
287+
288+
if (has_thread_id_support)
289+
*thread_id_address = current_thread_id.get();
290+
291+
*init_byte_address = PENDING_BIT;
292+
return INIT_IS_PENDING;
293+
}
294+
295+
void release_init_byte() {
296+
bool has_waiting;
297+
{
298+
LockGuard g("__cxa_guard_release");
299+
has_waiting = *init_byte_address & WAITING_BIT;
300+
*init_byte_address = COMPLETE_BIT;
301+
}
302+
if (has_waiting) {
303+
if (global_cond.broadcast()) {
304+
ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_release");
305+
}
306+
}
307+
}
308+
309+
void abort_init_byte() {
310+
bool has_waiting;
311+
{
312+
LockGuard g("__cxa_guard_abort");
313+
if (has_thread_id_support)
314+
*thread_id_address = 0;
315+
has_waiting = *init_byte_address & WAITING_BIT;
316+
*init_byte_address = UNSET;
317+
}
318+
if (has_waiting) {
319+
if (global_cond.broadcast()) {
320+
ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_abort");
321+
}
322+
}
323+
}
324+
325+
private:
326+
using BaseT::init_byte_address;
327+
using BaseT::thread_id_address;
328+
const bool has_thread_id_support;
329+
LazyValue<uint32_t, GetThreadID> current_thread_id;
330+
331+
private:
332+
struct LockGuard {
333+
LockGuard() = delete;
334+
LockGuard(LockGuard const&) = delete;
335+
LockGuard& operator=(LockGuard const&) = delete;
336+
337+
explicit LockGuard(const char* calling_func)
338+
: calling_func(calling_func) {
339+
if (global_mutex.lock())
340+
ABORT_WITH_MESSAGE("%s failed to acquire mutex", calling_func);
341+
}
342+
343+
~LockGuard() {
344+
if (global_mutex.unlock())
345+
ABORT_WITH_MESSAGE("%s failed to release mutex", calling_func);
346+
}
347+
348+
private:
349+
const char* const calling_func;
350+
};
351+
};
352+
353+
//===----------------------------------------------------------------------===//
354+
// Futex Implementation
355+
//===----------------------------------------------------------------------===//
356+
357+
#if defined(SYS_futex)
358+
void PlatformFutexWait(int* addr, int expect) {
359+
constexpr int WAIT = 0;
360+
syscall(SYS_futex, addr, WAIT, expect, 0);
361+
}
362+
void PlatformFutexWake(int* addr) {
363+
constexpr int WAKE = 1;
364+
syscall(SYS_futex, addr, WAKE, INT_MAX);
365+
}
366+
#else
367+
constexpr void (*PlatformFutexWait)(int*, int) = nullptr;
368+
constexpr void (*PlatformFutexWake)(int*) = nullptr;
369+
#endif
370+
371+
constexpr bool DoesPlatformSupportFutex() {
372+
#ifdef __clang__
373+
#pragma clang diagnostic push
374+
#pragma clang diagnostic ignored "-Wtautological-pointer-compare"
375+
#endif
376+
return +PlatformFutexWait != nullptr;
377+
#ifdef __clang__
378+
#pragma clang diagnostic pop
379+
#endif
380+
}
381+
382+
/// InitByteFutex - Manages initialization using atomics and the futex syscall
383+
/// for waiting and waking.
384+
template <void (*Wait)(int*, int) = PlatformFutexWait,
385+
void (*Wake)(int*) = PlatformFutexWake,
386+
uint32_t (*GetThreadIDArg)() = PlatformThreadID>
387+
struct InitByteFutex : GuardObject<InitByteFutex<Wait, Wake, GetThreadIDArg>> {
388+
using BaseT = typename InitByteFutex::GuardObject;
389+
390+
/// ARM Constructor
391+
explicit InitByteFutex(uint32_t *g) : BaseT(g),
392+
init_byte(this->init_byte_address),
393+
has_thread_id_support(this->thread_id_address && GetThreadIDArg),
394+
thread_id(this->thread_id_address) {}
395+
396+
/// Itanium Constructor
397+
explicit InitByteFutex(uint64_t *g) : BaseT(g),
398+
init_byte(this->init_byte_address),
399+
has_thread_id_support(this->thread_id_address && GetThreadIDArg),
400+
thread_id(this->thread_id_address) {}
401+
402+
public:
403+
AcquireResult acquire_init_byte() {
404+
while (true) {
405+
uint8_t last_val = UNSET;
406+
if (init_byte.compare_exchange(&last_val, PENDING_BIT, std::_AO_Acq_Rel,
407+
std::_AO_Acquire)) {
408+
if (has_thread_id_support) {
409+
thread_id.store(current_thread_id.get(), std::_AO_Relaxed);
410+
}
411+
return INIT_IS_PENDING;
412+
}
413+
414+
if (last_val == COMPLETE_BIT)
415+
return INIT_IS_DONE;
416+
417+
if (last_val & PENDING_BIT) {
418+
419+
// Check for recursive initialization
420+
if (has_thread_id_support && thread_id.load(std::_AO_Relaxed) == current_thread_id.get()) {
421+
ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
422+
}
423+
424+
if ((last_val & WAITING_BIT) == 0) {
425+
// This compare exchange can fail for several reasons
426+
// (1) another thread finished the whole thing before we got here
427+
// (2) another thread set the waiting bit we were trying to thread
428+
// (3) another thread had an exception and failed to finish
429+
if (!init_byte.compare_exchange(&last_val, PENDING_BIT | WAITING_BIT,
430+
std::_AO_Acq_Rel, std::_AO_Release)) {
431+
// (1) success, via someone else's work!
432+
if (last_val == COMPLETE_BIT)
433+
return INIT_IS_DONE;
434+
435+
// (3) someone else, bailed on doing the work, retry from the start!
436+
if (last_val == UNSET)
437+
continue;
438+
439+
// (2) the waiting bit got set, so we are happy to keep waiting
440+
}
441+
}
442+
wait_on_initialization();
443+
}
444+
}
445+
}
446+
447+
void release_init_byte() {
448+
uint8_t old = init_byte.exchange(COMPLETE_BIT, std::_AO_Acq_Rel);
449+
if (old & WAITING_BIT)
450+
wake_all();
451+
}
452+
453+
void abort_init_byte() {
454+
if (has_thread_id_support)
455+
thread_id.store(0, std::_AO_Relaxed);
456+
457+
uint8_t old = init_byte.exchange(0, std::_AO_Acq_Rel);
458+
if (old & WAITING_BIT)
459+
wake_all();
460+
}
461+
462+
private:
463+
/// Use the futex to wait on the current guard variable. Futex expects a
464+
/// 32-bit 4-byte aligned address as the first argument, so we have to use use
465+
/// the base address of the guard variable (not the init byte).
466+
void wait_on_initialization() {
467+
Wait(static_cast<int*>(this->base_address),
468+
expected_value_for_futex(PENDING_BIT | WAITING_BIT));
469+
}
470+
void wake_all() { Wake(static_cast<int*>(this->base_address)); }
471+
472+
private:
473+
AtomicInt<uint8_t> init_byte;
474+
475+
const bool has_thread_id_support;
476+
// Unsafe to use unless has_thread_id_support
477+
AtomicInt<uint32_t> thread_id;
478+
LazyValue<uint32_t, GetThreadIDArg> current_thread_id;
479+
480+
/// Create the expected integer value for futex `wait(int* addr, int expected)`.
481+
/// We pass the base address as the first argument, So this function creates
482+
/// an zero-initialized integer with `b` copied at the correct offset.
483+
static int expected_value_for_futex(uint8_t b) {
484+
int dest_val = 0;
485+
std::memcpy(reinterpret_cast<char*>(&dest_val) + 1, &b, 1);
486+
return dest_val;
487+
}
488+
489+
static_assert(Wait != nullptr && Wake != nullptr, "");
490+
};
491+
492+
//===----------------------------------------------------------------------===//
493+
//
494+
//===----------------------------------------------------------------------===//
495+
496+
template <class T>
497+
struct GlobalStatic {
498+
static T instance;
499+
};
500+
template <class T>
501+
_LIBCPP_SAFE_STATIC T GlobalStatic<T>::instance = {};
502+
503+
enum class Implementation {
504+
NoThreads,
505+
GlobalLock,
506+
Futex
507+
};
508+
509+
template <Implementation Impl>
510+
struct SelectImplementation;
511+
512+
template <>
513+
struct SelectImplementation<Implementation::NoThreads> {
514+
using type = InitByteNoThreads;
515+
};
516+
517+
template <>
518+
struct SelectImplementation<Implementation::GlobalLock> {
519+
using type = InitByteGlobalMutex<
520+
LibcppMutex, LibcppCondVar, GlobalStatic<LibcppMutex>::instance,
521+
GlobalStatic<LibcppCondVar>::instance, PlatformThreadID>;
522+
};
523+
524+
template <>
525+
struct SelectImplementation<Implementation::Futex> {
526+
using type =
527+
InitByteFutex<PlatformFutexWait, PlatformFutexWake, PlatformThreadID>;
528+
};
529+
530+
// TODO(EricWF): We should prefer the futex implementation when available. But
531+
// it should be done in a separate step from adding the implementation.
532+
constexpr Implementation CurrentImplementation =
533+
#if defined(_LIBCXXABI_HAS_NO_THREADS)
534+
Implementation::NoThreads;
535+
#elif defined(_LIBCXXABI_USE_FUTEX)
536+
Implementation::Futex;
537+
#else
538+
Implementation::GlobalLock;
539+
#endif
540+
541+
static_assert(CurrentImplementation != Implementation::Futex
542+
|| DoesPlatformSupportFutex(), "Futex selected but not supported");
543+
544+
using SelectedImplementation =
545+
SelectImplementation<CurrentImplementation>::type;
546+
547+
} // end namespace
548+
} // end namespace __cxxabiv1
549+
550+
#endif // LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H

‎libcxxabi/src/include/atomic_support.h

+31-1
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ _ValueType __libcpp_atomic_add(_ValueType* __val, _AddType __a,
150150
template <class _ValueType>
151151
inline _LIBCPP_INLINE_VISIBILITY
152152
_ValueType __libcpp_atomic_exchange(_ValueType* __target,
153-
_ValueType __value, int __order = _AO_Seq)
153+
_ValueType __value, int = _AO_Seq)
154154
{
155155
_ValueType old = *__target;
156156
*__target = __value;
@@ -177,4 +177,34 @@ bool __libcpp_atomic_compare_exchange(_ValueType* __val,
177177

178178
_LIBCPP_END_NAMESPACE_STD
179179

180+
namespace {
181+
182+
template <class IntType>
183+
class AtomicInt {
184+
public:
185+
using MemoryOrder = std::__libcpp_atomic_order;
186+
187+
explicit AtomicInt(IntType *b) : b(b) {}
188+
AtomicInt(AtomicInt const&) = delete;
189+
AtomicInt& operator=(AtomicInt const&) = delete;
190+
191+
IntType load(MemoryOrder ord) {
192+
return std::__libcpp_atomic_load(b, ord);
193+
}
194+
void store(IntType val, MemoryOrder ord) {
195+
std::__libcpp_atomic_store(b, val, ord);
196+
}
197+
IntType exchange(IntType new_val, MemoryOrder ord) {
198+
return std::__libcpp_atomic_exchange(b, new_val, ord);
199+
}
200+
bool compare_exchange(IntType *expected, IntType desired, MemoryOrder ord_success, MemoryOrder ord_failure) {
201+
return std::__libcpp_atomic_compare_exchange(b, expected, desired, ord_success, ord_failure);
202+
}
203+
204+
private:
205+
IntType *b;
206+
};
207+
208+
} // end namespace
209+
180210
#endif // ATOMIC_SUPPORT_H

‎libcxxabi/test/guard_test_basic.pass.cpp

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// UNSUPPORTED: c++98, c++03
10+
11+
#define TESTING_CXA_GUARD
12+
#include "../src/cxa_guard_impl.h"
13+
14+
using namespace __cxxabiv1;
15+
16+
template <class GuardType, class Impl>
17+
struct Tests {
18+
private:
19+
Tests() : g{}, impl(&g) {}
20+
GuardType g;
21+
Impl impl;
22+
23+
uint8_t first_byte() {
24+
uint8_t first;
25+
std::memcpy(&first, &g, 1);
26+
return first;
27+
}
28+
29+
void reset() { g = {}; }
30+
31+
public:
32+
// Test the post conditions on cxa_guard_acquire, cxa_guard_abort, and
33+
// cxa_guard_release. Specifically, that they leave the first byte with
34+
// the value 0 or 1 as specified by the ARM or Itanium specification.
35+
static void test() {
36+
Tests tests;
37+
tests.test_acquire();
38+
tests.test_abort();
39+
tests.test_release();
40+
}
41+
42+
void test_acquire() {
43+
{
44+
reset();
45+
assert(first_byte() == 0);
46+
assert(impl.cxa_guard_acquire() == INIT_IS_PENDING);
47+
assert(first_byte() == 0);
48+
}
49+
{
50+
reset();
51+
assert(first_byte() == 0);
52+
assert(impl.cxa_guard_acquire() == INIT_IS_PENDING);
53+
impl.cxa_guard_release();
54+
assert(first_byte() == 1);
55+
assert(impl.cxa_guard_acquire() == INIT_IS_DONE);
56+
}
57+
}
58+
59+
void test_release() {
60+
{
61+
reset();
62+
assert(first_byte() == 0);
63+
assert(impl.cxa_guard_acquire() == INIT_IS_PENDING);
64+
assert(first_byte() == 0);
65+
impl.cxa_guard_release();
66+
assert(first_byte() == 1);
67+
}
68+
}
69+
70+
void test_abort() {
71+
{
72+
reset();
73+
assert(first_byte() == 0);
74+
assert(impl.cxa_guard_acquire() == INIT_IS_PENDING);
75+
assert(first_byte() == 0);
76+
impl.cxa_guard_abort();
77+
assert(first_byte() == 0);
78+
assert(impl.cxa_guard_acquire() == INIT_IS_PENDING);
79+
assert(first_byte() == 0);
80+
}
81+
}
82+
};
83+
84+
struct NopMutex {
85+
bool lock() {
86+
assert(!is_locked);
87+
is_locked = true;
88+
return false;
89+
}
90+
bool unlock() {
91+
assert(is_locked);
92+
is_locked = false;
93+
return false;
94+
}
95+
96+
private:
97+
bool is_locked = false;
98+
};
99+
static NopMutex global_nop_mutex = {};
100+
101+
struct NopCondVar {
102+
bool broadcast() { return false; }
103+
bool wait(NopMutex&) { return false; }
104+
};
105+
static NopCondVar global_nop_cond = {};
106+
107+
void NopFutexWait(int*, int) { assert(false); }
108+
void NopFutexWake(int*) { assert(false); }
109+
uint32_t MockGetThreadID() { return 0; }
110+
111+
int main() {
112+
{
113+
#if defined(_LIBCXXABI_HAS_NO_THREADS)
114+
static_assert(CurrentImplementation == Implementation::NoThreads, "");
115+
static_assert(
116+
std::is_same<SelectedImplementation, InitByteNoThreads>::value, "");
117+
#else
118+
static_assert(CurrentImplementation == Implementation::GlobalLock, "");
119+
static_assert(
120+
std::is_same<
121+
SelectedImplementation,
122+
InitByteGlobalMutex<LibcppMutex, LibcppCondVar,
123+
GlobalStatic<LibcppMutex>::instance,
124+
GlobalStatic<LibcppCondVar>::instance>>::value,
125+
"");
126+
#endif
127+
}
128+
{
129+
#if defined(__APPLE__) || defined(__linux__)
130+
assert(PlatformThreadID);
131+
#endif
132+
if (+PlatformThreadID) {
133+
assert(PlatformThreadID() != 0);
134+
assert(PlatformThreadID() == PlatformThreadID());
135+
}
136+
}
137+
{
138+
Tests<uint32_t, InitByteNoThreads>::test();
139+
Tests<uint64_t, InitByteNoThreads>::test();
140+
}
141+
{
142+
using MutexImpl =
143+
InitByteGlobalMutex<NopMutex, NopCondVar, global_nop_mutex,
144+
global_nop_cond, MockGetThreadID>;
145+
Tests<uint32_t, MutexImpl>::test();
146+
Tests<uint64_t, MutexImpl>::test();
147+
}
148+
{
149+
using FutexImpl =
150+
InitByteFutex<&NopFutexWait, &NopFutexWake, &MockGetThreadID>;
151+
Tests<uint32_t, FutexImpl>::test();
152+
Tests<uint64_t, FutexImpl>::test();
153+
}
154+
}

‎libcxxabi/test/guard_threaded_test.pass.cpp

+378
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,378 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// UNSUPPORTED: c++98, c++03
10+
// UNSUPPORTED: libcxxabi-no-threads, libcxxabi-no-exceptions
11+
12+
#define TESTING_CXA_GUARD
13+
#include "../src/cxa_guard_impl.h"
14+
#include <unordered_map>
15+
#include <thread>
16+
#include <atomic>
17+
#include <array>
18+
#include <cassert>
19+
#include <memory>
20+
#include <vector>
21+
22+
23+
using namespace __cxxabiv1;
24+
25+
enum class InitResult {
26+
COMPLETE,
27+
PERFORMED,
28+
WAITED,
29+
ABORTED
30+
};
31+
constexpr InitResult COMPLETE = InitResult::COMPLETE;
32+
constexpr InitResult PERFORMED = InitResult::PERFORMED;
33+
constexpr InitResult WAITED = InitResult::WAITED;
34+
constexpr InitResult ABORTED = InitResult::ABORTED;
35+
36+
37+
template <class Impl, class GuardType, class Init>
38+
InitResult check_guard(GuardType *g, Init init) {
39+
uint8_t *first_byte = reinterpret_cast<uint8_t*>(g);
40+
if (std::__libcpp_atomic_load(first_byte, std::_AO_Acquire) == 0) {
41+
Impl impl(g);
42+
if (impl.cxa_guard_acquire() == INIT_IS_PENDING) {
43+
#ifndef LIBCXXABI_HAS_NO_EXCEPTIONS
44+
try {
45+
#endif
46+
init();
47+
impl.cxa_guard_release();
48+
return PERFORMED;
49+
#ifndef LIBCXXABI_HAS_NO_EXCEPTIONS
50+
} catch (...) {
51+
impl.cxa_guard_abort();
52+
return ABORTED;
53+
}
54+
#endif
55+
}
56+
return WAITED;
57+
}
58+
return COMPLETE;
59+
}
60+
61+
62+
template <class GuardType, class Impl>
63+
struct FunctionLocalStatic {
64+
FunctionLocalStatic() { reset(); }
65+
FunctionLocalStatic(FunctionLocalStatic const&) = delete;
66+
67+
template <class InitFunc>
68+
InitResult access(InitFunc&& init) {
69+
++waiting_threads;
70+
auto res = check_guard<Impl>(&guard_object, init);
71+
--waiting_threads;
72+
++result_counts[static_cast<int>(res)];
73+
return res;
74+
}
75+
76+
struct Accessor {
77+
explicit Accessor(FunctionLocalStatic& obj) : this_obj(&obj) {}
78+
79+
template <class InitFn>
80+
void operator()(InitFn && fn) const {
81+
this_obj->access(std::forward<InitFn>(fn));
82+
}
83+
private:
84+
FunctionLocalStatic *this_obj;
85+
};
86+
87+
Accessor get_access() {
88+
return Accessor(*this);
89+
}
90+
91+
void reset() {
92+
guard_object = 0;
93+
waiting_threads.store(0);
94+
for (auto& counter : result_counts) {
95+
counter.store(0);
96+
}
97+
}
98+
99+
int get_count(InitResult I) const {
100+
return result_counts[static_cast<int>(I)].load();
101+
}
102+
int num_completed() const {
103+
return get_count(COMPLETE) + get_count(PERFORMED) + get_count(WAITED);
104+
}
105+
int num_waiting() const {
106+
return waiting_threads.load();
107+
}
108+
109+
private:
110+
GuardType guard_object;
111+
std::atomic<int> waiting_threads;
112+
std::array<std::atomic<int>, 4> result_counts;
113+
static_assert(static_cast<int>(ABORTED) == 3, "only 4 result kinds expected");
114+
};
115+
116+
struct ThreadGroup {
117+
ThreadGroup() = default;
118+
ThreadGroup(ThreadGroup const&) = delete;
119+
120+
template <class ...Args>
121+
void Create(Args&& ...args) {
122+
threads.emplace_back(std::forward<Args>(args)...);
123+
}
124+
125+
void JoinAll() {
126+
for (auto& t : threads) {
127+
t.join();
128+
}
129+
}
130+
131+
private:
132+
std::vector<std::thread> threads;
133+
};
134+
135+
struct Barrier {
136+
explicit Barrier(int n) : m_wait_for(n) { reset(); }
137+
Barrier(Barrier const&) = delete;
138+
139+
void wait() {
140+
++m_entered;
141+
while (m_entered.load() < m_wait_for) {
142+
std::this_thread::yield();
143+
}
144+
assert(m_entered.load() == m_wait_for);
145+
++m_exited;
146+
}
147+
148+
int num_waiting() const {
149+
return m_entered.load() - m_exited.load();
150+
}
151+
152+
void reset() {
153+
m_entered.store(0);
154+
m_exited.store(0);
155+
}
156+
private:
157+
const int m_wait_for;
158+
std::atomic<int> m_entered;
159+
std::atomic<int> m_exited;
160+
};
161+
162+
struct Notification {
163+
Notification() { reset(); }
164+
Notification(Notification const&) = delete;
165+
166+
int num_waiting() const {
167+
return m_waiting.load();
168+
}
169+
170+
void wait() {
171+
if (m_cond.load())
172+
return;
173+
++m_waiting;
174+
while (!m_cond.load()) {
175+
std::this_thread::yield();
176+
}
177+
--m_waiting;
178+
}
179+
180+
void notify() {
181+
m_cond.store(true);
182+
}
183+
184+
template <class Cond>
185+
void notify_when(Cond &&c) {
186+
if (m_cond.load())
187+
return;
188+
while (!c()) {
189+
std::this_thread::yield();
190+
}
191+
m_cond.store(true);
192+
}
193+
194+
void reset() {
195+
m_cond.store(0);
196+
m_waiting.store(0);
197+
}
198+
private:
199+
std::atomic<bool> m_cond;
200+
std::atomic<int> m_waiting;
201+
};
202+
203+
204+
template <class GuardType, class Impl>
205+
void test_free_for_all() {
206+
const int num_waiting_threads = 10; // one initializing thread, 10 waiters.
207+
208+
FunctionLocalStatic<GuardType, Impl> test_obj;
209+
210+
Barrier start_init_barrier(num_waiting_threads);
211+
bool already_init = false;
212+
ThreadGroup threads;
213+
for (int i=0; i < num_waiting_threads; ++i) {
214+
threads.Create([&]() {
215+
start_init_barrier.wait();
216+
test_obj.access([&]() {
217+
assert(!already_init);
218+
already_init = true;
219+
});
220+
});
221+
}
222+
223+
// wait for the other threads to finish initialization.
224+
threads.JoinAll();
225+
226+
assert(test_obj.get_count(PERFORMED) == 1);
227+
assert(test_obj.get_count(COMPLETE) + test_obj.get_count(WAITED) == 9);
228+
}
229+
230+
template <class GuardType, class Impl>
231+
void test_waiting_for_init() {
232+
const int num_waiting_threads = 10; // one initializing thread, 10 waiters.
233+
234+
Notification init_pending;
235+
Notification init_barrier;
236+
FunctionLocalStatic<GuardType, Impl> test_obj;
237+
auto access_fn = test_obj.get_access();
238+
239+
ThreadGroup threads;
240+
threads.Create(access_fn,
241+
[&]() {
242+
init_pending.notify();
243+
init_barrier.wait();
244+
}
245+
);
246+
init_pending.wait();
247+
248+
assert(test_obj.num_waiting() == 1);
249+
250+
for (int i=0; i < num_waiting_threads; ++i) {
251+
threads.Create(access_fn, []() { assert(false); });
252+
}
253+
// unblock the initializing thread
254+
init_barrier.notify_when([&]() {
255+
return test_obj.num_waiting() == num_waiting_threads + 1;
256+
});
257+
258+
// wait for the other threads to finish initialization.
259+
threads.JoinAll();
260+
261+
assert(test_obj.get_count(PERFORMED) == 1);
262+
assert(test_obj.get_count(WAITED) == 10);
263+
assert(test_obj.get_count(COMPLETE) == 0);
264+
}
265+
266+
267+
template <class GuardType, class Impl>
268+
void test_aborted_init() {
269+
const int num_waiting_threads = 10; // one initializing thread, 10 waiters.
270+
271+
Notification init_pending;
272+
Notification init_barrier;
273+
FunctionLocalStatic<GuardType, Impl> test_obj;
274+
auto access_fn = test_obj.get_access();
275+
276+
ThreadGroup threads;
277+
threads.Create(access_fn,
278+
[&]() {
279+
init_pending.notify();
280+
init_barrier.wait();
281+
throw 42;
282+
}
283+
);
284+
init_pending.wait();
285+
286+
assert(test_obj.num_waiting() == 1);
287+
288+
bool already_init = false;
289+
for (int i=0; i < num_waiting_threads; ++i) {
290+
threads.Create(access_fn, [&]() {
291+
assert(!already_init);
292+
already_init = true;
293+
});
294+
}
295+
// unblock the initializing thread
296+
init_barrier.notify_when([&]() {
297+
return test_obj.num_waiting() == num_waiting_threads + 1;
298+
});
299+
300+
// wait for the other threads to finish initialization.
301+
threads.JoinAll();
302+
303+
assert(test_obj.get_count(ABORTED) == 1);
304+
assert(test_obj.get_count(PERFORMED) == 1);
305+
assert(test_obj.get_count(WAITED) == 9);
306+
assert(test_obj.get_count(COMPLETE) == 0);
307+
}
308+
309+
310+
template <class GuardType, class Impl>
311+
void test_completed_init() {
312+
const int num_waiting_threads = 10; // one initializing thread, 10 waiters.
313+
314+
Notification init_barrier;
315+
FunctionLocalStatic<GuardType, Impl> test_obj;
316+
317+
test_obj.access([]() {});
318+
assert(test_obj.num_waiting() == 0);
319+
assert(test_obj.num_completed() == 1);
320+
assert(test_obj.get_count(PERFORMED) == 1);
321+
322+
auto access_fn = test_obj.get_access();
323+
ThreadGroup threads;
324+
for (int i=0; i < num_waiting_threads; ++i) {
325+
threads.Create(access_fn, []() {
326+
assert(false);
327+
});
328+
}
329+
330+
// wait for the other threads to finish initialization.
331+
threads.JoinAll();
332+
333+
assert(test_obj.get_count(ABORTED) == 0);
334+
assert(test_obj.get_count(PERFORMED) == 1);
335+
assert(test_obj.get_count(WAITED) == 0);
336+
assert(test_obj.get_count(COMPLETE) == 10);
337+
}
338+
339+
template <class Impl>
340+
void test_impl() {
341+
{
342+
test_free_for_all<uint32_t, Impl>();
343+
test_free_for_all<uint32_t, Impl>();
344+
}
345+
{
346+
test_waiting_for_init<uint32_t, Impl>();
347+
test_waiting_for_init<uint64_t, Impl>();
348+
}
349+
{
350+
test_aborted_init<uint32_t, Impl>();
351+
test_aborted_init<uint64_t, Impl>();
352+
}
353+
{
354+
test_completed_init<uint32_t, Impl>();
355+
test_completed_init<uint64_t, Impl>();
356+
}
357+
}
358+
359+
int main() {
360+
using MutexImpl = SelectImplementation<Implementation::GlobalLock>::type;
361+
362+
// Attempt to test the Futex based implementation if it's supported on the
363+
// target platform.
364+
using RealFutexImpl = SelectImplementation<Implementation::Futex>::type;
365+
using FutexImpl = typename std::conditional<
366+
DoesPlatformSupportFutex(),
367+
RealFutexImpl,
368+
MutexImpl
369+
>::type;
370+
371+
// Run each test 5 times to help TSAN catch bugs.
372+
const int num_runs = 5;
373+
for (int i=0; i < num_runs; ++i) {
374+
test_impl<MutexImpl>();
375+
if (DoesPlatformSupportFutex())
376+
test_impl<FutexImpl>();
377+
}
378+
}

0 commit comments

Comments
 (0)
Please sign in to comment.