Index: include/llvm/Support/Threading.h =================================================================== --- include/llvm/Support/Threading.h +++ include/llvm/Support/Threading.h @@ -17,6 +17,7 @@ #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX #include "llvm/Support/Compiler.h" +#include #include // So we can check the C++ standard lib macros. #include @@ -33,7 +34,8 @@ #if LLVM_THREADING_USE_STD_CALL_ONCE #include #else -#include "llvm/Support/Atomic.h" +#include +#include #endif namespace llvm { @@ -67,11 +69,11 @@ #else enum InitStatus { Uninitialized = 0, Wait = 1, Done = 2 }; - typedef volatile sys::cas_flag once_flag; + typedef std::atomic once_flag; /// This macro is the only way you should define your once flag for LLVM's /// call_once. -#define LLVM_DEFINE_ONCE_FLAG(flag) static once_flag flag = Uninitialized +#define LLVM_DEFINE_ONCE_FLAG(flag) static once_flag flag = {Uninitialized} #endif @@ -88,31 +90,35 @@ /// \param flag Flag used for tracking whether or not this has run. /// \param F Function to call once. template - void call_once(once_flag &flag, Function &&F, Args &&... ArgList) { + void call_once(once_flag &Flag, Function &&F, Args &&... ArgList) { #if LLVM_THREADING_USE_STD_CALL_ONCE - std::call_once(flag, std::forward(F), + std::call_once(Flag, std::forward(F), std::forward(ArgList)...); #else // For other platforms we use a generic (if brittle) version based on our // atomics. - sys::cas_flag old_val = sys::CompareAndSwap(&flag, Wait, Uninitialized); - if (old_val == Uninitialized) { + InitStatus OldFlag = Uninitialized; + if (Flag.compare_exchange_strong(OldFlag, Wait, std::memory_order_acq_rel)) { + // We've moved the flag to the Wait state which means we have exclusive + // access to call the function. std::forward(F)(std::forward(ArgList)...); - sys::MemoryFence(); - TsanIgnoreWritesBegin(); - TsanHappensBefore(&flag); - flag = Done; - TsanIgnoreWritesEnd(); - } else { - // Wait until any thread doing the call has finished. - sys::cas_flag tmp = flag; - sys::MemoryFence(); - while (tmp != Done) { - tmp = flag; - sys::MemoryFence(); - } + // Now release any other threads spinning and mark that no other threads need to acquire + Flag.store(Done, std::memory_order_release); + return; } - TsanHappensAfter(&flag); + assert(OldFlag != Uninitialized && + "compare_exchange_strong failed to update!"); + + // If we already acquired the done state, no need to spin (and do more + // atomic operations). + if (OldFlag == Done) + return; + + // Otherwise we're in the wait state, so spin until we acquire the done + // state. + assert(OldFlag == Wait && "Corrupted flag value!"); + while (Flag.load(std::memory_order_acquire) != Done) + std::this_thread::yield(); #endif } }