Replace the Execution Engine's mutex with std::recursive_mutex

[zturner]

Summary

This replaces the EE's mutex with std::recursive_mutex.

This change has a bit of a trickle-down effect, due to the fact that the EE's mutex is not self-contained, and is used by a number of derived implementations, as well as exposed and used directly by the ValueMap class. So all of these must be changed at once in the same CL, and there's no good way to break it up.

[rnk]

lgtm

[zturner]

Closed by commit rL211214 (authored by @zturner).

[vadimcn]

FYI - this commit broke LLVM build using win32 threads flavor of the mingw toolchain. I am getting error: 'recursive_mutex' in namespace 'std' does not name a type.
Not sure if this would be considered a problem for LLVM...

[zturner]

+llvmdev.

I find this pretty surprising. Actually, we already use std::mutex and
std::recursive_mutex in clang, lld, and other llvm projects, it's just a
coincidence that it hadn't been introduced into LLVM until my commits.

I'm not sure what the right thing to do here is. If I understand
correctly, it seems like in order to encounter this, a) you must be using
GCC, b) you must be using the MinGW flavor of GCC, and c) you must be using
the threads-win32 flavor of this toolchain. Only if all 3 of those are
true, then std::mutex and std::recursive_mutex don't exist.

Anybody else have thoughts on whether this necessitates reverting the mutex
changes, or whether this toolchain configuration should be supported?

llvm/trunk/include/llvm/ExecutionEngine/ExecutionEngine.h

 
 #include "llvm-c/ExecutionEngine.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/IR/ValueMap.h"
 #include "llvm/MC/MCCodeGenInfo.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Mutex.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include <map>
+#include <mutex>
 #include <string>
 #include <vector>
 
 namespace llvm {
 
 struct GenericValue;
 class Constant;
 class DataLayout;
 class ExecutionEngine;
 class Function;
 class GlobalVariable;
 class GlobalValue;
 class JITEventListener;
 class JITMemoryManager;
 class MachineCodeInfo;
 class Module;
-class MutexGuard;
 class ObjectCache;
 class RTDyldMemoryManager;
 class Triple;
 class Type;
 
 namespace object {
   class Archive;
   class ObjectFile;
 }
 
 /// \brief Helper class for helping synchronize access to the global address map
 /// table.  Access to this class should be serialized under a mutex.
 class ExecutionEngineState {
 public:
   struct AddressMapConfig : public ValueMapConfig<const GlobalValue*> {
     typedef ExecutionEngineState *ExtraData;
-    static sys::Mutex *getMutex(ExecutionEngineState *EES);
+    static std::recursive_mutex *getMutex(ExecutionEngineState *EES);
     static void onDelete(ExecutionEngineState *EES, const GlobalValue *Old);
     static void onRAUW(ExecutionEngineState *, const GlobalValue *,
                        const GlobalValue *);
   };
 
   typedef ValueMap<const GlobalValue *, void *, AddressMapConfig>
       GlobalAddressMapTy;
 
protected:
   /// pointer is invoked to create it.  If this returns null, the JIT will
   /// abort.
   void *(*LazyFunctionCreator)(const std::string &);
 
 public:
   /// lock - This lock protects the ExecutionEngine, MCJIT, JIT, JITResolver and
   /// JITEmitter classes.  It must be held while changing the internal state of
   /// any of those classes.
-  sys::Mutex lock;
+  std::recursive_mutex lock;
 
   //===--------------------------------------------------------------------===//
   //  ExecutionEngine Startup
   //===--------------------------------------------------------------------===//
 
   virtual ~ExecutionEngine();
 
   /// create - This is the factory method for creating an execution engine which

llvm/trunk/include/llvm/IR/ValueMap.h

 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_IR_VALUEMAP_H
 #define LLVM_IR_VALUEMAP_H
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/IR/ValueHandle.h"
-#include "llvm/Support/Mutex.h"
 #include "llvm/Support/type_traits.h"
 #include <iterator>
+#include <mutex>
 
 namespace llvm {
 
 template<typename KeyT, typename ValueT, typename Config>
 class ValueMapCallbackVH;
 
 template<typename DenseMapT, typename KeyT>
 class ValueMapIterator;
 template<typename DenseMapT, typename KeyT>
 class ValueMapConstIterator;
 
 /// This class defines the default behavior for configurable aspects of
 /// ValueMap<>.  User Configs should inherit from this class to be as compatible
 /// as possible with future versions of ValueMap.
-template<typename KeyT, typename MutexT = sys::Mutex>
+template<typename KeyT, typename MutexT = std::recursive_mutex>
 struct ValueMapConfig {
   typedef MutexT mutex_type;
 
   /// If FollowRAUW is true, the ValueMap will update mappings on RAUW. If it's
   /// false, the ValueMap will leave the original mapping in place.
   enum { FollowRAUW = true };
 
   // All methods will be called with a first argument of type ExtraData.  The

 public:
   KeyT Unwrap() const { return cast_or_null<KeySansPointerT>(getValPtr()); }
 
   void deleted() override {
     // Make a copy that won't get changed even when *this is destroyed.
     ValueMapCallbackVH Copy(*this);
     typename Config::mutex_type *M = Config::getMutex(Copy.Map->Data);
     if (M)
-      M->acquire();
+      M->lock();
     Config::onDelete(Copy.Map->Data, Copy.Unwrap());  // May destroy *this.
     Copy.Map->Map.erase(Copy);  // Definitely destroys *this.
     if (M)
-      M->release();
+      M->unlock();
   }
   void allUsesReplacedWith(Value *new_key) override {
     assert(isa<KeySansPointerT>(new_key) &&
            "Invalid RAUW on key of ValueMap<>");
     // Make a copy that won't get changed even when *this is destroyed.
     ValueMapCallbackVH Copy(*this);
     typename Config::mutex_type *M = Config::getMutex(Copy.Map->Data);
     if (M)
-      M->acquire();
+      M->lock();
 
     KeyT typed_new_key = cast<KeySansPointerT>(new_key);
     // Can destroy *this:
     Config::onRAUW(Copy.Map->Data, Copy.Unwrap(), typed_new_key);
     if (Config::FollowRAUW) {
       typename ValueMapT::MapT::iterator I = Copy.Map->Map.find(Copy);
       // I could == Copy.Map->Map.end() if the onRAUW callback already
       // removed the old mapping.
       if (I != Copy.Map->Map.end()) {
         ValueT Target(I->second);
         Copy.Map->Map.erase(I);  // Definitely destroys *this.
         Copy.Map->insert(std::make_pair(typed_new_key, Target));
       }
     }
     if (M)
-      M->release();
+      M->unlock();
   }
 };
 
 template<typename KeyT, typename ValueT, typename Config>
 struct DenseMapInfo<ValueMapCallbackVH<KeyT, ValueT, Config> > {
   typedef ValueMapCallbackVH<KeyT, ValueT, Config> VH;
   typedef DenseMapInfo<KeyT> PointerInfo;
 

llvm/trunk/lib/ExecutionEngine/ExecutionEngine.cpp

else {
     GlobalAddressMap.erase(I);
   }
 
   GlobalAddressReverseMap.erase(OldVal);
   return OldVal;
 }
 
 void ExecutionEngine::addGlobalMapping(const GlobalValue *GV, void *Addr) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   DEBUG(dbgs() << "JIT: Map \'" << GV->getName()
         << "\' to [" << Addr << "]\n";);
   void *&CurVal = EEState.getGlobalAddressMap()[GV];
   assert((!CurVal || !Addr) && "GlobalMapping already established!");
   CurVal = Addr;
 
   // If we are using the reverse mapping, add it too.
   if (!EEState.getGlobalAddressReverseMap().empty()) {
     AssertingVH<const GlobalValue> &V =
       EEState.getGlobalAddressReverseMap()[Addr];
     assert((!V || !GV) && "GlobalMapping already established!");
     V = GV;
   }
 }
 
 void ExecutionEngine::clearAllGlobalMappings() {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   EEState.getGlobalAddressMap().clear();
   EEState.getGlobalAddressReverseMap().clear();
 }
 
 void ExecutionEngine::clearGlobalMappingsFromModule(Module *M) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   for (Module::iterator FI = M->begin(), FE = M->end(); FI != FE; ++FI)
     EEState.RemoveMapping(FI);
   for (Module::global_iterator GI = M->global_begin(), GE = M->global_end();
        GI != GE; ++GI)
     EEState.RemoveMapping(GI);
 }
 
 void *ExecutionEngine::updateGlobalMapping(const GlobalValue *GV, void *Addr) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   ExecutionEngineState::GlobalAddressMapTy &Map =
     EEState.getGlobalAddressMap();
 
   // Deleting from the mapping?
   if (!Addr)
     return EEState.RemoveMapping(GV);
 
AssertingVH<const GlobalValue> &V =
       EEState.getGlobalAddressReverseMap()[Addr];
     assert((!V || !GV) && "GlobalMapping already established!");
     V = GV;
   }
   return OldVal;
 }
 
 void *ExecutionEngine::getPointerToGlobalIfAvailable(const GlobalValue *GV) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   ExecutionEngineState::GlobalAddressMapTy::iterator I =
     EEState.getGlobalAddressMap().find(GV);
   return I != EEState.getGlobalAddressMap().end() ? I->second : nullptr;
 }
 
 const GlobalValue *ExecutionEngine::getGlobalValueAtAddress(void *Addr) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   // If we haven't computed the reverse mapping yet, do so first.
   if (EEState.getGlobalAddressReverseMap().empty()) {
     for (ExecutionEngineState::GlobalAddressMapTy::iterator
          I = EEState.getGlobalAddressMap().begin(),
          E = EEState.getGlobalAddressMap().end(); I != E; ++I)
       EEState.getGlobalAddressReverseMap().insert(std::make_pair(
                                                           I->second, I->first));
ExecutionEngine *EngineBuilder::create(TargetMachine *TM) {
 
   return nullptr;
 }
 
 void *ExecutionEngine::getPointerToGlobal(const GlobalValue *GV) {
   if (Function *F = const_cast<Function*>(dyn_cast<Function>(GV)))
     return getPointerToFunction(F);
 
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   if (void *P = EEState.getGlobalAddressMap()[GV])
     return P;
 
   // Global variable might have been added since interpreter started.
   if (GlobalVariable *GVar =
           const_cast<GlobalVariable *>(dyn_cast<GlobalVariable>(GV)))
     EmitGlobalVariable(GVar);
   else
void ExecutionEngine::EmitGlobalVariable(const GlobalVariable *GV) {
   NumInitBytes += (unsigned)GVSize;
   ++NumGlobals;
 }
 
 ExecutionEngineState::ExecutionEngineState(ExecutionEngine &EE)
   : EE(EE), GlobalAddressMap(this) {
 }
 
-sys::Mutex *
+std::recursive_mutex *
 ExecutionEngineState::AddressMapConfig::getMutex(ExecutionEngineState *EES) {
   return &EES->EE.lock;
 }
 
 void ExecutionEngineState::AddressMapConfig::onDelete(ExecutionEngineState *EES,
                                                       const GlobalValue *Old) {
   void *OldVal = EES->GlobalAddressMap.lookup(Old);
   EES->GlobalAddressReverseMap.erase(OldVal);
 }
 
 void ExecutionEngineState::AddressMapConfig::onRAUW(ExecutionEngineState *,
                                                     const GlobalValue *,
                                                     const GlobalValue *) {
   llvm_unreachable("The ExecutionEngine doesn't know how to handle a"
                    " RAUW on a value it has a global mapping for.");
 }

llvm/trunk/lib/ExecutionEngine/JIT/JIT.cpp

ExecutionEngine *JIT::createJIT(Module *M,
   }
 }
 
 namespace {
 /// This class supports the global getPointerToNamedFunction(), which allows
 /// bugpoint or gdb users to search for a function by name without any context.
 class JitPool {
   SmallPtrSet<JIT*, 1> JITs;  // Optimize for process containing just 1 JIT.
-  mutable sys::Mutex Lock;
+  mutable std::recursive_mutex Lock;
 public:
   void Add(JIT *jit) {
-    MutexGuard guard(Lock);
+    std::lock_guard<std::recursive_mutex> guard(Lock);
     JITs.insert(jit);
   }
   void Remove(JIT *jit) {
-    MutexGuard guard(Lock);
+    std::lock_guard<std::recursive_mutex> guard(Lock);
     JITs.erase(jit);
   }
   void *getPointerToNamedFunction(const char *Name) const {
-    MutexGuard guard(Lock);
+    std::lock_guard<std::recursive_mutex> guard(Lock);
     assert(JITs.size() != 0 && "No Jit registered");
     //search function in every instance of JIT
     for (SmallPtrSet<JIT*, 1>::const_iterator Jit = JITs.begin(),
            end = JITs.end();
          Jit != end; ++Jit) {
       if (Function *F = (*Jit)->FindFunctionNamed(Name))
         return (*Jit)->getPointerToFunction(F);
     }
JIT::JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji,
 
   // Initialize JCE
   JCE = createEmitter(*this, JMM, TM);
 
   // Register in global list of all JITs.
   AllJits->Add(this);
 
   // Add target data
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   FunctionPassManager &PM = jitstate->getPM();
   M->setDataLayout(TM.getDataLayout());
   PM.add(new DataLayoutPass(M));
 
   // Turn the machine code intermediate representation into bytes in memory that
   // may be executed.
   if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
     report_fatal_error("Target does not support machine code emission!");
JIT::~JIT() {
   delete JCE;
   // JMM is a ownership of JCE, so we no need delete JMM here.
   delete &TM;
 }
 
 /// addModule - Add a new Module to the JIT.  If we previously removed the last
 /// Module, we need re-initialize jitstate with a valid Module.
 void JIT::addModule(Module *M) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   if (Modules.empty()) {
     assert(!jitstate && "jitstate should be NULL if Modules vector is empty!");
 
     jitstate = new JITState(M);
 
     FunctionPassManager &PM = jitstate->getPM();
     M->setDataLayout(TM.getDataLayout());
void JIT::addModule(Module *M) {
   ExecutionEngine::addModule(M);
 }
 
 /// removeModule - If we are removing the last Module, invalidate the jitstate
 /// since the PassManager it contains references a released Module.
 bool JIT::removeModule(Module *M) {
   bool result = ExecutionEngine::removeModule(M);
 
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   if (jitstate && jitstate->getModule() == M) {
     delete jitstate;
     jitstate = nullptr;
   }
 
   if (!jitstate && !Modules.empty()) {
     jitstate = new JITState(Modules[0]);
GenericValue JIT::runFunction(Function *F,
   Stub->eraseFromParent();
   // And return the result.
   return Result;
 }
 
 void JIT::RegisterJITEventListener(JITEventListener *L) {
   if (!L)
     return;
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   EventListeners.push_back(L);
 }
 void JIT::UnregisterJITEventListener(JITEventListener *L) {
   if (!L)
     return;
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   std::vector<JITEventListener*>::reverse_iterator I=
       std::find(EventListeners.rbegin(), EventListeners.rend(), L);
   if (I != EventListeners.rend()) {
     std::swap(*I, EventListeners.back());
     EventListeners.pop_back();
   }
 }
 void JIT::NotifyFunctionEmitted(
     const Function &F,
     void *Code, size_t Size,
     const JITEvent_EmittedFunctionDetails &Details) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
     EventListeners[I]->NotifyFunctionEmitted(F, Code, Size, Details);
   }
 }
 
 void JIT::NotifyFreeingMachineCode(void *OldPtr) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
     EventListeners[I]->NotifyFreeingMachineCode(OldPtr);
   }
 }
 
 /// runJITOnFunction - Run the FunctionPassManager full of
 /// just-in-time compilation passes on F, hopefully filling in
 /// GlobalAddress[F] with the address of F's machine code.
 ///
 void JIT::runJITOnFunction(Function *F, MachineCodeInfo *MCI) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   class MCIListener : public JITEventListener {
     MachineCodeInfo *const MCI;
    public:
     MCIListener(MachineCodeInfo *mci) : MCI(mci) {}
     void NotifyFunctionEmitted(const Function &, void *Code, size_t Size,
                                const EmittedFunctionDetails &) override {
       MCI->setAddress(Code);

 /// getPointerToFunction - This method is used to get the address of the
 /// specified function, compiling it if necessary.
 ///
 void *JIT::getPointerToFunction(Function *F) {
 
   if (void *Addr = getPointerToGlobalIfAvailable(F))
     return Addr;   // Check if function already code gen'd
 
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   // Now that this thread owns the lock, make sure we read in the function if it
   // exists in this Module.
   std::string ErrorMsg;
   if (F->Materialize(&ErrorMsg)) {
     report_fatal_error("Error reading function '" + F->getName()+
                       "' from bitcode file: " + ErrorMsg);
   }
void *JIT::getPointerToFunction(Function *F) {
   runJITOnFunctionUnlocked(F);
 
   void *Addr = getPointerToGlobalIfAvailable(F);
   assert(Addr && "Code generation didn't add function to GlobalAddress table!");
   return Addr;
 }
 
 void JIT::addPointerToBasicBlock(const BasicBlock *BB, void *Addr) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   BasicBlockAddressMapTy::iterator I =
     getBasicBlockAddressMap().find(BB);
   if (I == getBasicBlockAddressMap().end()) {
     getBasicBlockAddressMap()[BB] = Addr;
   } else {
     // ignore repeats: some BBs can be split into few MBBs?
   }
 }
 
 void JIT::clearPointerToBasicBlock(const BasicBlock *BB) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   getBasicBlockAddressMap().erase(BB);
 }
 
 void *JIT::getPointerToBasicBlock(BasicBlock *BB) {
   // make sure it's function is compiled by JIT
   (void)getPointerToFunction(BB->getParent());
 
   // resolve basic block address
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   BasicBlockAddressMapTy::iterator I =
     getBasicBlockAddressMap().find(BB);
   if (I != getBasicBlockAddressMap().end()) {
     return I->second;
   } else {
     llvm_unreachable("JIT does not have BB address for address-of-label, was"
                      " it eliminated by optimizer?");
void *JIT::getPointerToNamedFunction(const std::string &Name,
   return nullptr;
 }
 
 
 /// getOrEmitGlobalVariable - Return the address of the specified global
 /// variable, possibly emitting it to memory if needed.  This is used by the
 /// Emitter.
 void *JIT::getOrEmitGlobalVariable(const GlobalVariable *GV) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   void *Ptr = getPointerToGlobalIfAvailable(GV);
   if (Ptr) return Ptr;
 
   // If the global is external, just remember the address.
   if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) {
 #if HAVE___DSO_HANDLE
     if (GV->getName() == "__dso_handle")
char* JIT::getMemoryForGV(const GlobalVariable* GV) {
   // Some applications require globals and code to live together, so they may
   // be allocated into the same buffer, but in general globals are allocated
   // through the memory manager which puts them near the code but not in the
   // same buffer.
   Type *GlobalType = GV->getType()->getElementType();
   size_t S = getDataLayout()->getTypeAllocSize(GlobalType);
   size_t A = getDataLayout()->getPreferredAlignment(GV);
   if (GV->isThreadLocal()) {
-    MutexGuard locked(lock);
+    std::lock_guard<std::recursive_mutex> locked(lock);
     Ptr = TJI.allocateThreadLocalMemory(S);
   } else if (TJI.allocateSeparateGVMemory()) {
     if (A <= 8) {
       Ptr = (char*)malloc(S);
     } else {
       // Allocate S+A bytes of memory, then use an aligned pointer within that
       // space.
       Ptr = (char*)malloc(S+A);
       unsigned MisAligned = ((intptr_t)Ptr & (A-1));
       Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0);
     }
   } else if (AllocateGVsWithCode) {
     Ptr = (char*)JCE->allocateSpace(S, A);
   } else {
     Ptr = (char*)JCE->allocateGlobal(S, A);
   }
   return Ptr;
 }
 
 void JIT::addPendingFunction(Function *F) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   jitstate->getPendingFunctions().push_back(F);
 }
 
 
 JITEventListener::~JITEventListener() {}

llvm/trunk/lib/ExecutionEngine/JIT/JITEmitter.cpp

 #include "llvm/IR/Operator.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/IR/ValueMap.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Disassembler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Memory.h"
-#include "llvm/Support/MutexGuard.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetJITInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include <algorithm>
 #ifndef NDEBUG
 #include <iomanip>
 #endif
+#include <mutex>
 using namespace llvm;
 
 #define DEBUG_TYPE "jit"
 
 STATISTIC(NumBytes, "Number of bytes of machine code compiled");
 STATISTIC(NumRelos, "Number of relocations applied");
 STATISTIC(NumRetries, "Number of retries with more memory");
 
#endif
   class StubToResolverMapTy {
     /// Map a stub address to a specific instance of a JITResolver so that
     /// lazily-compiled functions can find the right resolver to use.
     ///
     /// Guarded by Lock.
     std::map<void*, JITResolver*> Map;
 
     /// Guards Map from concurrent accesses.
-    mutable sys::Mutex Lock;
+    mutable std::recursive_mutex Lock;
 
   public:
     /// Registers a Stub to be resolved by Resolver.
     void RegisterStubResolver(void *Stub, JITResolver *Resolver) {
-      MutexGuard guard(Lock);
+      std::lock_guard<std::recursive_mutex> guard(Lock);
       Map.insert(std::make_pair(Stub, Resolver));
     }
     /// Unregisters the Stub when it's invalidated.
     void UnregisterStubResolver(void *Stub) {
-      MutexGuard guard(Lock);
+      std::lock_guard<std::recursive_mutex> guard(Lock);
       Map.erase(Stub);
     }
     /// Returns the JITResolver instance that owns the Stub.
     JITResolver *getResolverFromStub(void *Stub) const {
-      MutexGuard guard(Lock);
+      std::lock_guard<std::recursive_mutex> guard(Lock);
       // The address given to us for the stub may not be exactly right, it might
       // be a little bit after the stub.  As such, use upper_bound to find it.
       // This is the same trick as in LookupFunctionFromCallSite from
       // JITResolverState.
       std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub);
       assert(I != Map.begin() && "This is not a known stub!");
       --I;
       return I->second;
     }
     /// True if any stubs refer to the given resolver. Only used in an assert().
     /// O(N)
     bool ResolverHasStubs(JITResolver* Resolver) const {
-      MutexGuard guard(Lock);
+      std::lock_guard<std::recursive_mutex> guard(Lock);
       for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(),
              E = Map.end(); I != E; ++I) {
         if (I->second == Resolver)
           return true;
       }
       return false;
     }
   };
JITResolver::~JITResolver() {
   state.EraseAllCallSitesPrelocked();
   assert(!StubToResolverMap->ResolverHasStubs(this) &&
          "Resolver destroyed with stubs still alive.");
 }
 
 /// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub
 /// if it has already been created.
 void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) {
-  MutexGuard locked(TheJIT->lock);
+  std::lock_guard<std::recursive_mutex> guard(TheJIT->lock);
 
   // If we already have a stub for this function, recycle it.
   return state.getFunctionToLazyStubMap().lookup(F);
 }
 
 /// getFunctionStub - This returns a pointer to a function stub, creating
 /// one on demand as needed.
 void *JITResolver::getLazyFunctionStub(Function *F) {
-  MutexGuard locked(TheJIT->lock);
+  std::lock_guard<std::recursive_mutex> guard(TheJIT->lock);
 
   // If we already have a lazy stub for this function, recycle it.
   void *&Stub = state.getFunctionToLazyStubMap()[F];
   if (Stub) return Stub;
 
   // Call the lazy resolver function if we are JIT'ing lazily.  Otherwise we
   // must resolve the symbol now.
   void *Actual = TheJIT->isCompilingLazily()
void *JITResolver::getLazyFunctionStub(Function *F) {
   }
 
   return Stub;
 }
 
 /// getGlobalValueIndirectSym - Return a lazy pointer containing the specified
 /// GV address.
 void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) {
-  MutexGuard locked(TheJIT->lock);
+  std::lock_guard<std::recursive_mutex> guard(TheJIT->lock);
 
   // If we already have a stub for this global variable, recycle it.
   void *&IndirectSym = state.getGlobalToIndirectSymMap()[GV];
   if (IndirectSym) return IndirectSym;
 
   // Otherwise, codegen a new indirect symbol.
   IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress,
                                                                 JE);
void *JITResolver::JITCompilerFn(void *Stub) {
 
   Function* F = nullptr;
   void* ActualPtr = nullptr;
 
   {
     // Only lock for getting the Function. The call getPointerToFunction made
     // in this function might trigger function materializing, which requires
     // JIT lock to be unlocked.
-    MutexGuard locked(JR->TheJIT->lock);
+    std::lock_guard<std::recursive_mutex> guard(JR->TheJIT->lock);
 
     // The address given to us for the stub may not be exactly right, it might
     // be a little bit after the stub.  As such, use upper_bound to find it.
     std::pair<void*, Function*> I =
       JR->state.LookupFunctionFromCallSite(Stub);
     F = I.second;
     ActualPtr = I.first;
   }
DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName()
           << "' In stub ptr = " << Stub << " actual ptr = "
           << ActualPtr << "\n");
     (void)ActualPtr;
 
     Result = JR->TheJIT->getPointerToFunction(F);
   }
 
   // Reacquire the lock to update the GOT map.
-  MutexGuard locked(JR->TheJIT->lock);
+  std::lock_guard<std::recursive_mutex> locked(JR->TheJIT->lock);
 
   // We might like to remove the call site from the CallSiteToFunction map, but
   // we can't do that! Multiple threads could be stuck, waiting to acquire the
   // lock above. As soon as the 1st function finishes compiling the function,
   // the next one will be released, and needs to be able to find the function it
   // needs to call.
 
   // FIXME: We could rewrite all references to this stub if we knew them.

llvm/trunk/lib/ExecutionEngine/MCJIT/MCJIT.cpp

 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Module.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/PassManager.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/MutexGuard.h"
 #include "llvm/Target/TargetLowering.h"
 
 using namespace llvm;
 
 namespace {
 
 static struct RegisterJIT {
   RegisterJIT() { MCJIT::Register(); }
MCJIT::MCJIT(Module *m, TargetMachine *tm, RTDyldMemoryManager *MM,
   : ExecutionEngine(m), TM(tm), Ctx(nullptr), MemMgr(this, MM), Dyld(&MemMgr),
     ObjCache(nullptr) {
 
   OwnedModules.addModule(m);
   setDataLayout(TM->getDataLayout());
 }
 
 MCJIT::~MCJIT() {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   // FIXME: We are managing our modules, so we do not want the base class
   // ExecutionEngine to manage them as well. To avoid double destruction
   // of the first (and only) module added in ExecutionEngine constructor
   // we remove it from EE and will destruct it ourselves.
   //
   // It may make sense to move our module manager (based on SmallStPtr) back
   // into EE if the JIT and Interpreter can live with it.
   // If so, additional functions: addModule, removeModule, FindFunctionNamed,
for (ArIt = Archives.begin(), ArEnd = Archives.end(); ArIt != ArEnd; ++ArIt) {
     delete A;
   }
   Archives.clear();
 
   delete TM;
 }
 
 void MCJIT::addModule(Module *M) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   OwnedModules.addModule(M);
 }
 
 bool MCJIT::removeModule(Module *M) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   return OwnedModules.removeModule(M);
 }
 
 
 
 void MCJIT::addObjectFile(std::unique_ptr<object::ObjectFile> Obj) {
   ObjectImage *LoadedObject = Dyld.loadObject(std::move(Obj));
   if (!LoadedObject || Dyld.hasError())
     report_fatal_error(Dyld.getErrorString());
 
   LoadedObjects.push_back(LoadedObject);
 
   NotifyObjectEmitted(*LoadedObject);
 }
 
 void MCJIT::addArchive(object::Archive *A) {
   Archives.push_back(A);
 }
 
 
 void MCJIT::setObjectCache(ObjectCache* NewCache) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   ObjCache = NewCache;
 }
 
 ObjectBufferStream* MCJIT::emitObject(Module *M) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   // This must be a module which has already been added but not loaded to this
   // MCJIT instance, since these conditions are tested by our caller,
   // generateCodeForModule.
 
   PassManager PM;
 
   M->setDataLayout(TM->getDataLayout());
if (ObjCache) {
     ObjCache->notifyObjectCompiled(M, MB.get());
   }
 
   return CompiledObject.release();
 }
 
 void MCJIT::generateCodeForModule(Module *M) {
   // Get a thread lock to make sure we aren't trying to load multiple times
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   // This must be a module which has already been added to this MCJIT instance.
   assert(OwnedModules.ownsModule(M) &&
          "MCJIT::generateCodeForModule: Unknown module.");
 
   // Re-compilation is not supported
   if (OwnedModules.hasModuleBeenLoaded(M))
     return;
void MCJIT::generateCodeForModule(Module *M) {
   LoadedObject->registerWithDebugger();
 
   NotifyObjectEmitted(*LoadedObject);
 
   OwnedModules.markModuleAsLoaded(M);
 }
 
 void MCJIT::finalizeLoadedModules() {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   // Resolve any outstanding relocations.
   Dyld.resolveRelocations();
 
   OwnedModules.markAllLoadedModulesAsFinalized();
 
   // Register EH frame data for any module we own which has been loaded
   Dyld.registerEHFrames();
 
   // Set page permissions.
   MemMgr.finalizeMemory();
 }
 
 // FIXME: Rename this.
 void MCJIT::finalizeObject() {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   for (ModulePtrSet::iterator I = OwnedModules.begin_added(),
                               E = OwnedModules.end_added();
        I != E; ++I) {
     Module *M = *I;
     generateCodeForModule(M);
   }
 
   finalizeLoadedModules();
 }
 
 void MCJIT::finalizeModule(Module *M) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   // This must be a module which has already been added to this MCJIT instance.
   assert(OwnedModules.ownsModule(M) && "MCJIT::finalizeModule: Unknown module.");
 
   // If the module hasn't been compiled, just do that.
   if (!OwnedModules.hasModuleBeenLoaded(M))
     generateCodeForModule(M);
 
   finalizeLoadedModules();
 }
 
 void *MCJIT::getPointerToBasicBlock(BasicBlock *BB) {
   report_fatal_error("not yet implemented");
 }
 
 uint64_t MCJIT::getExistingSymbolAddress(const std::string &Name) {
   Mangler Mang(TM->getDataLayout());
   SmallString<128> FullName;
   Mang.getNameWithPrefix(FullName, Name);
   return Dyld.getSymbolLoadAddress(FullName);
 }
 
 Module *MCJIT::findModuleForSymbol(const std::string &Name,
                                    bool CheckFunctionsOnly) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   // If it hasn't already been generated, see if it's in one of our modules.
   for (ModulePtrSet::iterator I = OwnedModules.begin_added(),
                               E = OwnedModules.end_added();
        I != E; ++I) {
     Module *M = *I;
     Function *F = M->getFunction(Name);
     if (F && !F->isDeclaration())
       return M;
     if (!CheckFunctionsOnly) {
       GlobalVariable *G = M->getGlobalVariable(Name);
       if (G && !G->isDeclaration())
         return M;
       // FIXME: Do we need to worry about global aliases?
     }
   }
   // We didn't find the symbol in any of our modules.
   return nullptr;
 }
 
 uint64_t MCJIT::getSymbolAddress(const std::string &Name,
                                  bool CheckFunctionsOnly)
 {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   // First, check to see if we already have this symbol.
   uint64_t Addr = getExistingSymbolAddress(Name);
   if (Addr)
     return Addr;
 
   SmallVector<object::Archive*, 2>::iterator I, E;
   for (I = Archives.begin(), E = Archives.end(); I != E; ++I) {
uint64_t MCJIT::getSymbolAddress(const std::string &Name,
 
   generateCodeForModule(M);
 
   // Check the RuntimeDyld table again, it should be there now.
   return getExistingSymbolAddress(Name);
 }
 
 uint64_t MCJIT::getGlobalValueAddress(const std::string &Name) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   uint64_t Result = getSymbolAddress(Name, false);
   if (Result != 0)
     finalizeLoadedModules();
   return Result;
 }
 
 uint64_t MCJIT::getFunctionAddress(const std::string &Name) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   uint64_t Result = getSymbolAddress(Name, true);
   if (Result != 0)
     finalizeLoadedModules();
   return Result;
 }
 
 // Deprecated.  Use getFunctionAddress instead.
 void *MCJIT::getPointerToFunction(Function *F) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
 
   if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) {
     bool AbortOnFailure = !F->hasExternalWeakLinkage();
     void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure);
     addGlobalMapping(F, Addr);
     return Addr;
   }
 
report_fatal_error("Program used external function '"+Name+
                        "' which could not be resolved!");
   }
   return nullptr;
 }
 
 void MCJIT::RegisterJITEventListener(JITEventListener *L) {
   if (!L)
     return;
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   EventListeners.push_back(L);
 }
 void MCJIT::UnregisterJITEventListener(JITEventListener *L) {
   if (!L)
     return;
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   SmallVector<JITEventListener*, 2>::reverse_iterator I=
       std::find(EventListeners.rbegin(), EventListeners.rend(), L);
   if (I != EventListeners.rend()) {
     std::swap(*I, EventListeners.back());
     EventListeners.pop_back();
   }
 }
 void MCJIT::NotifyObjectEmitted(const ObjectImage& Obj) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   MemMgr.notifyObjectLoaded(this, &Obj);
   for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
     EventListeners[I]->NotifyObjectEmitted(Obj);
   }
 }
 void MCJIT::NotifyFreeingObject(const ObjectImage& Obj) {
-  MutexGuard locked(lock);
+  std::lock_guard<std::recursive_mutex> locked(lock);
   for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
     EventListeners[I]->NotifyFreeingObject(Obj);
   }
 }
 
 uint64_t LinkingMemoryManager::getSymbolAddress(const std::string &Name) {
   uint64_t Result = ParentEngine->getSymbolAddress(Name, false);
   // If the symbols wasn't found and it begins with an underscore, try again
   // without the underscore.
   if (!Result && Name[0] == '_')
     Result = ParentEngine->getSymbolAddress(Name.substr(1), false);
   if (Result)
     return Result;
   return ClientMM->getSymbolAddress(Name);
 }

llvm/trunk/unittests/IR/ValueMapTest.cpp

 struct LockMutex : ValueMapConfig<KeyT, MutexT> {
   struct ExtraData {
     MutexT *M;
     bool *CalledRAUW;
     bool *CalledDeleted;
   };
   static void onRAUW(const ExtraData &Data, KeyT Old, KeyT New) {
     *Data.CalledRAUW = true;
-    EXPECT_FALSE(Data.M->tryacquire()) << "Mutex should already be locked.";
+    EXPECT_FALSE(Data.M->try_lock()) << "Mutex should already be locked.";
   }
   static void onDelete(const ExtraData &Data, KeyT Old) {
     *Data.CalledDeleted = true;
-    EXPECT_FALSE(Data.M->tryacquire()) << "Mutex should already be locked.";
+    EXPECT_FALSE(Data.M->try_lock()) << "Mutex should already be locked.";
   }
   static MutexT *getMutex(const ExtraData &Data) { return Data.M; }
 };
 #if LLVM_ENABLE_THREADS
 TYPED_TEST(ValueMapTest, LocksMutex) {
-  sys::Mutex M(false);  // Not recursive.
+  std::mutex M;  // Not recursive.
   bool CalledRAUW = false, CalledDeleted = false;
-  typedef LockMutex<TypeParam*, sys::Mutex> ConfigType;
+  typedef LockMutex<TypeParam*, std::mutex> ConfigType;
   typename ConfigType::ExtraData Data = {&M, &CalledRAUW, &CalledDeleted};
   ValueMap<TypeParam*, int, ConfigType> VM(Data);
   VM[this->BitcastV.get()] = 7;
   this->BitcastV->replaceAllUsesWith(this->AddV.get());
   this->AddV.reset();
   EXPECT_TRUE(CalledRAUW);
   EXPECT_TRUE(CalledDeleted);
 }