Index: cfe/trunk/lib/CodeGen/CGCXXABI.h =================================================================== --- cfe/trunk/lib/CodeGen/CGCXXABI.h +++ cfe/trunk/lib/CodeGen/CGCXXABI.h @@ -484,30 +484,40 @@ /// Emit code to force the execution of a destructor during global /// teardown. The default implementation of this uses atexit. /// - /// \param dtor - a function taking a single pointer argument - /// \param addr - a pointer to pass to the destructor function. + /// \param Dtor - a function taking a single pointer argument + /// \param Addr - a pointer to pass to the destructor function. virtual void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, - llvm::Constant *dtor, llvm::Constant *addr); + llvm::Constant *Dtor, + llvm::Constant *Addr) = 0; /*************************** thread_local initialization ********************/ /// Emits ABI-required functions necessary to initialize thread_local /// variables in this translation unit. /// - /// \param Decls The thread_local declarations in this translation unit. - /// \param InitFunc If this translation unit contains any non-constant - /// initialization or non-trivial destruction for thread_local - /// variables, a function to perform the initialization. Otherwise, 0. + /// \param CXXThreadLocals - The thread_local declarations in this translation + /// unit. + /// \param CXXThreadLocalInits - If this translation unit contains any + /// non-constant initialization or non-trivial destruction for + /// thread_local variables, a list of functions to perform the + /// initialization. virtual void EmitThreadLocalInitFuncs( - ArrayRef > Decls, - llvm::Function *InitFunc); + CodeGenModule &CGM, + ArrayRef> + CXXThreadLocals, + ArrayRef CXXThreadLocalInits, + ArrayRef CXXThreadLocalInitVars) = 0; + + // Determine if references to thread_local global variables can be made + // directly or require access through a thread wrapper function. + virtual bool usesThreadWrapperFunction() const = 0; /// Emit a reference to a non-local thread_local variable (including /// triggering the initialization of all thread_local variables in its /// translation unit). virtual LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD, - QualType LValType); + QualType LValType) = 0; /// Emit a single constructor/destructor with the given type from a C++ /// constructor Decl. Index: cfe/trunk/lib/CodeGen/CGCXXABI.cpp =================================================================== --- cfe/trunk/lib/CodeGen/CGCXXABI.cpp +++ cfe/trunk/lib/CodeGen/CGCXXABI.cpp @@ -246,17 +246,6 @@ return llvm::ConstantInt::get(CGF.SizeTy, 0); } -void CGCXXABI::registerGlobalDtor(CodeGenFunction &CGF, - const VarDecl &D, - llvm::Constant *dtor, - llvm::Constant *addr) { - if (D.getTLSKind()) - CGM.ErrorUnsupported(&D, "non-trivial TLS destruction"); - - // The default behavior is to use atexit. - CGF.registerGlobalDtorWithAtExit(D, dtor, addr); -} - /// Returns the adjustment, in bytes, required for the given /// member-pointer operation. Returns null if no adjustment is /// required. @@ -310,18 +299,6 @@ return nullptr; } -void CGCXXABI::EmitThreadLocalInitFuncs( - ArrayRef > Decls, - llvm::Function *InitFunc) { -} - -LValue CGCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, - const VarDecl *VD, - QualType LValType) { - ErrorUnsupportedABI(CGF, "odr-use of thread_local global"); - return LValue(); -} - bool CGCXXABI::NeedsVTTParameter(GlobalDecl GD) { return false; } Index: cfe/trunk/lib/CodeGen/CGDeclCXX.cpp =================================================================== --- cfe/trunk/lib/CodeGen/CGDeclCXX.cpp +++ cfe/trunk/lib/CodeGen/CGDeclCXX.cpp @@ -155,17 +155,11 @@ EmitStoreOfScalar(RV.getScalarVal(), DeclPtr, false, Alignment, T); } -static llvm::Function * -CreateGlobalInitOrDestructFunction(CodeGenModule &CGM, - llvm::FunctionType *ty, - const Twine &name, - bool TLS = false); - /// Create a stub function, suitable for being passed to atexit, /// which passes the given address to the given destructor function. -static llvm::Constant *createAtExitStub(CodeGenModule &CGM, const VarDecl &VD, - llvm::Constant *dtor, - llvm::Constant *addr) { +llvm::Constant *CodeGenFunction::createAtExitStub(const VarDecl &VD, + llvm::Constant *dtor, + llvm::Constant *addr) { // Get the destructor function type, void(*)(void). llvm::FunctionType *ty = llvm::FunctionType::get(CGM.VoidTy, false); SmallString<256> FnName; @@ -173,8 +167,7 @@ llvm::raw_svector_ostream Out(FnName); CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(&VD, Out); } - llvm::Function *fn = - CreateGlobalInitOrDestructFunction(CGM, ty, FnName.str()); + llvm::Function *fn = CGM.CreateGlobalInitOrDestructFunction(ty, FnName.str()); CodeGenFunction CGF(CGM); @@ -198,7 +191,7 @@ llvm::Constant *dtor, llvm::Constant *addr) { // Create a function which calls the destructor. - llvm::Constant *dtorStub = createAtExitStub(CGM, VD, dtor, addr); + llvm::Constant *dtorStub = createAtExitStub(VD, dtor, addr); // extern "C" int atexit(void (*f)(void)); llvm::FunctionType *atexitTy = @@ -226,31 +219,29 @@ CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit); } -static llvm::Function * -CreateGlobalInitOrDestructFunction(CodeGenModule &CGM, - llvm::FunctionType *FTy, - const Twine &Name, bool TLS) { +llvm::Function * +CodeGenModule::CreateGlobalInitOrDestructFunction(llvm::FunctionType *FTy, + const Twine &Name, bool TLS) { llvm::Function *Fn = llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage, - Name, &CGM.getModule()); - if (!CGM.getLangOpts().AppleKext && !TLS) { + Name, &getModule()); + if (!getLangOpts().AppleKext && !TLS) { // Set the section if needed. - if (const char *Section = - CGM.getTarget().getStaticInitSectionSpecifier()) + if (const char *Section = getTarget().getStaticInitSectionSpecifier()) Fn->setSection(Section); } - Fn->setCallingConv(CGM.getRuntimeCC()); + Fn->setCallingConv(getRuntimeCC()); - if (!CGM.getLangOpts().Exceptions) + if (!getLangOpts().Exceptions) Fn->setDoesNotThrow(); - if (!CGM.getSanitizerBlacklist().isIn(*Fn)) { - if (CGM.getLangOpts().Sanitize.Address) + if (!getSanitizerBlacklist().isIn(*Fn)) { + if (getLangOpts().Sanitize.Address) Fn->addFnAttr(llvm::Attribute::SanitizeAddress); - if (CGM.getLangOpts().Sanitize.Thread) + if (getLangOpts().Sanitize.Thread) Fn->addFnAttr(llvm::Attribute::SanitizeThread); - if (CGM.getLangOpts().Sanitize.Memory) + if (getLangOpts().Sanitize.Memory) Fn->addFnAttr(llvm::Attribute::SanitizeMemory); } @@ -295,8 +286,7 @@ } // Create a variable initialization function. - llvm::Function *Fn = - CreateGlobalInitOrDestructFunction(*this, FTy, FnName.str()); + llvm::Function *Fn = CreateGlobalInitOrDestructFunction(FTy, FnName.str()); auto *ISA = D->getAttr(); CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr, @@ -312,6 +302,7 @@ // FIXME: We only need to register one __cxa_thread_atexit function for the // entire TU. CXXThreadLocalInits.push_back(Fn); + CXXThreadLocalInitVars.push_back(Addr); } else if (PerformInit && ISA) { EmitPointerToInitFunc(D, Addr, Fn, ISA); DelayedCXXInitPosition.erase(D); @@ -354,23 +345,11 @@ } void CodeGenModule::EmitCXXThreadLocalInitFunc() { - llvm::Function *InitFn = nullptr; - if (!CXXThreadLocalInits.empty()) { - // Generate a guarded initialization function. - llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false); - InitFn = CreateGlobalInitOrDestructFunction(*this, FTy, "__tls_init", - /*TLS*/ true); - llvm::GlobalVariable *Guard = new llvm::GlobalVariable( - getModule(), Int8Ty, false, llvm::GlobalVariable::InternalLinkage, - llvm::ConstantInt::get(Int8Ty, 0), "__tls_guard"); - Guard->setThreadLocal(true); - CodeGenFunction(*this) - .GenerateCXXGlobalInitFunc(InitFn, CXXThreadLocalInits, Guard); - } - - getCXXABI().EmitThreadLocalInitFuncs(CXXThreadLocals, InitFn); + getCXXABI().EmitThreadLocalInitFuncs( + *this, CXXThreadLocals, CXXThreadLocalInits, CXXThreadLocalInitVars); CXXThreadLocalInits.clear(); + CXXThreadLocalInitVars.clear(); CXXThreadLocals.clear(); } @@ -387,7 +366,7 @@ // Create our global initialization function. if (!PrioritizedCXXGlobalInits.empty()) { - SmallVector LocalCXXGlobalInits; + SmallVector LocalCXXGlobalInits; llvm::array_pod_sort(PrioritizedCXXGlobalInits.begin(), PrioritizedCXXGlobalInits.end()); // Iterate over "chunks" of ctors with same priority and emit each chunk @@ -406,10 +385,9 @@ std::string PrioritySuffix = llvm::utostr(Priority); // Priority is always <= 65535 (enforced by sema). PrioritySuffix = std::string(6-PrioritySuffix.size(), '0')+PrioritySuffix; - llvm::Function *Fn = - CreateGlobalInitOrDestructFunction(*this, FTy, - "_GLOBAL__I_" + PrioritySuffix); - + llvm::Function *Fn = CreateGlobalInitOrDestructFunction( + FTy, "_GLOBAL__I_" + PrioritySuffix); + for (; I < PrioE; ++I) LocalCXXGlobalInits.push_back(I->second); @@ -437,7 +415,7 @@ } llvm::Function *Fn = CreateGlobalInitOrDestructFunction( - *this, FTy, llvm::Twine("_GLOBAL__sub_I_", FileName)); + FTy, llvm::Twine("_GLOBAL__sub_I_", FileName)); CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXGlobalInits); AddGlobalCtor(Fn); @@ -453,8 +431,7 @@ llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false); // Create our global destructor function. - llvm::Function *Fn = - CreateGlobalInitOrDestructFunction(*this, FTy, "_GLOBAL__D_a"); + llvm::Function *Fn = CreateGlobalInitOrDestructFunction(FTy, "_GLOBAL__D_a"); CodeGenFunction(*this).GenerateCXXGlobalDtorsFunc(Fn, CXXGlobalDtors); AddGlobalDtor(Fn); @@ -488,7 +465,7 @@ void CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn, - ArrayRef Decls, + ArrayRef Decls, llvm::GlobalVariable *Guard) { { ArtificialLocation AL(*this, Builder); @@ -575,8 +552,8 @@ const CGFunctionInfo &FI = CGM.getTypes().arrangeFreeFunctionDeclaration( getContext().VoidTy, args, FunctionType::ExtInfo(), /*variadic=*/false); llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); - llvm::Function *fn = - CreateGlobalInitOrDestructFunction(CGM, FTy, "__cxx_global_array_dtor"); + llvm::Function *fn = + CGM.CreateGlobalInitOrDestructFunction(FTy, "__cxx_global_array_dtor"); StartFunction(VD, getContext().VoidTy, fn, FI, args); Index: cfe/trunk/lib/CodeGen/CGExpr.cpp =================================================================== --- cfe/trunk/lib/CodeGen/CGExpr.cpp +++ cfe/trunk/lib/CodeGen/CGExpr.cpp @@ -1822,7 +1822,8 @@ QualType T = E->getType(); // If it's thread_local, emit a call to its wrapper function instead. - if (VD->getTLSKind() == VarDecl::TLS_Dynamic) + if (VD->getTLSKind() == VarDecl::TLS_Dynamic && + CGF.CGM.getCXXABI().usesThreadWrapperFunction()) return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T); llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD); Index: cfe/trunk/lib/CodeGen/CodeGenFunction.h =================================================================== --- cfe/trunk/lib/CodeGen/CodeGenFunction.h +++ cfe/trunk/lib/CodeGen/CodeGenFunction.h @@ -2512,6 +2512,9 @@ void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::Constant *DeclPtr, bool PerformInit); + llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::Constant *Dtor, + llvm::Constant *Addr); + /// Call atexit() with a function that passes the given argument to /// the given function. void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::Constant *fn, @@ -2528,7 +2531,7 @@ /// GenerateCXXGlobalInitFunc - Generates code for initializing global /// variables. void GenerateCXXGlobalInitFunc(llvm::Function *Fn, - ArrayRef Decls, + ArrayRef CXXThreadLocals, llvm::GlobalVariable *Guard = nullptr); /// GenerateCXXGlobalDtorsFunc - Generates code for destroying global Index: cfe/trunk/lib/CodeGen/CodeGenModule.h =================================================================== --- cfe/trunk/lib/CodeGen/CodeGenModule.h +++ cfe/trunk/lib/CodeGen/CodeGenModule.h @@ -387,10 +387,11 @@ /// \brief thread_local variables with initializers that need to run /// before any thread_local variable in this TU is odr-used. - std::vector CXXThreadLocalInits; + std::vector CXXThreadLocalInits; + std::vector CXXThreadLocalInitVars; /// Global variables with initializers that need to run before main. - std::vector CXXGlobalInits; + std::vector CXXGlobalInits; /// When a C++ decl with an initializer is deferred, null is /// appended to CXXGlobalInits, and the index of that null is placed @@ -683,6 +684,10 @@ CreateOrReplaceCXXRuntimeVariable(StringRef Name, llvm::Type *Ty, llvm::GlobalValue::LinkageTypes Linkage); + llvm::Function *CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, + const Twine &name, + bool TLS = false); + /// Return the address space of the underlying global variable for D, as /// determined by its declaration. Normally this is the same as the address /// space of D's type, but in CUDA, address spaces are associated with Index: cfe/trunk/lib/CodeGen/ItaniumCXXABI.cpp =================================================================== --- cfe/trunk/lib/CodeGen/ItaniumCXXABI.cpp +++ cfe/trunk/lib/CodeGen/ItaniumCXXABI.cpp @@ -237,8 +237,13 @@ llvm::Function *getOrCreateThreadLocalWrapper(const VarDecl *VD, llvm::Value *Val); void EmitThreadLocalInitFuncs( - ArrayRef > Decls, - llvm::Function *InitFunc) override; + CodeGenModule &CGM, + ArrayRef> + CXXThreadLocals, + ArrayRef CXXThreadLocalInits, + ArrayRef CXXThreadLocalInitVars) override; + + bool usesThreadWrapperFunction() const override { return true; } LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD, QualType LValType) override; @@ -1897,11 +1902,28 @@ } void ItaniumCXXABI::EmitThreadLocalInitFuncs( - ArrayRef > Decls, - llvm::Function *InitFunc) { - for (unsigned I = 0, N = Decls.size(); I != N; ++I) { - const VarDecl *VD = Decls[I].first; - llvm::GlobalVariable *Var = Decls[I].second; + CodeGenModule &CGM, + ArrayRef> + CXXThreadLocals, ArrayRef CXXThreadLocalInits, + ArrayRef CXXThreadLocalInitVars) { + llvm::Function *InitFunc = nullptr; + if (!CXXThreadLocalInits.empty()) { + // Generate a guarded initialization function. + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); + InitFunc = CGM.CreateGlobalInitOrDestructFunction(FTy, "__tls_init", + /*TLS=*/true); + llvm::GlobalVariable *Guard = new llvm::GlobalVariable( + CGM.getModule(), CGM.Int8Ty, /*isConstant=*/false, + llvm::GlobalVariable::InternalLinkage, + llvm::ConstantInt::get(CGM.Int8Ty, 0), "__tls_guard"); + Guard->setThreadLocal(true); + CodeGenFunction(CGM) + .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits, Guard); + } + for (unsigned I = 0, N = CXXThreadLocals.size(); I != N; ++I) { + const VarDecl *VD = CXXThreadLocals[I].first; + llvm::GlobalVariable *Var = CXXThreadLocals[I].second; // Some targets require that all access to thread local variables go through // the thread wrapper. This means that we cannot attempt to create a thread Index: cfe/trunk/lib/CodeGen/MicrosoftCXXABI.cpp =================================================================== --- cfe/trunk/lib/CodeGen/MicrosoftCXXABI.cpp +++ cfe/trunk/lib/CodeGen/MicrosoftCXXABI.cpp @@ -255,9 +255,22 @@ llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, const ReturnAdjustment &RA) override; + void EmitThreadLocalInitFuncs( + CodeGenModule &CGM, + ArrayRef> + CXXThreadLocals, + ArrayRef CXXThreadLocalInits, + ArrayRef CXXThreadLocalInitVars) override; + + bool usesThreadWrapperFunction() const override { return false; } + LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD, + QualType LValType) override; + void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *DeclPtr, bool PerformInit) override; + void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, + llvm::Constant *Dtor, llvm::Constant *Addr) override; // ==== Notes on array cookies ========= // @@ -1711,6 +1724,94 @@ cookieSize.getQuantity()); } +static void emitGlobalDtorWithTLRegDtor(CodeGenFunction &CGF, const VarDecl &VD, + llvm::Constant *Dtor, + llvm::Constant *Addr) { + // Create a function which calls the destructor. + llvm::Constant *DtorStub = CGF.createAtExitStub(VD, Dtor, Addr); + + // extern "C" int __tlregdtor(void (*f)(void)); + llvm::FunctionType *TLRegDtorTy = llvm::FunctionType::get( + CGF.IntTy, DtorStub->getType(), /*IsVarArg=*/false); + + llvm::Constant *TLRegDtor = + CGF.CGM.CreateRuntimeFunction(TLRegDtorTy, "__tlregdtor"); + if (llvm::Function *TLRegDtorFn = dyn_cast(TLRegDtor)) + TLRegDtorFn->setDoesNotThrow(); + + CGF.EmitNounwindRuntimeCall(TLRegDtor, DtorStub); +} + +void MicrosoftCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, + llvm::Constant *Dtor, + llvm::Constant *Addr) { + if (D.getTLSKind()) + return emitGlobalDtorWithTLRegDtor(CGF, D, Dtor, Addr); + + // The default behavior is to use atexit. + CGF.registerGlobalDtorWithAtExit(D, Dtor, Addr); +} + +void MicrosoftCXXABI::EmitThreadLocalInitFuncs( + CodeGenModule &CGM, + ArrayRef> + CXXThreadLocals, + ArrayRef CXXThreadLocalInits, + ArrayRef CXXThreadLocalInitVars) { + // This will create a GV in the .CRT$XDU section. It will point to our + // initialization function. The CRT will call all of these function + // pointers at start-up time and, eventually, at thread-creation time. + auto AddToXDU = [&CGM](llvm::Function *InitFunc) { + llvm::GlobalVariable *InitFuncPtr = new llvm::GlobalVariable( + CGM.getModule(), InitFunc->getType(), /*IsConstant=*/true, + llvm::GlobalVariable::InternalLinkage, InitFunc, + Twine(InitFunc->getName(), "$initializer$")); + InitFuncPtr->setSection(".CRT$XDU"); + // This variable has discardable linkage, we have to add it to @llvm.used to + // ensure it won't get discarded. + CGM.addUsedGlobal(InitFuncPtr); + return InitFuncPtr; + }; + + std::vector NonComdatInits; + for (size_t I = 0, E = CXXThreadLocalInitVars.size(); I != E; ++I) { + llvm::GlobalVariable *GV = CXXThreadLocalInitVars[I]; + llvm::Function *F = CXXThreadLocalInits[I]; + + // If the GV is already in a comdat group, then we have to join it. + llvm::Comdat *C = GV->getComdat(); + + // LinkOnce and Weak linkage are lowered down to a single-member comdat + // group. + // Make an explicit group so we can join it. + if (!C && (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage())) { + C = CGM.getModule().getOrInsertComdat(GV->getName()); + GV->setComdat(C); + AddToXDU(F)->setComdat(C); + } else { + NonComdatInits.push_back(F); + } + } + + if (!NonComdatInits.empty()) { + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); + llvm::Function *InitFunc = + CGM.CreateGlobalInitOrDestructFunction(FTy, "__tls_init", + /*TLS=*/true); + CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(InitFunc, NonComdatInits); + + AddToXDU(InitFunc); + } +} + +LValue MicrosoftCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, + const VarDecl *VD, + QualType LValType) { + CGF.CGM.ErrorUnsupported(VD, "thread wrappers"); + return LValue(); +} + void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *GV, bool PerformInit) { Index: cfe/trunk/test/CodeGenCXX/ms-thread_local.cpp =================================================================== --- cfe/trunk/test/CodeGenCXX/ms-thread_local.cpp +++ cfe/trunk/test/CodeGenCXX/ms-thread_local.cpp @@ -0,0 +1,28 @@ +// RUN: %clang_cc1 %s -std=c++1y -triple=i686-pc-win32 -emit-llvm -o - | FileCheck %s + +struct A { + A(); + ~A(); +}; + +// CHECK-DAG: $"\01??$a@X@@3UA@@A" = comdat any +// CHECK-DAG: @"\01??$a@X@@3UA@@A" = linkonce_odr thread_local global %struct.A zeroinitializer, comdat $"\01??$a@X@@3UA@@A" +// CHECK-DAG: @"\01??__E?$a@X@@YAXXZ$initializer$" = internal constant void ()* @"\01??__E?$a@X@@YAXXZ", section ".CRT$XDU", comdat $"\01??$a@X@@3UA@@A" +template +thread_local A a = A(); + +// CHECK-DAG: @"\01?b@@3UA@@A" = thread_local global %struct.A zeroinitializer, align 1 +// CHECK-DAG: @"__tls_init$initializer$" = internal constant void ()* @__tls_init, section ".CRT$XDU" +thread_local A b; + +// CHECK-LABEL: define internal void @__tls_init() +// CHECK: call void @"\01??__Eb@@YAXXZ" + +thread_local A &c = b; +thread_local A &d = c; + +A f() { + (void)a; + (void)b; + return c; +}