Index: lib/CodeGen/CGCXXABI.h =================================================================== --- lib/CodeGen/CGCXXABI.h +++ lib/CodeGen/CGCXXABI.h @@ -539,8 +539,8 @@ /// - a static local variable /// - a static data member of a class template instantiation virtual void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, - llvm::GlobalVariable *DeclPtr, - bool PerformInit) = 0; + llvm::GlobalVariable *DeclPtr, bool PerformInit, + bool EmitInitOnly, bool EmitDtorOnly) = 0; /// Emit code to force the execution of a destructor during global /// teardown. The default implementation of this uses atexit. Index: lib/CodeGen/CGDeclCXX.cpp =================================================================== --- lib/CodeGen/CGDeclCXX.cpp +++ lib/CodeGen/CGDeclCXX.cpp @@ -62,7 +62,7 @@ /// Emit code to cause the destruction of the given variable with /// static storage duration. static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D, - ConstantAddress addr) { + ConstantAddress addr, bool EmitDtorOnly) { CodeGenModule &CGM = CGF.CGM; // FIXME: __attribute__((cleanup)) ? @@ -115,6 +115,13 @@ argument = llvm::Constant::getNullValue(CGF.Int8PtrTy); } + // Only emit the call if that was requested, otherwise register the destructor + // following the ABI rules. + if (EmitDtorOnly) { + CGF.Builder.CreateCall(function, argument); + return; + } + CGM.getCXXABI().registerGlobalDtor(CGF, D, function, argument); } @@ -142,7 +149,9 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::Constant *DeclPtr, - bool PerformInit) { + bool PerformInit, + bool EmitInitOnly, + bool EmitDtorOnly) { const Expr *Init = D.getInit(); QualType T = D.getType(); @@ -179,12 +188,17 @@ &D, DeclAddr, D.getAttr()->getLocation(), PerformInit, this); } - if (PerformInit) + if (PerformInit && !EmitDtorOnly) EmitDeclInit(*this, D, DeclAddr); + + // If all we need to emit is the initializer, we are done. + if (EmitInitOnly) + return; + if (CGM.isTypeConstant(D.getType(), true)) EmitDeclInvariant(*this, D, DeclPtr); else - EmitDeclDestroy(*this, D, DeclAddr); + EmitDeclDestroy(*this, D, DeclAddr, EmitDtorOnly); return; } @@ -250,7 +264,8 @@ void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr, - bool PerformInit) { + bool PerformInit, bool EmitInitOnly, + bool EmitDtorOnly) { // If we've been asked to forbid guard variables, emit an error now. // This diagnostic is hard-coded for Darwin's use case; we can find // better phrasing if someone else needs it. @@ -259,7 +274,8 @@ "this initialization requires a guard variable, which " "the kernel does not support"); - CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit); + CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit, EmitInitOnly, + EmitDtorOnly); } void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit, @@ -378,6 +394,11 @@ D->hasAttr())) return; + // If we are in OpenMP device mode we need to generate an entry point for each + // structor instead of the normal initialization. + if (OpenMPRuntime && OpenMPRuntime->emitDeviceCtorDtor(*D, Addr, PerformInit)) + return; + // Check if we've already initialized this decl. auto I = DelayedCXXInitPosition.find(D); if (I != DelayedCXXInitPosition.end() && I->second == ~0U) @@ -540,10 +561,9 @@ } /// Emit the code necessary to initialize the given global variable. -void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, - const VarDecl *D, - llvm::GlobalVariable *Addr, - bool PerformInit) { +void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc( + llvm::Function *Fn, const VarDecl *D, llvm::GlobalVariable *Addr, + bool PerformInit, bool EmitInitOnly, bool EmitDtorOnly) { // Check if we need to emit debug info for variable initializer. if (D->hasAttr()) DebugInfo = nullptr; // disable debug info indefinitely for this function @@ -559,11 +579,16 @@ // occurs for, e.g., instantiated static data members and // definitions explicitly marked weak. if (Addr->hasWeakLinkage() || Addr->hasLinkOnceLinkage()) { - EmitCXXGuardedInit(*D, Addr, PerformInit); + EmitCXXGuardedInit(*D, Addr, PerformInit, EmitInitOnly, EmitDtorOnly); } else { - EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit); + EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit, EmitInitOnly, EmitDtorOnly); } + // Register initializers and destructors for this variable. + if (CGM.getLangOpts().OpenMP) + CGM.getOpenMPRuntime().registerDeviceCtorDtorLaunching(*this, *D, Addr, + PerformInit); + FinishFunction(); } Index: lib/CodeGen/CGOpenMPRuntime.h =================================================================== --- lib/CodeGen/CGOpenMPRuntime.h +++ lib/CodeGen/CGOpenMPRuntime.h @@ -241,6 +241,18 @@ /// \brief Returns pointer to ident_t type. llvm::Type *getIdentTyPointerTy(); + /// Register target region related with the launching of Ctor/Dtors entry. + /// \param DeviceID The device ID of the target region in the system. + /// \param FileID The file ID of the target region in the system. + /// \param RegionName The name of the region. + /// \param Line Line where the declaration the target region refers to is + /// defined. + /// \param Fn The function that implements the target region. + /// \param IsDtor True if what being registered is a destructor. + virtual void registerCtorDtorEntry(unsigned DeviceID, unsigned FileID, + StringRef RegionName, unsigned Line, + llvm::Function *Fn, bool IsDtor); + /// \brief Gets thread id value for the current thread. /// llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc); @@ -250,6 +262,9 @@ // virtual StringRef getOutlinedHelperName() const { return ".omp_outlined."; } +public: + virtual StringRef RenameStandardFunction(StringRef name); + /// Emits \p Callee function call with arguments \p Args with location \p Loc. void emitCall(CodeGenFunction &CGF, llvm::Value *Callee, ArrayRef Args = llvm::None, @@ -370,18 +385,22 @@ class OffloadEntriesInfoManagerTy { CodeGenModule &CGM; - /// \brief Number of entries registered so far. - unsigned OffloadingEntriesNum; + /// \brief Number of ordered entries registered so far. + unsigned OffloadingOrderedEntriesNum = 0u; public: - /// Base class of the entries info. + /// \brief Base class of the entries info. class OffloadEntryInfo { public: - /// Kind of a given entry. Currently, only target regions are + /// \brief Kind of a given entry. Currently, only target regions are /// supported. enum OffloadingEntryInfoKinds : unsigned { // Entry is a target region. OFFLOAD_ENTRY_INFO_TARGET_REGION = 0, + // Entry is a device global variable. + OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR = 1, + // Entry is a device function. + OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION = 2, // Invalid entry info. OFFLOAD_ENTRY_INFO_INVALID = ~0u }; @@ -403,7 +422,7 @@ /// Flags associated with the device global. int32_t Flags; - /// Order this entry was emitted. + // \brief Order this entry was emitted. unsigned Order; OffloadingEntryInfoKinds Kind; @@ -411,10 +430,11 @@ /// \brief Return true if a there are no entries defined. bool empty() const; - /// \brief Return number of entries defined so far. - unsigned size() const { return OffloadingEntriesNum; } - OffloadEntriesInfoManagerTy(CodeGenModule &CGM) - : CGM(CGM), OffloadingEntriesNum(0) {} + /// \brief Return number of ordered entries defined so far. + unsigned getOrderedEntriesNum() const { + return OffloadingOrderedEntriesNum; + } + OffloadEntriesInfoManagerTy(CodeGenModule &CGM) : CGM(CGM) {} /// /// Target region entries related. @@ -471,6 +491,102 @@ void actOnTargetRegionEntriesInfo( const OffloadTargetRegionEntryInfoActTy &Action); + /// + /// Device global variable entries related. + /// + /// \brief Device global variable entries info. + class OffloadEntryInfoDeviceGlobalVar : public OffloadEntryInfo { + // \brief Address of the entity that has to be mapped for offloading. + llvm::Constant *Addr; + // \brief Type of the global variable. + QualType Ty; + // \brief Only generate metadata for this offload entry + bool OnlyMetadataFlag = false; + + public: + OffloadEntryInfoDeviceGlobalVar() + : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR, ~0u, + /*Flags=*/0u), + Addr(nullptr) {} + explicit OffloadEntryInfoDeviceGlobalVar(unsigned Order, + llvm::Constant *Addr, + QualType Ty, int32_t Flags) + : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR, Order, + Flags), + Addr(Addr), Ty(Ty) {} + + llvm::Constant *getAddress() const { return Addr; } + void setAddress(llvm::Constant *V) { + assert(!Addr && "Address as been set before!"); + Addr = V; + } + QualType getType() const { return Ty; } + void setType(QualType QTy) { Ty = QTy; } + bool getOnlyMetadataFlag() { return OnlyMetadataFlag; } + void setOnlyMetadataFlag(bool B) { OnlyMetadataFlag = B; } + static bool classof(const OffloadEntryInfo *Info) { + return Info->getKind() == OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR; + } + }; + /// \brief Initialize device global variable entry. + void initializeDeviceGlobalVarEntryInfo(StringRef MangledName, + unsigned Order); + /// \brief Register device global variable entry. + void registerDeviceGlobalVarEntryInfo(StringRef MangledName, + llvm::Constant *Addr, QualType Ty, + int32_t Flags, + bool isExternallyVisible); + /// \brief Return true if a device global variable entry with the provided + /// information exists. + bool hasDeviceGlobalVarEntryInfo(StringRef MangledName) const; + /// brief Applies action \a Action on all registered entries. + typedef llvm::function_ref + OffloadDeviceGlobalVarEntryInfoActTy; + void actOnDeviceGlobalVarEntriesInfo( + const OffloadDeviceGlobalVarEntryInfoActTy &Action); + + /// + /// Device function entries related. + /// + /// \brief Device global variable entries info. + class OffloadEntryInfoDeviceFunction : public OffloadEntryInfo { + // \brief Set to true if this entry was registered. + bool IsRegistered = false; + + public: + OffloadEntryInfoDeviceFunction() + : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION, ~0u, + /*Flags=*/0u) {} + explicit OffloadEntryInfoDeviceFunction(bool IsRegistered) + : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION, ~0u, + /*Flags=*/0u), + IsRegistered(IsRegistered) {} + + bool isRegistered() const { return IsRegistered; } + void setIsRegistered(bool Val) { + assert(!IsRegistered && "It was registered before!"); + IsRegistered = Val; + } + + static bool classof(const OffloadEntryInfo *Info) { + return Info->getKind() == OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION; + } + }; + /// \brief Initialize device function entry. + void initializeDeviceFunctionEntryInfo(StringRef MangledName); + /// \brief Register device function entry. + void registerDeviceFunctionEntryInfo(StringRef MangledName); + /// \brief Return true if a device function entry with the provided + /// information exists. + bool hasDeviceFunctionEntryInfo(StringRef MangledName) const; + /// brief Applies action \a Action on all registered entries. + typedef llvm::function_ref + OffloadDeviceFunctionEntryInfoActTy; + void actOnDeviceFunctionEntriesInfo( + const OffloadDeviceFunctionEntryInfoActTy &Action); + private: // Storage for target region entries kind. The storage is to be indexed by // file ID, device ID, parent function name and line number. @@ -483,6 +599,18 @@ typedef llvm::DenseMap OffloadEntriesTargetRegionPerDevice; typedef OffloadEntriesTargetRegionPerDevice OffloadEntriesTargetRegionTy; + + // Storage for device global variable entries kind. The storage is to be + // indexed by mangled name. + typedef llvm::StringMap + OffloadEntriesDeviceGlobalVarTy; + OffloadEntriesDeviceGlobalVarTy OffloadEntriesDeviceGlobalVar; + + // Storage for device function entries kind. The storage is to be indexed by + // mangled name. + typedef llvm::StringMap + OffloadEntriesDeviceFunctionTy; + OffloadEntriesDeviceFunctionTy OffloadEntriesDeviceFunction; OffloadEntriesTargetRegionTy OffloadEntriesTargetRegion; }; OffloadEntriesInfoManagerTy OffloadEntriesInfoManager; @@ -616,11 +744,43 @@ llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data); + /// This contains all the decls which were not specified under declare target + /// region, which are deferred for device code emission. + /// If a decl is used in target region implicitly without specifying under + /// declare target, deferred decl is emitted during Codegen::Release for + /// device codegen. + llvm::StringMap TrackedDecls; + + /// Struct that keeps information about the emitted definitions and + /// ctors/dtors so that it can be revisited when emitting declare target + /// entries. + struct DeclareTargetEntryInfo { + /// The declaration associated with this information. + const Decl *Variable; + /// Address of the variable or null if there is no variable. + llvm::Constant *VariableAddr = nullptr; + /// True if the associated variables requires Ctor or Dtor. + bool RequiresCtorDtor = false; + /// True if the variable associated with this information required + /// initialization. + bool PerformInitialization = false; + }; + + /// Map between a declaration name and its declare target information. + llvm::StringMap DeclareTargetEntryInfoMap; + public: explicit CGOpenMPRuntime(CodeGenModule &CGM); virtual ~CGOpenMPRuntime() {} virtual void clear(); + /// The function is added tracked functions list. + virtual void addTrackedFunction(StringRef MangledName, GlobalDecl GD); + + /// The function register all tracked functions if they have + /// OMPDeclareTargetDeclAttr + virtual void registerTrackedFunction(); + /// Emit code for the specified user defined reduction construct. virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D); @@ -1229,6 +1389,48 @@ /// \param GD Global to scan. virtual bool emitTargetGlobal(GlobalDecl GD); + /// \brief Emit the entry points (target regions) that implement the + /// initializers and destructors of the global \a D if that is meaningful for + /// the device. Returns true if the emission was successful. + /// \param D Global whose initializers destructors should be emitted. + /// \param Addr Address of the global being initialized/destroyed. + /// \param PerformInit True if the initializer should be emitted. + virtual bool emitDeviceCtorDtor(const VarDecl &D, llvm::GlobalVariable *Addr, + bool PerformInit); + + /// \brief Register that a variable requires a Ctor/Dtor. + /// \param D Global whose initializers destructors should be emitted. + /// \param Addr Address of the global being initialized/destroyed. + /// \param PerformInit True if the initializer should be emitted. + virtual void registerDeviceCtorDtorLaunching(CodeGenFunction &CGF, + const VarDecl &D, + llvm::GlobalVariable *Addr, + bool PerformInit); + + /// \brief Check whether the function definition in \a GD must be emitted for + /// the device or not. + /// \param GD Global declaration whose definition is being emitted. + virtual bool MustBeEmittedForDevice(GlobalDecl GD); + + /// \brief Register the function definition \a GD as meaningful for the + /// target. + /// \param GD Global declaration whose definition is being emitted. + virtual void registerTargetFunctionDefinition(GlobalDecl GD); + + /// \brief Register the global variable definition \a D as meaningful for the + /// target. + /// \param D Global declaration whose definition is being emitted. + /// \param Addr Address of the global. + virtual void registerTargetVariableDefinition(const VarDecl *D, + llvm::Constant *Addr); + + /// \brief Register a global that is replacing some other. If the global being + /// declare has a declare target attribute, the new one is registered as such. + /// \param MangledName Name of the global being replaced. + /// \param NewVal Global that is used to replace. + virtual void registerGlobalReplacement(StringRef MangledName, + llvm::GlobalValue *NewVal); + /// \brief Creates the offloading descriptor in the event any target region /// was emitted in the current module and return the function that registers /// it. Index: lib/CodeGen/CGOpenMPRuntime.cpp =================================================================== --- lib/CodeGen/CGOpenMPRuntime.cpp +++ lib/CodeGen/CGOpenMPRuntime.cpp @@ -3325,7 +3325,9 @@ bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const { // FIXME: Add other entries type when they become supported. - return OffloadEntriesTargetRegion.empty(); + return OffloadEntriesTargetRegion.empty() && + OffloadEntriesDeviceGlobalVar.empty() && + OffloadEntriesDeviceFunction.empty(); } /// \brief Initialize target region entry. @@ -3339,7 +3341,7 @@ OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr, /*Flags=*/0); - ++OffloadingEntriesNum; + ++OffloadingOrderedEntriesNum; } void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: @@ -3360,7 +3362,8 @@ Entry.setFlags(Flags); return; } else { - OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum++, Addr, ID, Flags); + OffloadEntryInfoTargetRegion Entry(OffloadingOrderedEntriesNum++, Addr, ID, + Flags); OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry; } } @@ -3396,16 +3399,114 @@ Action(D.first, F.first, P.first(), L.first, L.second); } +/// \brief Initialize device global variable entry. +void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: + initializeDeviceGlobalVarEntryInfo(StringRef MangledName, unsigned Order) { + assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is " + "only required for the device " + "code generation."); + OffloadEntriesDeviceGlobalVar[MangledName] = OffloadEntryInfoDeviceGlobalVar( + Order, /*Addr=*/nullptr, QualType(), /*Flags=*/0); + ++OffloadingOrderedEntriesNum; +} + +void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: + registerDeviceGlobalVarEntryInfo(StringRef MangledName, + llvm::Constant *Addr, QualType Ty, + int32_t Flags, bool isExternallyVisible) { + // If we are emitting code for a target, the entry is already initialized, + // only has to be registered. + if (CGM.getLangOpts().OpenMPIsDevice) { + assert(hasDeviceGlobalVarEntryInfo(MangledName) && "Entry must exist."); + auto &Entry = OffloadEntriesDeviceGlobalVar[MangledName]; + assert(Entry.isValid() && "Entry not initialized!"); + Entry.setAddress(Addr); + Entry.setType(Ty); + Entry.setFlags(Flags); + Entry.setOnlyMetadataFlag(!isExternallyVisible); + return; + } else { + OffloadEntryInfoDeviceGlobalVar Entry(OffloadingOrderedEntriesNum++, Addr, + Ty, Flags); + Entry.setOnlyMetadataFlag(!isExternallyVisible); + OffloadEntriesDeviceGlobalVar[MangledName] = Entry; + } +} + +bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasDeviceGlobalVarEntryInfo( + StringRef MangledName) const { + auto Entry = OffloadEntriesDeviceGlobalVar.find(MangledName); + if (Entry == OffloadEntriesDeviceGlobalVar.end()) + return false; + // Fail if this entry is already registered. + if (Entry->second.getAddress()) + return false; + return true; +} + +void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: + actOnDeviceGlobalVarEntriesInfo( + const OffloadDeviceGlobalVarEntryInfoActTy &Action) { + // Scan all target region entries and perform the provided action. + for (auto &E : OffloadEntriesDeviceGlobalVar) + Action(E.first(), E.second); +} + +/// \brief Initialize device function entry. +void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: + initializeDeviceFunctionEntryInfo(StringRef MangledName) { + assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is " + "only required for the device " + "code generation."); + OffloadEntriesDeviceFunction[MangledName] = OffloadEntryInfoDeviceFunction(); +} + +void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: + registerDeviceFunctionEntryInfo(StringRef MangledName) { + // If we are emitting code for a target, the entry is already initialized, + // only has to be registered. + if (CGM.getLangOpts().OpenMPIsDevice) { + assert(hasDeviceFunctionEntryInfo(MangledName) && "Entry must exist."); + auto &Entry = OffloadEntriesDeviceFunction[MangledName]; + Entry.setIsRegistered(/*Val=*/true); + return; + } else + OffloadEntriesDeviceFunction[MangledName] = + OffloadEntryInfoDeviceFunction(/*IsRegistred=*/true); +} + +bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasDeviceFunctionEntryInfo( + StringRef MangledName) const { + auto Entry = OffloadEntriesDeviceFunction.find(MangledName); + if (Entry == OffloadEntriesDeviceFunction.end()) + return false; + // Fail if this entry is already registered. + if (Entry->second.isRegistered()) + return false; + return true; +} + +void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: + actOnDeviceFunctionEntriesInfo( + const OffloadDeviceFunctionEntryInfoActTy &Action) { + // Scan all target region entries and perform the provided action. + for (auto &E : OffloadEntriesDeviceFunction) + Action(E.first(), E.second); +} + /// \brief Create a Ctor/Dtor-like function whose body is emitted through -/// \a Codegen. This is used to emit the two functions that register and -/// unregister the descriptor of the current compilation unit. +/// \a Codegen. This is used to emit functions that register and +/// unregister descriptors, initializers or destructors in the current +/// compilation unit. static llvm::Function * -createOffloadingBinaryDescriptorFunction(CodeGenModule &CGM, StringRef Name, - const RegionCodeGenTy &Codegen) { +createOffloadingHelperFunction(CodeGenModule &CGM, StringRef Name, + bool RequiresArgument, + const RegionCodeGenTy &Codegen) { auto &C = CGM.getContext(); FunctionArgList Args; ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other); - Args.push_back(&DummyPtr); + if (RequiresArgument) + Args.push_back(&DummyPtr); CodeGenFunction CGF(CGM); // Disable debug info for global (de-)initializer because they are not part of @@ -3510,14 +3611,14 @@ ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(), SourceLocation(), IdentInfo, C.CharTy, ImplicitParamDecl::Other); - auto *UnRegFn = createOffloadingBinaryDescriptorFunction( - CGM, ".omp_offloading.descriptor_unreg", + auto *UnRegFn = createOffloadingHelperFunction( + CGM, ".omp_offloading.descriptor_unreg", /*RequiresArgument=*/true, [&](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib), Desc); }); - auto *RegFn = createOffloadingBinaryDescriptorFunction( - CGM, ".omp_offloading.descriptor_reg", + auto *RegFn = createOffloadingHelperFunction( + CGM, ".omp_offloading.descriptor_reg", /*RequiresArgument=*/false, [&](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc); @@ -3561,6 +3662,10 @@ Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); llvm::Constant *StrPtr = llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy); + // Decide linkage type of the entry struct by looking at the linkage type of + // the variable. By default the linkage is Link-Once. + auto EntryLinkage = llvm::GlobalValue::WeakAnyLinkage; + // We can't have any padding between symbols, so we need to have 1-byte // alignment. auto Align = CharUnits::fromQuantity(1); @@ -3573,9 +3678,10 @@ EntryInit.addInt(CGM.SizeTy, Size); EntryInit.addInt(CGM.Int32Ty, Flags); EntryInit.addInt(CGM.Int32Ty, 0); - llvm::GlobalVariable *Entry = EntryInit.finishAndCreateGlobal( - Twine(".omp_offloading.entry.") + Name, Align, - /*constant*/ true, llvm::GlobalValue::ExternalLinkage); + SmallString<128> EntryGblName(".omp_offloading.entry."); + EntryGblName += Name; + llvm::GlobalVariable *Entry = EntryInit.finishAndCreateGlobal(EntryGblName, + Align, /*constant*/ true, EntryLinkage); // The entry has to be created in the section the linker expects it to be. Entry->setSection(".omp_offloading.entries"); @@ -3598,7 +3704,7 @@ llvm::Module &M = CGM.getModule(); llvm::LLVMContext &C = M.getContext(); SmallVector - OrderedEntries(OffloadEntriesInfoManager.size()); + OrderedEntries(OffloadEntriesInfoManager.getOrderedEntriesNum()); // Create the offloading info metadata node. llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info"); @@ -3615,7 +3721,7 @@ auto &&TargetRegionMetadataEmitter = [&]( unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned Line, OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) { - llvm::SmallVector Ops; + llvm::SmallVector Ops; // Generate metadata for target regions. Each entry of this metadata // contains: // - Entry 0 -> Kind of this type of metadata (0). @@ -3642,6 +3748,51 @@ OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo( TargetRegionMetadataEmitter); + // Create function that emits metadata for each device global variable entry; + auto &&DeviceGlobalVarMetadataEmitter = + [&](StringRef MangledName, + OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar &E) { + llvm::SmallVector Ops; + // Generate metadata for global variables. Each entry of this metadata + // contains: + // - Entry 0 -> Kind of this type of metadata (1). + // - Entry 1 -> Mangled name of the variable. + // - Entry 2 -> Order the entry was created. + // The first element of the metadata node is the kind. + Ops.push_back(getMDInt(E.getKind())); + Ops.push_back(getMDString(MangledName)); + Ops.push_back(getMDInt(E.getOrder())); + + // Save this entry in the right position of the ordered entries array. + OrderedEntries[E.getOrder()] = &E; + + // Add metadata to the named metadata node. + MD->addOperand(llvm::MDNode::get(C, Ops)); + }; + + OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo( + DeviceGlobalVarMetadataEmitter); + + // Create function that emits metadata for each device function entry; + auto &&DeviceFunctionMetadataEmitter = + [&](StringRef MangledName, + OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceFunction &E) { + llvm::SmallVector Ops; + // Generate metadata for global variables. Each entry of this metadata + // contains: + // - Entry 0 -> Kind of this type of metadata (2). + // - Entry 1 -> Mangled name of the variable. + // The first element of the metadata node is the kind. + Ops.push_back(getMDInt(E.getKind())); + Ops.push_back(getMDString(MangledName)); + + // Add metadata to the named metadata node. + MD->addOperand(llvm::MDNode::get(C, Ops)); + }; + + OffloadEntriesInfoManager.actOnDeviceFunctionEntriesInfo( + DeviceFunctionMetadataEmitter); + for (auto *E : OrderedEntries) { assert(E && "All ordered entries must exist!"); if (auto *CE = @@ -3649,9 +3800,20 @@ E)) { assert(CE->getID() && CE->getAddress() && "Entry ID and Addr are invalid!"); - createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0); + createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0, + CE->getFlags()); + } else if (auto *CE = dyn_cast(E)) { + assert(CE->getAddress() && "Entry Addr is invalid!"); + // The global address can be used as ID. + if (!CE->getOnlyMetadataFlag()) { + createOffloadEntry( + CE->getAddress(), CE->getAddress(), + CGM.getContext().getTypeSizeInChars(CE->getType()).getQuantity(), + CE->getFlags()); + } } else - llvm_unreachable("Unsupported entry kind."); + llvm_unreachable("Unsupported ordered entry kind."); } } @@ -3707,6 +3869,17 @@ /*ParentName=*/getMDString(3), /*Line=*/getMDInt(4), /*Order=*/getMDInt(5)); break; + case OffloadEntriesInfoManagerTy::OffloadEntryInfo:: + OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR: + OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo( + /*MangledName=*/getMDString(1), + /*Order=*/getMDInt(2)); + break; + case OffloadEntriesInfoManagerTy::OffloadEntryInfo:: + OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION: + OffloadEntriesInfoManager.initializeDeviceFunctionEntryInfo( + /*MangledName=*/getMDString(1)); + break; } } } @@ -7390,6 +7563,11 @@ if (!CGM.getLangOpts().OpenMPIsDevice) return false; + // Emit this function normally if it is a device function. + if (OffloadEntriesInfoManager.hasDeviceFunctionEntryInfo( + CGM.getMangledName(GD))) + return false; + // Try to detect target regions in the function. scanForTargetRegionsFunctions(FD.getBody(), CGM.getMangledName(GD)); @@ -7421,9 +7599,10 @@ } } - // If we are in target mode, we do not emit any global (declare target is not - // implemented yet). Therefore we signal that GD was processed in this case. - return true; + // If we are in target mode we only emit global variables that we could find + // in the host metadata. + return !OffloadEntriesInfoManager.hasDeviceGlobalVarEntryInfo( + CGM.getMangledName(GD)); } bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) { @@ -7434,7 +7613,323 @@ return emitTargetGlobalVariable(GD); } +bool CGOpenMPRuntime::MustBeEmittedForDevice(GlobalDecl GD) { + if (!CGM.getLangOpts().OpenMPIsDevice && + CGM.getLangOpts().OMPTargetTriples.empty()) + return false; + + // Should be in device if it has metadata + return OffloadEntriesInfoManager.hasDeviceFunctionEntryInfo( + CGM.getMangledName(GD)); +} + +/// \brief Return the declare target attribute if the declaration is marked as +// 'declare target', i.e. the declaration itself, its template declaration, or +/// any of its redeclarations have the 'declare target' attribute. +static OMPDeclareTargetDeclAttr * +IsDeclareTargetDeclaration(const ValueDecl *VD) { + const Decl *RelevantDecl = VD; + + // Try to get the original template if any. + if (auto *FD = dyn_cast(VD)) { + if (auto *Tmpl = FD->getPrimaryTemplate()) + RelevantDecl = Tmpl; + } + + // Check if the declaration or any of its redeclarations have a declare target + // attribute. + if (auto *Attr = RelevantDecl->getAttr()) + return Attr; + + if (auto *Attr = VD->getAttr()) + return Attr; + + for (const Decl *RD : RelevantDecl->redecls()) + if (auto *Attr = RD->getAttr()) + return Attr; + + return nullptr; +} + +namespace { +enum OpenMPOffloadingDeclareTargetFlags { + /// \brief Mark the entry has having a 'link' attribute. + OMP_DECLARE_TARGET_LINK = 0x01, + /// \brief Mark the entry has being a global constructor. + OMP_DECLARE_TARGET_CTOR = 0x02, + /// \brief Mark the entry has being a global destructor. + OMP_DECLARE_TARGET_DTOR = 0x04, +}; +} + +void CGOpenMPRuntime::registerCtorDtorEntry(unsigned DeviceID, unsigned FileID, + StringRef RegionName, unsigned Line, + llvm::Function *Fn, bool IsDtor) { + OffloadEntriesInfoManager.registerTargetRegionEntryInfo( + DeviceID, FileID, RegionName, Line, Fn, Fn, + IsDtor ? OMP_DECLARE_TARGET_DTOR : OMP_DECLARE_TARGET_CTOR); +} + +bool CGOpenMPRuntime::emitDeviceCtorDtor(const VarDecl &D, + llvm::GlobalVariable *Addr, + bool PerformInit) { + // If this is not an OpenMP device, don't have to generate anything. + if (!CGM.getLangOpts().OpenMPIsDevice) + return false; + + // Produce the unique prefix to identify the new target regions. We use the + // source location of the variable declaration which we know to not conflict + // with any target region. + unsigned DeviceID; + unsigned FileID; + unsigned Line; + getTargetEntryUniqueInfo(CGM.getContext(), D.getLocStart(), DeviceID, FileID, + Line); + + // Check if we have a dtor target region specified by the host. The parent + // name is the name of the declaration with suffix _dtor and/or _init if that + // is a destructor or initializer target region. + SmallString<128> ParentNameDtor; + ParentNameDtor += "__omp_offloading_dtor_"; + ParentNameDtor += D.getName(); + + // If we don't have a Dtor specified by the host, we don't have anything to + // do, but we return true to prevent the default initializer codegen. + if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID, + ParentNameDtor, Line)) + return true; + + auto &Ctx = CGM.getContext(); + SmallString<64> PrefixName; + { + llvm::raw_svector_ostream OS(PrefixName); + OS << "__omp_offloading" << llvm::format("_%x", DeviceID) + << llvm::format("_%x_", FileID) << D.getName() << "_l" << Line; + } + + // If we need to perform an initialization, we need to create a function that + // calls the initializer as entry point. + if (PerformInit) { + SmallString<128> ParentNameInit; + ParentNameInit += "__omp_offloading_init_"; + ParentNameInit += D.getName(); + + assert(OffloadEntriesInfoManager.hasTargetRegionEntryInfo( + DeviceID, FileID, ParentNameInit, Line) && + "Expecting initializer to be defined by the OpenMP host!"); + + auto &FnInfo = + CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, None); + auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo); + auto *Fn = + llvm::Function::Create(FnTy, llvm::GlobalValue::ExternalLinkage, + Twine(PrefixName) + "_init", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, FnInfo, Fn); + CodeGenFunction(CGM).GenerateCXXGlobalVarDeclInitFunc( + Fn, &D, Addr, /*PerformInit=*/true, /*emitInitOnly=*/true); + + // Register the information for the entry associated with this target + // region. + registerCtorDtorEntry(DeviceID, FileID, ParentNameInit, Line, Fn, + /*IsDtor=*/false); + } + + // Create the target region for the destructor. + auto &FnInfo = + CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, None); + auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo); + auto *Fn = + llvm::Function::Create(FnTy, llvm::GlobalValue::ExternalLinkage, + Twine(PrefixName) + "_dtor", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, FnInfo, Fn); + CodeGenFunction(CGM).GenerateCXXGlobalVarDeclInitFunc( + Fn, &D, Addr, PerformInit, /*emitInitOnly=*/false, /*emitDtorOnly=*/true); + + // Register the information for the entry associated with this target region. + registerCtorDtorEntry(DeviceID, FileID, ParentNameDtor, Line, Fn, + /*IsDtor=*/true); + + // We successfully generated the target regions to initialize and destroy the + // global. + return true; +} + +void CGOpenMPRuntime::registerDeviceCtorDtorLaunching( + CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *Addr, + bool PerformInit) { + // If we are not producing code for the host or we don't have any devices + // specified, we don't have to launch anything. + if (CGM.getLangOpts().OpenMPIsDevice || + CGM.getLangOpts().OMPTargetTriples.empty()) + return; + + // Save the information in the declare target info map. + auto &Info = DeclareTargetEntryInfoMap[CGM.getMangledName(GlobalDecl(&D))]; + Info.RequiresCtorDtor = true; + Info.VariableAddr = Addr; + Info.Variable = &D; + Info.PerformInitialization = PerformInit; + + return; +} + +void CGOpenMPRuntime::registerTargetFunctionDefinition(GlobalDecl GD) { + // We don't have to register anything if compiling for the host with no target + // devices specified. + if (!CGM.getLangOpts().OpenMPIsDevice && + CGM.getLangOpts().OMPTargetTriples.empty()) + return; + + if (auto *FD = dyn_cast(GD.getDecl())) { + auto &Info = DeclareTargetEntryInfoMap[CGM.getMangledName(GD)]; + Info.Variable = FD; + } +} + +void CGOpenMPRuntime::registerTargetVariableDefinition(const VarDecl *D, + llvm::Constant *Addr) { + // We don't have to register anything if compiling for the host with no target + // devices specified. + if (!CGM.getLangOpts().OpenMPIsDevice && + CGM.getLangOpts().OMPTargetTriples.empty()) + return; + + assert(D && Addr && "Invalid variable information!"); + + auto &Info = DeclareTargetEntryInfoMap[CGM.getMangledName(GlobalDecl(D))]; + Info.Variable = D; + Info.VariableAddr = Addr; +} + +void CGOpenMPRuntime::registerGlobalReplacement(StringRef MangledName, + llvm::GlobalValue *NewVal) { + // We don't have to register anything if compiling for the host with no target + // devices specified. + if (!CGM.getLangOpts().OpenMPIsDevice && + CGM.getLangOpts().OMPTargetTriples.empty()) + return; + + assert(MangledName != NewVal->getName() && + "Replacing global with the same name??"); + + // If the existing global is registered, make sure the new global also is. + auto It = DeclareTargetEntryInfoMap.find(MangledName); + if (It == DeclareTargetEntryInfoMap.end()) + return; + + // Register the global. Copy the existing declare target record, but use the + // new address. + auto &Info = DeclareTargetEntryInfoMap[NewVal->getName()]; + Info = It->second; + Info.VariableAddr = NewVal; +} + llvm::Function *CGOpenMPRuntime::emitRegistrationFunction() { + auto &Ctx = CGM.getContext(); + + // Figure out all the declare target data that should be registered as such. + for (auto II = DeclareTargetEntryInfoMap.begin(), + IE = DeclareTargetEntryInfoMap.end(); + II != IE; ++II) { + DeclareTargetEntryInfo &Info = II->second; + + // Utility function to register the Ctors/Dtors. + auto &&RegisterCtorDtor = [&Info, &Ctx, this]() { + const VarDecl &D = *cast(Info.Variable); + llvm::Constant *Addr = Info.VariableAddr; + assert(Addr && "declaration address is not defined??"); + + // Produce the unique prefix to identify the new target regions. We use + // the source location of the variable declaration which we know to not + // conflict with any target region. + unsigned DeviceID; + unsigned FileID; + unsigned Line; + getTargetEntryUniqueInfo(CGM.getContext(), D.getLocStart(), DeviceID, + FileID, Line); + SmallString<64> PrefixName; + { + llvm::raw_svector_ostream OS(PrefixName); + OS << "__omp_offloading" << llvm::format("_%x", DeviceID) + << llvm::format("_%x_", FileID) << D.getName() << "_l" << Line; + } + + // If we have to perform an initialization, create a target region to + // launch that on the device. + if (Info.PerformInitialization) { + // Produce an ID whose address uniquely identifies the target region. + auto ID = new llvm::GlobalVariable( + CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, + llvm::Constant::getNullValue(CGM.Int8Ty), + Twine(PrefixName) + "_init"); + + // We define the parent name of this target region as the name of the + // global + // followed by the suffix _init. This suffix is what distinguishes a + // initializer from the destructor. + SmallString<128> ParentName; + ParentName += "__omp_offloading_init_"; + ParentName += D.getName(); + + // Register the information for the entry associated with this target + // region. + OffloadEntriesInfoManager.registerTargetRegionEntryInfo( + DeviceID, FileID, ParentName, Line, ID, ID, + OMP_DECLARE_TARGET_CTOR); + } + + // Produce an ID whose address uniquely identifies the target dtor. + auto ID = new llvm::GlobalVariable( + CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, + llvm::Constant::getNullValue(CGM.Int8Ty), + Twine(PrefixName) + "_dtor"); + + // We define the parent name of this target region as the name of + // the global followed by the suffix _dtor. This suffix is what + // distinguishes a initializer from the destructor. + SmallString<128> ParentName; + ParentName += "__omp_offloading_dtor_"; + ParentName += D.getName(); + + // Register the information for the entry associated with this + // target + // region. + OffloadEntriesInfoManager.registerTargetRegionEntryInfo( + DeviceID, FileID, ParentName, Line, ID, ID, OMP_DECLARE_TARGET_DTOR); + }; + + // If we have a variable, register it and register any ctors/dtors entries. + if (auto *D = dyn_cast(Info.Variable)) { + assert(Info.VariableAddr && "No variable address defined??"); + + if (auto *Attr = IsDeclareTargetDeclaration(D)) { + // If we have a link attribute we need to set the link flag. + int64_t Flags = 0; + if (Attr->getMapType() == OMPDeclareTargetDeclAttr::MT_Link) + Flags |= OMP_DECLARE_TARGET_LINK; + + // Register the variable as declare target. + OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo( + II->first(), Info.VariableAddr, D->getType(), Flags, + D->isExternallyVisible()); + + // Emit the ctor/dtor launching if required. + if (Info.RequiresCtorDtor) + RegisterCtorDtor(); + } + continue; + } + + const FunctionDecl *FD = cast(Info.Variable); + // Only declare target functions are registered. + if (!IsDeclareTargetDeclaration(FD)) + continue; + + OffloadEntriesInfoManager.registerDeviceFunctionEntryInfo(II->first()); + } + // If we have offloading in the current module, we need to emit the entries // now and register the offloading descriptor. createOffloadEntriesAndInfoMetadata(); @@ -7950,6 +8445,11 @@ } } + +StringRef CGOpenMPRuntime::RenameStandardFunction(StringRef name) { + return name; +} + namespace { /// Cleanup action for doacross support. class DoacrossCleanupTy final : public EHScopeStack::Cleanup { @@ -8078,6 +8578,16 @@ emitCall(CGF, OutlinedFn, Args, Loc); } + +void CGOpenMPRuntime::addTrackedFunction(StringRef MangledName, GlobalDecl GD) { + TrackedDecls[MangledName] = GD; +} + +void CGOpenMPRuntime::registerTrackedFunction() { + for (auto &GD : TrackedDecls) + registerTargetFunctionDefinition(GD.second); +} + Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const { @@ -8377,4 +8887,3 @@ const VarDecl *TargetParam) const { llvm_unreachable("Not supported in SIMD-only mode"); } - Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -3633,9 +3633,12 @@ /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++ - /// variable with global storage. + /// variable with global storage. If \a EmitInitOnly or \a EmitDtorOnly are + /// set to true, only the call to the initializer or destructor is emitted, + /// respectively. void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::Constant *DeclPtr, - bool PerformInit); + bool PerformInit, bool EmitInitOnly, + bool EmitDtorOnly); llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::Constant *Dtor, llvm::Constant *Addr); @@ -3651,7 +3654,8 @@ /// once, e.g. with a static local variable or a static data member /// of a class template. void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr, - bool PerformInit); + bool PerformInit, bool EmitInitOnly = false, + bool EmitDtorOnly = false); enum class GuardKind { VariableGuard, TlsGuard }; @@ -3674,10 +3678,12 @@ const std::vector> &DtorsAndObjects); - void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, - const VarDecl *D, + + void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D, llvm::GlobalVariable *Addr, - bool PerformInit); + bool PerformInit, + bool EmitInitOnly = false, + bool EmitDtorOnly = false); void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest); Index: lib/CodeGen/CodeGenModule.cpp =================================================================== --- lib/CodeGen/CodeGenModule.cpp +++ lib/CodeGen/CodeGenModule.cpp @@ -244,6 +244,12 @@ } } + // Tell the OpenMP runtime we are replacing a global that can potentially be + // marked "declare target". + if (OpenMPRuntime) + if (auto *NewGV = dyn_cast(Replacement)) + OpenMPRuntime->registerGlobalReplacement(MangledName, NewGV); + // Replace old with new, but keep the old order. OldF->replaceAllUsesWith(Replacement); if (NewF) { @@ -266,6 +272,12 @@ GV->replaceAllUsesWith(C); GV->eraseFromParent(); + + // Tell the OpenMP runtime we are replacing a global that can potentially be + // marked "declare target". + if (OpenMPRuntime) + if (auto *NewGV = dyn_cast(C)) + OpenMPRuntime->registerGlobalReplacement(GV->getName(), NewGV); } } @@ -1537,6 +1549,10 @@ assert(DeferredVTables.empty()); } + if (LangOpts.OpenMP && !LangOpts.OpenMPIsDevice) { + OpenMPRuntime->registerTrackedFunction(); + } + // Stop if we're out of both deferred vtables and deferred declarations. if (DeferredDeclsToEmit.empty()) return; @@ -1756,6 +1772,11 @@ if (LangOpts.EmitAllDecls) return true; + // In OpenMP device mode all declarations that were not filtered should be + // emitted. + if (LangOpts.OpenMPIsDevice) + return true; + return getContext().DeclMustBeEmitted(Global); } @@ -1879,6 +1900,21 @@ } if (LangOpts.OpenMP) { + if (LangOpts.OpenMPIsDevice) { + if (const auto *FD = dyn_cast(Global)) + if (FD->isThisDeclarationADefinition()) { + if (OpenMPRuntime->MustBeEmittedForDevice(GD)) { + EmitGlobalDefinition(GD); + return; + } + } + } + + if (!LangOpts.OpenMPIsDevice) { + StringRef MangledName = getMangledName(GD); + OpenMPRuntime->addTrackedFunction(MangledName, GD); + } + // If this is OpenMP device, check if it is legal to emit this global // normally. if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD)) @@ -2136,6 +2172,11 @@ Context.getSourceManager(), "Generating code for declaration"); + // If this is OpenMP device, check if it is legal to emit this global + // normally. + if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD)) + return; + if (isa(D)) { // At -O0, don't generate IR for functions with available_externally // linkage. @@ -2145,11 +2186,15 @@ if (const auto *Method = dyn_cast(D)) { // Make sure to emit the definition(s) before we emit the thunks. // This is necessary for the generation of certain thunks. - if (const auto *CD = dyn_cast(Method)) + if (const auto *CD = dyn_cast(Method)) { + if (OpenMPRuntime) + OpenMPRuntime->registerTargetFunctionDefinition(GD); ABI->emitCXXStructor(CD, getFromCtorType(GD.getCtorType())); - else if (const auto *DD = dyn_cast(Method)) + } else if (const auto *DD = dyn_cast(Method)) { + if (OpenMPRuntime) + OpenMPRuntime->registerTargetFunctionDefinition(GD); ABI->emitCXXStructor(DD, getFromDtorType(GD.getDtorType())); - else + } else EmitGlobalFunctionDefinition(GD, GV); if (Method->isVirtual()) @@ -2264,6 +2309,11 @@ } } + // Process function name as required by the OpenMP runtime + if (OpenMPRuntime) { + MangledName = OpenMPRuntime->RenameStandardFunction(MangledName); + } + // Lookup the entry, lazily creating it if necessary. llvm::GlobalValue *Entry = GetGlobalValue(MangledName); if (Entry) { @@ -2824,6 +2874,11 @@ void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) { assert(!D->getInit() && "Cannot emit definite definitions here!"); + // If this is OpenMP device, check if it is legal to emit this global + // normally. + if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(D)) + return; + StringRef MangledName = getMangledName(D); llvm::GlobalValue *GV = GetGlobalValue(MangledName); @@ -3091,6 +3146,9 @@ } } + if (OpenMPRuntime) + OpenMPRuntime->registerTargetVariableDefinition(D, GV); + GV->setInitializer(Init); if (emitter) emitter->finalize(GV); @@ -3448,6 +3506,9 @@ if (!GV->isDeclaration()) return; + if (OpenMPRuntime) + OpenMPRuntime->registerTargetFunctionDefinition(GD); + // We need to set linkage and visibility on the function before // generating code for it because various parts of IR generation // want to propagate this information down (e.g. to local static Index: lib/CodeGen/ItaniumCXXABI.cpp =================================================================== --- lib/CodeGen/ItaniumCXXABI.cpp +++ lib/CodeGen/ItaniumCXXABI.cpp @@ -338,7 +338,8 @@ void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *DeclPtr, - bool PerformInit) override; + bool PerformInit, + bool EmitInitOnly, bool EmitDtorOnly) override; void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, llvm::Constant *dtor, llvm::Constant *addr) override; @@ -1996,7 +1997,8 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *var, - bool shouldPerformInit) { + bool shouldPerformInit, bool EmitInitOnly, + bool EmitDtorOnly) { CGBuilderTy &Builder = CGF.Builder; // Inline variables that weren't instantiated from variable templates have @@ -2162,7 +2164,8 @@ } // Emit the initializer and add a global destructor if appropriate. - CGF.EmitCXXGlobalVarDeclInit(D, var, shouldPerformInit); + CGF.EmitCXXGlobalVarDeclInit(D, var, shouldPerformInit, EmitInitOnly, + EmitDtorOnly); if (threadsafe) { // Pop the guard-abort cleanup if we pushed one. Index: lib/CodeGen/MicrosoftCXXABI.cpp =================================================================== --- lib/CodeGen/MicrosoftCXXABI.cpp +++ lib/CodeGen/MicrosoftCXXABI.cpp @@ -387,8 +387,8 @@ QualType LValType) override; void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, - llvm::GlobalVariable *DeclPtr, - bool PerformInit) override; + llvm::GlobalVariable *DeclPtr, bool PerformInit, + bool EmitInitOnly, bool EmitDtorOnly) override; void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, llvm::Constant *Dtor, llvm::Constant *Addr) override; @@ -2387,7 +2387,8 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *GV, - bool PerformInit) { + bool PerformInit, bool EmitInitOnly, + bool EmitDtorOnly) { // MSVC only uses guards for static locals. if (!D.isStaticLocal()) { assert(GV->hasWeakLinkage() || GV->hasLinkOnceLinkage()); @@ -2395,7 +2396,8 @@ llvm::Function *F = CGF.CurFn; F->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage); F->setComdat(CGM.getModule().getOrInsertComdat(F->getName())); - CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); + CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit, EmitInitOnly, + EmitDtorOnly); return; } @@ -2496,7 +2498,8 @@ CGF.EmitBlock(InitBlock); Builder.CreateStore(Builder.CreateOr(LI, Bit), GuardAddr); CGF.EHStack.pushCleanup(EHCleanup, GuardAddr, GuardNum); - CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); + CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit, EmitInitOnly, + EmitDtorOnly); CGF.PopCleanupBlock(); Builder.CreateBr(EndBlock); @@ -2542,7 +2545,8 @@ // Ok, we ended up getting selected as the initializing thread. CGF.EmitBlock(InitBlock); CGF.EHStack.pushCleanup(EHCleanup, GuardAddr); - CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); + CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit, EmitInitOnly, + EmitDtorOnly); CGF.PopCleanupBlock(); CGF.EmitNounwindRuntimeCall(getInitThreadFooterFn(CGM), GuardAddr.getPointer()); Index: lib/Parse/ParseOpenMP.cpp =================================================================== --- lib/Parse/ParseOpenMP.cpp +++ lib/Parse/ParseOpenMP.cpp @@ -758,6 +758,7 @@ if (!Actions.ActOnStartOpenMPDeclareTargetDirective(DTLoc)) return DeclGroupPtrTy(); + SmallVector Decls; DKind = ParseOpenMPDirectiveKind(*this); while (DKind != OMPD_end_declare_target && DKind != OMPD_declare_target && Tok.isNot(tok::eof) && Tok.isNot(tok::r_brace)) { @@ -781,6 +782,12 @@ else TPA.Commit(); } + + // Save the declarations so that we can create the declare target group + // later on. + if (Ptr) + for (auto *V : Ptr.get()) + Decls.push_back(V); } if (DKind == OMPD_end_declare_target) { @@ -795,8 +802,17 @@ } else { Diag(Tok, diag::err_expected_end_declare_target); Diag(DTLoc, diag::note_matching) << "'#pragma omp declare target'"; + // We have an error, so we don't have to attempt to generate code for the + // declarations. + Decls.clear(); } Actions.ActOnFinishOpenMPDeclareTargetDirective(); + + // If we have decls generate the group so that code can be generated for it + // later on. + if (!Decls.empty()) + return Actions.BuildDeclaratorGroup(Decls); + return DeclGroupPtrTy(); } case OMPD_unknown: