Index: clang/lib/CodeGen/CGOpenMPRuntime.h =================================================================== --- clang/lib/CodeGen/CGOpenMPRuntime.h +++ clang/lib/CodeGen/CGOpenMPRuntime.h @@ -520,214 +520,7 @@ QualType KmpDimTy; /// Entity that registers the offloading constants that were emitted so /// far. - class OffloadEntriesInfoManagerTy { - CodeGenModule &CGM; - - /// Number of entries registered so far. - unsigned OffloadingEntriesNum = 0; - - public: - /// Base class of the entries info. - class OffloadEntryInfo { - public: - /// Kind of a given entry. - enum OffloadingEntryInfoKinds : unsigned { - /// Entry is a target region. - OffloadingEntryInfoTargetRegion = 0, - /// Entry is a declare target variable. - OffloadingEntryInfoDeviceGlobalVar = 1, - /// Invalid entry info. - OffloadingEntryInfoInvalid = ~0u - }; - - protected: - OffloadEntryInfo() = delete; - explicit OffloadEntryInfo(OffloadingEntryInfoKinds Kind) : Kind(Kind) {} - explicit OffloadEntryInfo(OffloadingEntryInfoKinds Kind, unsigned Order, - uint32_t Flags) - : Flags(Flags), Order(Order), Kind(Kind) {} - ~OffloadEntryInfo() = default; - - public: - bool isValid() const { return Order != ~0u; } - unsigned getOrder() const { return Order; } - OffloadingEntryInfoKinds getKind() const { return Kind; } - uint32_t getFlags() const { return Flags; } - void setFlags(uint32_t NewFlags) { Flags = NewFlags; } - llvm::Constant *getAddress() const { - return cast_or_null(Addr); - } - void setAddress(llvm::Constant *V) { - assert(!Addr.pointsToAliveValue() && "Address has been set before!"); - Addr = V; - } - static bool classof(const OffloadEntryInfo *Info) { return true; } - - private: - /// Address of the entity that has to be mapped for offloading. - llvm::WeakTrackingVH Addr; - - /// Flags associated with the device global. - uint32_t Flags = 0u; - - /// Order this entry was emitted. - unsigned Order = ~0u; - - OffloadingEntryInfoKinds Kind = OffloadingEntryInfoInvalid; - }; - - /// Return true if a there are no entries defined. - bool empty() const; - /// Return number of entries defined so far. - unsigned size() const { return OffloadingEntriesNum; } - OffloadEntriesInfoManagerTy(CodeGenModule &CGM) : CGM(CGM) {} - - // - // Target region entries related. - // - - /// Kind of the target registry entry. - enum OMPTargetRegionEntryKind : uint32_t { - /// Mark the entry as target region. - OMPTargetRegionEntryTargetRegion = 0x0, - /// Mark the entry as a global constructor. - OMPTargetRegionEntryCtor = 0x02, - /// Mark the entry as a global destructor. - OMPTargetRegionEntryDtor = 0x04, - }; - - /// Target region entries info. - class OffloadEntryInfoTargetRegion final : public OffloadEntryInfo { - /// Address that can be used as the ID of the entry. - llvm::Constant *ID = nullptr; - - public: - OffloadEntryInfoTargetRegion() - : OffloadEntryInfo(OffloadingEntryInfoTargetRegion) {} - explicit OffloadEntryInfoTargetRegion(unsigned Order, - llvm::Constant *Addr, - llvm::Constant *ID, - OMPTargetRegionEntryKind Flags) - : OffloadEntryInfo(OffloadingEntryInfoTargetRegion, Order, Flags), - ID(ID) { - setAddress(Addr); - } - - llvm::Constant *getID() const { return ID; } - void setID(llvm::Constant *V) { - assert(!ID && "ID has been set before!"); - ID = V; - } - static bool classof(const OffloadEntryInfo *Info) { - return Info->getKind() == OffloadingEntryInfoTargetRegion; - } - }; - - /// Initialize target region entry. - void initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, - StringRef ParentName, unsigned LineNum, - unsigned Order); - /// Register target region entry. - void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, - StringRef ParentName, unsigned LineNum, - llvm::Constant *Addr, llvm::Constant *ID, - OMPTargetRegionEntryKind Flags); - /// Return true if a target region entry with the provided information - /// exists. - bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, - StringRef ParentName, unsigned LineNum, - bool IgnoreAddressId = false) const; - /// brief Applies action \a Action on all registered entries. - typedef llvm::function_ref - OffloadTargetRegionEntryInfoActTy; - void actOnTargetRegionEntriesInfo( - const OffloadTargetRegionEntryInfoActTy &Action); - - // - // Device global variable entries related. - // - - /// Kind of the global variable entry.. - enum OMPTargetGlobalVarEntryKind : uint32_t { - /// Mark the entry as a to declare target. - OMPTargetGlobalVarEntryTo = 0x0, - /// Mark the entry as a to declare target link. - OMPTargetGlobalVarEntryLink = 0x1, - }; - - /// Device global variable entries info. - class OffloadEntryInfoDeviceGlobalVar final : public OffloadEntryInfo { - /// Type of the global variable. - CharUnits VarSize; - llvm::GlobalValue::LinkageTypes Linkage; - - public: - OffloadEntryInfoDeviceGlobalVar() - : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar) {} - explicit OffloadEntryInfoDeviceGlobalVar(unsigned Order, - OMPTargetGlobalVarEntryKind Flags) - : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar, Order, Flags) {} - explicit OffloadEntryInfoDeviceGlobalVar( - unsigned Order, llvm::Constant *Addr, CharUnits VarSize, - OMPTargetGlobalVarEntryKind Flags, - llvm::GlobalValue::LinkageTypes Linkage) - : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar, Order, Flags), - VarSize(VarSize), Linkage(Linkage) { - setAddress(Addr); - } - - CharUnits getVarSize() const { return VarSize; } - void setVarSize(CharUnits Size) { VarSize = Size; } - llvm::GlobalValue::LinkageTypes getLinkage() const { return Linkage; } - void setLinkage(llvm::GlobalValue::LinkageTypes LT) { Linkage = LT; } - static bool classof(const OffloadEntryInfo *Info) { - return Info->getKind() == OffloadingEntryInfoDeviceGlobalVar; - } - }; - - /// Initialize device global variable entry. - void initializeDeviceGlobalVarEntryInfo(StringRef Name, - OMPTargetGlobalVarEntryKind Flags, - unsigned Order); - - /// Register device global variable entry. - void - registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr, - CharUnits VarSize, - OMPTargetGlobalVarEntryKind Flags, - llvm::GlobalValue::LinkageTypes Linkage); - /// Checks if the variable with the given name has been registered already. - bool hasDeviceGlobalVarEntryInfo(StringRef VarName) const { - return OffloadEntriesDeviceGlobalVar.count(VarName) > 0; - } - /// Applies action \a Action on all registered entries. - typedef llvm::function_ref - OffloadDeviceGlobalVarEntryInfoActTy; - void actOnDeviceGlobalVarEntriesInfo( - const OffloadDeviceGlobalVarEntryInfoActTy &Action); - - private: - // Storage for target region entries kind. The storage is to be indexed by - // file ID, device ID, parent function name and line number. - typedef llvm::DenseMap - OffloadEntriesTargetRegionPerLine; - typedef llvm::StringMap - OffloadEntriesTargetRegionPerParentName; - typedef llvm::DenseMap - OffloadEntriesTargetRegionPerFile; - typedef llvm::DenseMap - OffloadEntriesTargetRegionPerDevice; - typedef OffloadEntriesTargetRegionPerDevice OffloadEntriesTargetRegionTy; - OffloadEntriesTargetRegionTy OffloadEntriesTargetRegion; - /// Storage for device global variable entries kind. The storage is to be - /// indexed by mangled name. - typedef llvm::StringMap - OffloadEntriesDeviceGlobalVarTy; - OffloadEntriesDeviceGlobalVarTy OffloadEntriesDeviceGlobalVar; - }; - OffloadEntriesInfoManagerTy OffloadEntriesInfoManager; + llvm::OffloadEntriesInfoManager OffloadEntriesInfoManager; bool ShouldMarkAsGlobal = true; /// List of the emitted declarations. Index: clang/lib/CodeGen/CGOpenMPRuntime.cpp =================================================================== --- clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -1060,7 +1060,7 @@ CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator, StringRef Separator) : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator), - OMPBuilder(CGM.getModule()), OffloadEntriesInfoManager(CGM) { + OMPBuilder(CGM.getModule()), OffloadEntriesInfoManager() { KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8); // Initialize Types used in OpenMPIRBuilder from OMPKinds.def @@ -1914,7 +1914,8 @@ Out.clear(); OffloadEntriesInfoManager.registerTargetRegionEntryInfo( DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor, - ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor); + ID, llvm::OffloadEntriesInfoManager::OMPTargetRegionEntryCtor, + CGM.getLangOpts().OpenMPIsDevice); } if (VD->getType().isDestructedType() != QualType::DK_none) { llvm::Constant *Dtor; @@ -1960,7 +1961,8 @@ Out.clear(); OffloadEntriesInfoManager.registerTargetRegionEntryInfo( DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor, - ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor); + ID, llvm::OffloadEntriesInfoManager::OMPTargetRegionEntryDtor, + CGM.getLangOpts().OpenMPIsDevice); } return CGM.getLangOpts().OpenMPIsDevice; } @@ -2951,143 +2953,6 @@ }; } // anonymous namespace -bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const { - return OffloadEntriesTargetRegion.empty() && - OffloadEntriesDeviceGlobalVar.empty(); -} - -/// Initialize target region entry. -void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: - initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, - StringRef ParentName, unsigned LineNum, - unsigned Order) { - assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is " - "only required for the device " - "code generation."); - OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = - OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr, - OMPTargetRegionEntryTargetRegion); - ++OffloadingEntriesNum; -} - -void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: - registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, - StringRef ParentName, unsigned LineNum, - llvm::Constant *Addr, llvm::Constant *ID, - OMPTargetRegionEntryKind Flags) { - // If we are emitting code for a target, the entry is already initialized, - // only has to be registered. - if (CGM.getLangOpts().OpenMPIsDevice) { - // This could happen if the device compilation is invoked standalone. - if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) - return; - auto &Entry = - OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum]; - Entry.setAddress(Addr); - Entry.setID(ID); - Entry.setFlags(Flags); - } else { - if (Flags == - OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion && - hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum, - /*IgnoreAddressId*/ true)) - return; - assert(!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) && - "Target region entry already registered!"); - OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags); - OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry; - ++OffloadingEntriesNum; - } -} - -bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo( - unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, - bool IgnoreAddressId) const { - auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID); - if (PerDevice == OffloadEntriesTargetRegion.end()) - return false; - auto PerFile = PerDevice->second.find(FileID); - if (PerFile == PerDevice->second.end()) - return false; - auto PerParentName = PerFile->second.find(ParentName); - if (PerParentName == PerFile->second.end()) - return false; - auto PerLine = PerParentName->second.find(LineNum); - if (PerLine == PerParentName->second.end()) - return false; - // Fail if this entry is already registered. - if (!IgnoreAddressId && - (PerLine->second.getAddress() || PerLine->second.getID())) - return false; - return true; -} - -void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo( - const OffloadTargetRegionEntryInfoActTy &Action) { - // Scan all target region entries and perform the provided action. - for (const auto &D : OffloadEntriesTargetRegion) - for (const auto &F : D.second) - for (const auto &P : F.second) - for (const auto &L : P.second) - Action(D.first, F.first, P.first(), L.first, L.second); -} - -void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: - initializeDeviceGlobalVarEntryInfo(StringRef Name, - OMPTargetGlobalVarEntryKind Flags, - unsigned Order) { - assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is " - "only required for the device " - "code generation."); - OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags); - ++OffloadingEntriesNum; -} - -void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: - registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr, - CharUnits VarSize, - OMPTargetGlobalVarEntryKind Flags, - llvm::GlobalValue::LinkageTypes Linkage) { - if (CGM.getLangOpts().OpenMPIsDevice) { - // This could happen if the device compilation is invoked standalone. - if (!hasDeviceGlobalVarEntryInfo(VarName)) - return; - auto &Entry = OffloadEntriesDeviceGlobalVar[VarName]; - if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) { - if (Entry.getVarSize().isZero()) { - Entry.setVarSize(VarSize); - Entry.setLinkage(Linkage); - } - return; - } - Entry.setVarSize(VarSize); - Entry.setLinkage(Linkage); - Entry.setAddress(Addr); - } else { - if (hasDeviceGlobalVarEntryInfo(VarName)) { - auto &Entry = OffloadEntriesDeviceGlobalVar[VarName]; - assert(Entry.isValid() && Entry.getFlags() == Flags && - "Entry not initialized!"); - if (Entry.getVarSize().isZero()) { - Entry.setVarSize(VarSize); - Entry.setLinkage(Linkage); - } - return; - } - OffloadEntriesDeviceGlobalVar.try_emplace( - VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage); - ++OffloadingEntriesNum; - } -} - -void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: - actOnDeviceGlobalVarEntriesInfo( - const OffloadDeviceGlobalVarEntryInfoActTy &Action) { - // Scan all target region entries and perform the provided action. - for (const auto &E : OffloadEntriesDeviceGlobalVar) - Action(E.getKey(), E.getValue()); -} - void CGOpenMPRuntime::createOffloadEntry( llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags, llvm::GlobalValue::LinkageTypes Linkage) { @@ -3111,9 +2976,10 @@ llvm::Module &M = CGM.getModule(); llvm::LLVMContext &C = M.getContext(); - SmallVector, - 16> + SmallVector< + std::tuple, + 16> OrderedEntries(OffloadEntriesInfoManager.size()); llvm::SmallVector ParentFunctions( OffloadEntriesInfoManager.size()); @@ -3135,7 +3001,8 @@ &GetMDString]( unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned Line, - const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) { + const llvm::OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion + &E) { // Generate metadata for target regions. Each entry of this metadata // contains: // - Entry 0 -> Kind of this type of metadata (0). @@ -3174,10 +3041,10 @@ // Create function that emits metadata for each device global variable entry; auto &&DeviceGlobalVarMetadataEmitter = - [&C, &OrderedEntries, &GetMDInt, &GetMDString, - MD](StringRef MangledName, - const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar - &E) { + [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD]( + StringRef MangledName, + const llvm::OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar + &E) { // Generate metadata for global variables. Each entry of this metadata // contains: // - Entry 0 -> Kind of this type of metadata (1). @@ -3202,9 +3069,9 @@ for (const auto &E : OrderedEntries) { assert(std::get<0>(E) && "All ordered entries must exist!"); - if (const auto *CE = - dyn_cast( - std::get<0>(E))) { + if (const auto *CE = dyn_cast< + llvm::OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>( + std::get<0>(E))) { if (!CE->getID() || !CE->getAddress()) { // Do not blame the entry if the parent funtion is not emitted. StringRef FnName = ParentFunctions[CE->getOrder()]; @@ -3219,14 +3086,15 @@ } createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0, CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage); - } else if (const auto *CE = dyn_cast( std::get<0>(E))) { - OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags = - static_cast( + llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags = + static_cast< + llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>( CE->getFlags()); switch (Flags) { - case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: { + case llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo: { if (CGM.getLangOpts().OpenMPIsDevice && CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()) continue; @@ -3239,11 +3107,11 @@ continue; } // The vaiable has no definition - no need to add the entry. - if (CE->getVarSize().isZero()) + if (CE->getVarSize() == 0) continue; break; } - case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink: + case llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink: assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && "Declaret target link address is set."); @@ -3266,9 +3134,8 @@ if (GV->hasLocalLinkage() || GV->hasHiddenVisibility()) continue; - createOffloadEntry(CE->getAddress(), CE->getAddress(), - CE->getVarSize().getQuantity(), Flags, - CE->getLinkage()); + createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(), + Flags, CE->getLinkage()); } else { llvm_unreachable("Unsupported entry kind."); } @@ -3325,18 +3192,25 @@ default: llvm_unreachable("Unexpected metadata!"); break; - case OffloadEntriesInfoManagerTy::OffloadEntryInfo:: + case llvm::OffloadEntriesInfoManager::OffloadEntryInfo:: OffloadingEntryInfoTargetRegion: + assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is " + "only required for the " + "device code generation."); OffloadEntriesInfoManager.initializeTargetRegionEntryInfo( /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2), /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4), /*Order=*/GetMDInt(5)); break; - case OffloadEntriesInfoManagerTy::OffloadEntryInfo:: + case llvm::OffloadEntriesInfoManager::OffloadEntryInfo:: OffloadingEntryInfoDeviceGlobalVar: + assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is " + "only required for the " + "device code generation."); OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo( /*MangledName=*/GetMDString(1), - static_cast( + static_cast< + llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>( /*Flags=*/GetMDInt(2)), /*Order=*/GetMDInt(3)); break; @@ -6482,7 +6356,8 @@ // Register the information for the entry associated with this target region. OffloadEntriesInfoManager.registerTargetRegionEntryInfo( DeviceID, FileID, ParentName, Line, TargetRegionEntryAddr, OutlinedFnID, - OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion); + llvm::OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion, + CGM.getLangOpts().OpenMPIsDevice); // Add NumTeams and ThreadLimit attributes to the outlined GPU function int32_t DefaultValTeams = -1; @@ -10715,20 +10590,21 @@ return; } // Register declare target variables. - OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags; + llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags; StringRef VarName; - CharUnits VarSize; + int64_t VarSize; llvm::GlobalValue::LinkageTypes Linkage; if (*Res == OMPDeclareTargetDeclAttr::MT_To && !HasRequiresUnifiedSharedMemory) { - Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo; + Flags = llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo; VarName = CGM.getMangledName(VD); if (VD->hasDefinition(CGM.getContext()) != VarDecl::DeclarationOnly) { - VarSize = CGM.getContext().getTypeSizeInChars(VD->getType()); - assert(!VarSize.isZero() && "Expected non-zero size of the variable"); + VarSize = + CGM.getContext().getTypeSizeInChars(VD->getType()).getQuantity(); + assert(VarSize != 0 && "Expected non-zero size of the variable"); } else { - VarSize = CharUnits::Zero(); + VarSize = 0; } Linkage = CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false); // Temp solution to prevent optimizations of the internal variables. @@ -10754,9 +10630,9 @@ HasRequiresUnifiedSharedMemory)) && "Declare target attribute must link or to with unified memory."); if (*Res == OMPDeclareTargetDeclAttr::MT_Link) - Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink; + Flags = llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink; else - Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo; + Flags = llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo; if (CGM.getLangOpts().OpenMPIsDevice) { VarName = Addr->getName(); @@ -10765,12 +10641,12 @@ VarName = getAddrOfDeclareTargetVar(VD).getName(); Addr = cast(getAddrOfDeclareTargetVar(VD).getPointer()); } - VarSize = CGM.getPointerSize(); + VarSize = CGM.getPointerSize().getQuantity(); Linkage = llvm::GlobalValue::WeakAnyLinkage; } OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo( - VarName, Addr, VarSize, Flags, Linkage); + VarName, Addr, VarSize, Flags, Linkage, CGM.getLangOpts().OpenMPIsDevice); } bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) { Index: llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h =================================================================== --- llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -1664,6 +1664,212 @@ const Twine &Name = {}); }; +/// Class that manages information about offload code regions and data +class OffloadEntriesInfoManager { + /// Number of entries registered so far. + unsigned OffloadingEntriesNum = 0; + +public: + /// Base class of the entries info. + class OffloadEntryInfo { + public: + /// Kind of a given entry. + enum OffloadingEntryInfoKinds : unsigned { + /// Entry is a target region. + OffloadingEntryInfoTargetRegion = 0, + /// Entry is a declare target variable. + OffloadingEntryInfoDeviceGlobalVar = 1, + /// Invalid entry info. + OffloadingEntryInfoInvalid = ~0u + }; + + protected: + OffloadEntryInfo() = delete; + explicit OffloadEntryInfo(OffloadingEntryInfoKinds Kind) : Kind(Kind) {} + explicit OffloadEntryInfo(OffloadingEntryInfoKinds Kind, unsigned Order, + uint32_t Flags) + : Flags(Flags), Order(Order), Kind(Kind) {} + ~OffloadEntryInfo() = default; + + public: + bool isValid() const { return Order != ~0u; } + unsigned getOrder() const { return Order; } + OffloadingEntryInfoKinds getKind() const { return Kind; } + uint32_t getFlags() const { return Flags; } + void setFlags(uint32_t NewFlags) { Flags = NewFlags; } + Constant *getAddress() const { return cast_or_null(Addr); } + void setAddress(Constant *V) { + assert(!Addr.pointsToAliveValue() && "Address has been set before!"); + Addr = V; + } + static bool classof(const OffloadEntryInfo *Info) { return true; } + + private: + /// Address of the entity that has to be mapped for offloading. + WeakTrackingVH Addr; + + /// Flags associated with the device global. + uint32_t Flags = 0u; + + /// Order this entry was emitted. + unsigned Order = ~0u; + + OffloadingEntryInfoKinds Kind = OffloadingEntryInfoInvalid; + }; + + /// Return true if a there are no entries defined. + bool empty() const; + /// Return number of entries defined so far. + unsigned size() const { return OffloadingEntriesNum; } + explicit OffloadEntriesInfoManager() {} + + // + // Target region entries related. + // + + /// Kind of the target registry entry. + enum OMPTargetRegionEntryKind : uint32_t { + /// Mark the entry as target region. + OMPTargetRegionEntryTargetRegion = 0x0, + /// Mark the entry as a global constructor. + OMPTargetRegionEntryCtor = 0x02, + /// Mark the entry as a global destructor. + OMPTargetRegionEntryDtor = 0x04, + }; + + /// Target region entries info. + class OffloadEntryInfoTargetRegion final : public OffloadEntryInfo { + /// Address that can be used as the ID of the entry. + Constant *ID = nullptr; + + public: + OffloadEntryInfoTargetRegion() + : OffloadEntryInfo(OffloadingEntryInfoTargetRegion) {} + explicit OffloadEntryInfoTargetRegion(unsigned Order, Constant *Addr, + Constant *ID, + OMPTargetRegionEntryKind Flags) + : OffloadEntryInfo(OffloadingEntryInfoTargetRegion, Order, Flags), + ID(ID) { + setAddress(Addr); + } + + Constant *getID() const { return ID; } + void setID(Constant *V) { + assert(!ID && "ID has been set before!"); + ID = V; + } + static bool classof(const OffloadEntryInfo *Info) { + return Info->getKind() == OffloadingEntryInfoTargetRegion; + } + }; + + /// Initialize target region entry. + /// This is ONLY needed for DEVICE compilation. + void initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, + StringRef ParentName, unsigned LineNum, + unsigned Order); + /// Register target region entry. + void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, + StringRef ParentName, unsigned LineNum, + Constant *Addr, Constant *ID, + OMPTargetRegionEntryKind Flags, + bool IsDevice); + /// Return true if a target region entry with the provided information + /// exists. + bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, + StringRef ParentName, unsigned LineNum, + bool IgnoreAddressId = false) const; + /// brief Applies action \a Action on all registered entries. + typedef function_ref + OffloadTargetRegionEntryInfoActTy; + void + actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action); + + // + // Device global variable entries related. + // + + /// Kind of the global variable entry.. + enum OMPTargetGlobalVarEntryKind : uint32_t { + /// Mark the entry as a to declare target. + OMPTargetGlobalVarEntryTo = 0x0, + /// Mark the entry as a to declare target link. + OMPTargetGlobalVarEntryLink = 0x1, + }; + + /// Device global variable entries info. + class OffloadEntryInfoDeviceGlobalVar final : public OffloadEntryInfo { + /// Type of the global variable. + int64_t VarSize; + GlobalValue::LinkageTypes Linkage; + + public: + OffloadEntryInfoDeviceGlobalVar() + : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar) {} + explicit OffloadEntryInfoDeviceGlobalVar(unsigned Order, + OMPTargetGlobalVarEntryKind Flags) + : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar, Order, Flags) {} + explicit OffloadEntryInfoDeviceGlobalVar(unsigned Order, Constant *Addr, + int64_t VarSize, + OMPTargetGlobalVarEntryKind Flags, + GlobalValue::LinkageTypes Linkage) + : OffloadEntryInfo(OffloadingEntryInfoDeviceGlobalVar, Order, Flags), + VarSize(VarSize), Linkage(Linkage) { + setAddress(Addr); + } + + int64_t getVarSize() const { return VarSize; } + void setVarSize(int64_t Size) { VarSize = Size; } + GlobalValue::LinkageTypes getLinkage() const { return Linkage; } + void setLinkage(GlobalValue::LinkageTypes LT) { Linkage = LT; } + static bool classof(const OffloadEntryInfo *Info) { + return Info->getKind() == OffloadingEntryInfoDeviceGlobalVar; + } + }; + + /// Initialize device global variable entry. + /// This is ONLY used for DEVICE compilation. + void initializeDeviceGlobalVarEntryInfo(StringRef Name, + OMPTargetGlobalVarEntryKind Flags, + unsigned Order); + + /// Register device global variable entry. + void registerDeviceGlobalVarEntryInfo(StringRef VarName, Constant *Addr, + int64_t VarSize, + OMPTargetGlobalVarEntryKind Flags, + GlobalValue::LinkageTypes Linkage, + bool IsDevice); + /// Checks if the variable with the given name has been registered already. + bool hasDeviceGlobalVarEntryInfo(StringRef VarName) const { + return OffloadEntriesDeviceGlobalVar.count(VarName) > 0; + } + /// Applies action \a Action on all registered entries. + typedef function_ref + OffloadDeviceGlobalVarEntryInfoActTy; + void actOnDeviceGlobalVarEntriesInfo( + const OffloadDeviceGlobalVarEntryInfoActTy &Action); + +private: + // Storage for target region entries kind. The storage is to be indexed by + // file ID, device ID, parent function name and line number. + typedef DenseMap + OffloadEntriesTargetRegionPerLine; + typedef StringMap + OffloadEntriesTargetRegionPerParentName; + typedef DenseMap + OffloadEntriesTargetRegionPerFile; + typedef DenseMap + OffloadEntriesTargetRegionPerDevice; + typedef OffloadEntriesTargetRegionPerDevice OffloadEntriesTargetRegionTy; + OffloadEntriesTargetRegionTy OffloadEntriesTargetRegion; + /// Storage for device global variable entries kind. The storage is to be + /// indexed by mangled name. + typedef StringMap + OffloadEntriesDeviceGlobalVarTy; + OffloadEntriesDeviceGlobalVarTy OffloadEntriesDeviceGlobalVar; +}; + /// Class to represented the control flow structure of an OpenMP canonical loop. /// /// The control-flow structure is standardized for easy consumption by Index: llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp =================================================================== --- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -4624,6 +4624,130 @@ } } +bool OffloadEntriesInfoManager::empty() const { + return OffloadEntriesTargetRegion.empty() && + OffloadEntriesDeviceGlobalVar.empty(); +} + +/// Initialize target region entry. +void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo( + unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, + unsigned Order) { + OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = + OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr, + OMPTargetRegionEntryTargetRegion); + ++OffloadingEntriesNum; +} + +void OffloadEntriesInfoManager::registerTargetRegionEntryInfo( + unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, + Constant *Addr, Constant *ID, OMPTargetRegionEntryKind Flags, + bool IsDevice) { + // If we are emitting code for a target, the entry is already initialized, + // only has to be registered. + if (IsDevice) { + // This could happen if the device compilation is invoked standalone. + if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) + return; + auto &Entry = + OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum]; + Entry.setAddress(Addr); + Entry.setID(ID); + Entry.setFlags(Flags); + } else { + if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion && + hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum, + /*IgnoreAddressId*/ true)) + return; + assert(!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) && + "Target region entry already registered!"); + OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags); + OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry; + ++OffloadingEntriesNum; + } +} + +bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo( + unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, + bool IgnoreAddressId) const { + auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID); + if (PerDevice == OffloadEntriesTargetRegion.end()) + return false; + auto PerFile = PerDevice->second.find(FileID); + if (PerFile == PerDevice->second.end()) + return false; + auto PerParentName = PerFile->second.find(ParentName); + if (PerParentName == PerFile->second.end()) + return false; + auto PerLine = PerParentName->second.find(LineNum); + if (PerLine == PerParentName->second.end()) + return false; + // Fail if this entry is already registered. + if (!IgnoreAddressId && + (PerLine->second.getAddress() || PerLine->second.getID())) + return false; + return true; +} + +void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo( + const OffloadTargetRegionEntryInfoActTy &Action) { + // Scan all target region entries and perform the provided action. + for (const auto &D : OffloadEntriesTargetRegion) + for (const auto &F : D.second) + for (const auto &P : F.second) + for (const auto &L : P.second) + Action(D.first, F.first, P.first(), L.first, L.second); +} + +void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo( + StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) { + OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags); + ++OffloadingEntriesNum; +} + +void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo( + StringRef VarName, Constant *Addr, int64_t VarSize, + OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage, + bool IsDevice) { + if (IsDevice) { + // This could happen if the device compilation is invoked standalone. + if (!hasDeviceGlobalVarEntryInfo(VarName)) + return; + auto &Entry = OffloadEntriesDeviceGlobalVar[VarName]; + if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) { + if (Entry.getVarSize() == 0) { + Entry.setVarSize(VarSize); + Entry.setLinkage(Linkage); + } + return; + } + Entry.setVarSize(VarSize); + Entry.setLinkage(Linkage); + Entry.setAddress(Addr); + } else { + if (hasDeviceGlobalVarEntryInfo(VarName)) { + auto &Entry = OffloadEntriesDeviceGlobalVar[VarName]; + assert(Entry.isValid() && Entry.getFlags() == Flags && + "Entry not initialized!"); + if (Entry.getVarSize() == 0) { + Entry.setVarSize(VarSize); + Entry.setLinkage(Linkage); + } + return; + } + OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum, + Addr, VarSize, Flags, Linkage); + ++OffloadingEntriesNum; + } +} + +void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo( + const OffloadDeviceGlobalVarEntryInfoActTy &Action) { + // Scan all target region entries and perform the provided action. + for (const auto &E : OffloadEntriesDeviceGlobalVar) + Action(E.getKey(), E.getValue()); +} + void CanonicalLoopInfo::collectControlBlocks( SmallVectorImpl &BBs) { // We only count those BBs as control block for which we do not need to Index: llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp =================================================================== --- llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -5359,4 +5359,19 @@ EXPECT_EQ(RTArgs.MappersArray->getType(), VoidPtrPtrTy); EXPECT_EQ(RTArgs.MapNamesArray->getType(), VoidPtrPtrTy); } + +TEST_F(OpenMPIRBuilderTest, OffloadEntriesInfoManager) { + OffloadEntriesInfoManager InfoManager; + InfoManager.initializeTargetRegionEntryInfo(1, 2, "parent", 4, 0); + InfoManager.initializeDeviceGlobalVarEntryInfo( + "gvar", OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo, 0); + InfoManager.registerTargetRegionEntryInfo( + 1, 2, "parent", 4, nullptr, nullptr, + OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion, true); + InfoManager.registerDeviceGlobalVarEntryInfo( + "gvar", 0x0, 8, OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo, + GlobalValue::WeakAnyLinkage, true); + EXPECT_TRUE(InfoManager.hasTargetRegionEntryInfo(1, 2, "parent", 4, true)); + EXPECT_TRUE(InfoManager.hasDeviceGlobalVarEntryInfo("gvar")); +} } // namespace