Index: clang/lib/CodeGen/CGOpenMPRuntime.h =================================================================== --- clang/lib/CodeGen/CGOpenMPRuntime.h +++ clang/lib/CodeGen/CGOpenMPRuntime.h @@ -315,12 +315,6 @@ explicit CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator, StringRef Separator); - /// Creates offloading entry for the provided entry ID \a ID, - /// address \a Addr, size \a Size, and flags \a Flags. - virtual void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, - uint64_t Size, int32_t Flags, - llvm::GlobalValue::LinkageTypes Linkage); - /// Helper to emit outlined function for 'target' directive. /// \param D Directive to emit. /// \param ParentName Name of the function that encloses the target region. @@ -713,6 +707,9 @@ virtual ~CGOpenMPRuntime() {} virtual void clear(); + /// Returns true if the current target is a GPU. + virtual bool isTargetCodegen() const { return false; } + /// Emits code for OpenMP 'if' clause using specified \a CodeGen /// function. Here is the logic: /// if (Cond) { Index: clang/lib/CodeGen/CGOpenMPRuntime.cpp =================================================================== --- clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -2950,194 +2950,57 @@ }; } // anonymous namespace -void CGOpenMPRuntime::createOffloadEntry( - llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags, - llvm::GlobalValue::LinkageTypes Linkage) { - OMPBuilder.emitOffloadingEntry(ID, Addr->getName(), Size, Flags); -} - void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() { - // Emit the offloading entries and metadata so that the device codegen side - // can easily figure out what to emit. The produced metadata looks like - // this: - // - // !omp_offload.info = !{!1, ...} - // - // Right now we only generate metadata for function that contain target - // regions. - // If we are in simd mode or there are no entries, we don't need to do // anything. if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty()) return; - llvm::Module &M = CGM.getModule(); - llvm::LLVMContext &C = M.getContext(); - SmallVector< - std::tuple, - 16> - OrderedEntries(OffloadEntriesInfoManager.size()); - llvm::SmallVector ParentFunctions( - OffloadEntriesInfoManager.size()); - - // Auxiliary methods to create metadata values and strings. - auto &&GetMDInt = [this](unsigned V) { - return llvm::ConstantAsMetadata::get( - llvm::ConstantInt::get(CGM.Int32Ty, V)); - }; - - auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); }; - - // Create the offloading info metadata node. - llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info"); - - // Create function that emits metadata for each target region entry; - auto &&TargetRegionMetadataEmitter = - [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt, - &GetMDString]( - const llvm::TargetRegionEntryInfo &EntryInfo, - const llvm::OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion - &E) { - // Generate metadata for target regions. Each entry of this metadata - // contains: - // - Entry 0 -> Kind of this type of metadata (0). - // - Entry 1 -> Device ID of the file where the entry was identified. - // - Entry 2 -> File ID of the file where the entry was identified. - // - Entry 3 -> Mangled name of the function where the entry was - // identified. - // - Entry 4 -> Line in the file where the entry was identified. - // - Entry 5 -> Order the entry was created. - // The first element of the metadata node is the kind. - llvm::Metadata *Ops[] = { - GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID), - GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName), - GetMDInt(EntryInfo.Line), GetMDInt(E.getOrder())}; - - SourceLocation Loc; - for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(), - E = CGM.getContext().getSourceManager().fileinfo_end(); - I != E; ++I) { - if (I->getFirst()->getUniqueID().getDevice() == EntryInfo.DeviceID && - I->getFirst()->getUniqueID().getFile() == EntryInfo.FileID) { - Loc = CGM.getContext().getSourceManager().translateFileLineCol( - I->getFirst(), EntryInfo.Line, 1); - break; - } - } - // Save this entry in the right position of the ordered entries array. - OrderedEntries[E.getOrder()] = - std::make_tuple(&E, Loc, StringRef(EntryInfo.ParentName)); - ParentFunctions[E.getOrder()] = StringRef(EntryInfo.ParentName); - - // Add metadata to the named metadata node. - MD->addOperand(llvm::MDNode::get(C, Ops)); - }; - - OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo( - TargetRegionMetadataEmitter); - - // Create function that emits metadata for each device global variable entry; - auto &&DeviceGlobalVarMetadataEmitter = - [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD]( - StringRef MangledName, - const llvm::OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar - &E) { - // Generate metadata for global variables. Each entry of this metadata - // contains: - // - Entry 0 -> Kind of this type of metadata (1). - // - Entry 1 -> Mangled name of the variable. - // - Entry 2 -> Declare target kind. - // - Entry 3 -> Order the entry was created. - // The first element of the metadata node is the kind. - llvm::Metadata *Ops[] = { - GetMDInt(E.getKind()), GetMDString(MangledName), - GetMDInt(E.getFlags()), GetMDInt(E.getOrder())}; - - // Save this entry in the right position of the ordered entries array. - OrderedEntries[E.getOrder()] = - std::make_tuple(&E, SourceLocation(), MangledName); - - // Add metadata to the named metadata node. - MD->addOperand(llvm::MDNode::get(C, Ops)); - }; - - OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo( - DeviceGlobalVarMetadataEmitter); - - for (const auto &E : OrderedEntries) { - assert(std::get<0>(E) && "All ordered entries must exist!"); - if (const auto *CE = dyn_cast< - llvm::OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>( - std::get<0>(E))) { - if (!CE->getID() || !CE->getAddress()) { - // Do not blame the entry if the parent funtion is not emitted. - StringRef FnName = ParentFunctions[CE->getOrder()]; - if (!CGM.GetGlobalValue(FnName)) - continue; - unsigned DiagID = CGM.getDiags().getCustomDiagID( - DiagnosticsEngine::Error, - "Offloading entry for target region in %0 is incorrect: either the " - "address or the ID is invalid."); - CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName; - continue; - } - createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0, - CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage); - } else if (const auto *CE = dyn_cast( - std::get<0>(E))) { - llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags = - static_cast< - llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>( - CE->getFlags()); - switch (Flags) { - case llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo: { - if (CGM.getLangOpts().OpenMPIsDevice && - CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()) - continue; - if (!CE->getAddress()) { - unsigned DiagID = CGM.getDiags().getCustomDiagID( - DiagnosticsEngine::Error, "Offloading entry for declare target " - "variable %0 is incorrect: the " - "address is invalid."); - CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E); - continue; - } - // The vaiable has no definition - no need to add the entry. - if (CE->getVarSize() == 0) - continue; - break; - } - case llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink: - assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || - (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && - "Declaret target link address is set."); - if (CGM.getLangOpts().OpenMPIsDevice) - continue; - if (!CE->getAddress()) { - unsigned DiagID = CGM.getDiags().getCustomDiagID( - DiagnosticsEngine::Error, - "Offloading entry for declare target variable is incorrect: the " - "address is invalid."); - CGM.getDiags().Report(DiagID); - continue; + llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn = + [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind, + const llvm::TargetRegionEntryInfo &EntryInfo) -> void { + SourceLocation Loc; + if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) { + for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(), + E = CGM.getContext().getSourceManager().fileinfo_end(); + I != E; ++I) { + if (I->getFirst()->getUniqueID().getDevice() == EntryInfo.DeviceID && + I->getFirst()->getUniqueID().getFile() == EntryInfo.FileID) { + Loc = CGM.getContext().getSourceManager().translateFileLineCol( + I->getFirst(), EntryInfo.Line, 1); + break; } - break; } - - // Hidden or internal symbols on the device are not externally visible. We - // should not attempt to register them by creating an offloading entry. - if (auto *GV = dyn_cast(CE->getAddress())) - if (GV->hasLocalLinkage() || GV->hasHiddenVisibility()) - continue; - - createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(), - Flags, CE->getLinkage()); - } else { - llvm_unreachable("Unsupported entry kind."); } - } + switch (Kind) { + case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: { + unsigned DiagID = CGM.getDiags().getCustomDiagID( + DiagnosticsEngine::Error, "Offloading entry for target region in " + "%0 is incorrect: either the " + "address or the ID is invalid."); + CGM.getDiags().Report(Loc, DiagID) << EntryInfo.ParentName; + } break; + case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: { + unsigned DiagID = CGM.getDiags().getCustomDiagID( + DiagnosticsEngine::Error, "Offloading entry for declare target " + "variable %0 is incorrect: the " + "address is invalid."); + CGM.getDiags().Report(Loc, DiagID) << EntryInfo.ParentName; + } break; + case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: { + unsigned DiagID = CGM.getDiags().getCustomDiagID( + DiagnosticsEngine::Error, + "Offloading entry for declare target variable is incorrect: the " + "address is invalid."); + CGM.getDiags().Report(DiagID); + } break; + } + }; + + OMPBuilder.createOffloadEntriesAndInfoMetadata( + OffloadEntriesInfoManager, isTargetCodegen(), + CGM.getLangOpts().OpenMPIsDevice, + CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory(), ErrorReportFn); } /// Loads all the offload entries information from the host IR Index: clang/lib/CodeGen/CGOpenMPRuntimeGPU.h =================================================================== --- clang/lib/CodeGen/CGOpenMPRuntimeGPU.h +++ clang/lib/CodeGen/CGOpenMPRuntimeGPU.h @@ -64,12 +64,6 @@ // Base class overrides. // - /// Creates offloading entry for the provided entry ID \a ID, - /// address \a Addr, size \a Size, and flags \a Flags. - void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, - uint64_t Size, int32_t Flags, - llvm::GlobalValue::LinkageTypes Linkage) override; - /// Emit outlined function specialized for the Fork-Join /// programming model for applicable target directives on the NVPTX device. /// \param D Directive to emit. @@ -169,6 +163,8 @@ explicit CGOpenMPRuntimeGPU(CodeGenModule &CGM); void clear() override; + bool isTargetCodegen() const override { return true; }; + /// Declare generalized virtual functions which need to be defined /// by all specializations of OpenMPGPURuntime Targets like AMDGCN /// and NVPTX. Index: clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp =================================================================== --- clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -840,33 +840,6 @@ CGM.addCompilerUsedGlobal(GVMode); } -void CGOpenMPRuntimeGPU::createOffloadEntry(llvm::Constant *ID, - llvm::Constant *Addr, - uint64_t Size, int32_t, - llvm::GlobalValue::LinkageTypes) { - // TODO: Add support for global variables on the device after declare target - // support. - llvm::Function *Fn = dyn_cast(Addr); - if (!Fn) - return; - - llvm::Module &M = CGM.getModule(); - llvm::LLVMContext &Ctx = CGM.getLLVMContext(); - - // Get "nvvm.annotations" metadata node. - llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations"); - - llvm::Metadata *MDVals[] = { - llvm::ConstantAsMetadata::get(Fn), llvm::MDString::get(Ctx, "kernel"), - llvm::ConstantAsMetadata::get( - llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))}; - // Append metadata to nvvm.annotations. - MD->addOperand(llvm::MDNode::get(Ctx, MDVals)); - - // Add a function attribute for the kernel. - Fn->addFnAttr(llvm::Attribute::get(Ctx, "kernel")); -} - void CGOpenMPRuntimeGPU::emitTargetOutlinedFunction( const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, Index: llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h =================================================================== --- llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -24,6 +24,7 @@ namespace llvm { class CanonicalLoopInfo; +struct TargetRegionEntryInfo; class OffloadEntriesInfoManager; /// Move the instruction after an InsertPoint to the beginning of another @@ -1093,6 +1094,37 @@ bool EmitDebug = false, bool ForEndCall = false); + /// Creates offloading entry for the provided entry ID \a ID, + /// address \a Addr, size \a Size, and flags \a Flags. + void createOffloadEntry(bool IsTargetCodegen, Constant *ID, Constant *Addr, + uint64_t Size, int32_t Flags, + GlobalValue::LinkageTypes); + + /// The kind of errors that can occur when emitting the offload entries and + /// metadata. + enum EmitMetadataErrorKind { + EMIT_MD_TARGET_REGION_ERROR, + EMIT_MD_DECLARE_TARGET_ERROR, + EMIT_MD_GLOBAL_VAR_LINK_ERROR + }; + + /// Callback function type + using EmitMetadataErrorReportFunctionTy = + std::function; + + // Emit the offloading entries and metadata so that the device codegen side + // can easily figure out what to emit. The produced metadata looks like + // this: + // + // !omp_offload.info = !{!1, ...} + // + // We only generate metadata for function that contain target regions. + void createOffloadEntriesAndInfoMetadata( + OffloadEntriesInfoManager &OffloadEntriesInfoManager, + bool IsTargetCodegen, bool IsEmbedded, + bool HasRequiresUnifiedSharedMemory, + EmitMetadataErrorReportFunctionTy &ErrorReportFunction); + public: /// Generator for __kmpc_copyprivate /// Index: llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp =================================================================== --- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -4692,6 +4692,181 @@ } } +void OpenMPIRBuilder::createOffloadEntry(bool IsTargetCodegen, Constant *ID, + Constant *Addr, uint64_t Size, + int32_t Flags, + GlobalValue::LinkageTypes) { + if (!IsTargetCodegen) { + emitOffloadingEntry(ID, Addr->getName(), Size, Flags); + return; + } + // TODO: Add support for global variables on the device after declare target + // support. + Function *Fn = dyn_cast(Addr); + if (!Fn) + return; + + Module &M = *(Fn->getParent()); + LLVMContext &Ctx = M.getContext(); + + // Get "nvvm.annotations" metadata node. + NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations"); + + Metadata *MDVals[] = { + ConstantAsMetadata::get(Fn), MDString::get(Ctx, "kernel"), + ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(Ctx), 1))}; + // Append metadata to nvvm.annotations. + MD->addOperand(MDNode::get(Ctx, MDVals)); + + // Add a function attribute for the kernel. + Fn->addFnAttr(Attribute::get(Ctx, "kernel")); +} + +// We only generate metadata for function that contain target regions. +void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata( + OffloadEntriesInfoManager &OffloadEntriesInfoManager, bool IsTargetCodegen, + bool IsEmbedded, bool HasRequiresUnifiedSharedMemory, + EmitMetadataErrorReportFunctionTy &ErrorFn) { + + // If there are no entries, we don't need to do anything. + if (OffloadEntriesInfoManager.empty()) + return; + + LLVMContext &C = M.getContext(); + SmallVector, + 16> + OrderedEntries(OffloadEntriesInfoManager.size()); + + // Auxiliary methods to create metadata values and strings. + auto &&GetMDInt = [this](unsigned V) { + return ConstantAsMetadata::get(ConstantInt::get(Builder.getInt32Ty(), V)); + }; + + auto &&GetMDString = [&C](StringRef V) { return MDString::get(C, V); }; + + // Create the offloading info metadata node. + NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info"); + auto &&TargetRegionMetadataEmitter = + [this, &C, MD, &OrderedEntries, &GetMDInt, &GetMDString]( + const TargetRegionEntryInfo &EntryInfo, + const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) { + // Generate metadata for target regions. Each entry of this metadata + // contains: + // - Entry 0 -> Kind of this type of metadata (0). + // - Entry 1 -> Device ID of the file where the entry was identified. + // - Entry 2 -> File ID of the file where the entry was identified. + // - Entry 3 -> Mangled name of the function where the entry was + // identified. + // - Entry 4 -> Line in the file where the entry was identified. + // - Entry 5 -> Order the entry was created. + // The first element of the metadata node is the kind. + Metadata *Ops[] = { + GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID), + GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName), + GetMDInt(EntryInfo.Line), GetMDInt(E.getOrder())}; + + // Save this entry in the right position of the ordered entries array. + OrderedEntries[E.getOrder()] = std::make_pair(&E, EntryInfo); + + // Add metadata to the named metadata node. + MD->addOperand(MDNode::get(C, Ops)); + }; + + OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo( + TargetRegionMetadataEmitter); + + // Create function that emits metadata for each device global variable entry; + auto &&DeviceGlobalVarMetadataEmitter = + [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD]( + StringRef MangledName, + const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) { + // Generate metadata for global variables. Each entry of this metadata + // contains: + // - Entry 0 -> Kind of this type of metadata (1). + // - Entry 1 -> Mangled name of the variable. + // - Entry 2 -> Declare target kind. + // - Entry 3 -> Order the entry was created. + // The first element of the metadata node is the kind. + Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName), + GetMDInt(E.getFlags()), GetMDInt(E.getOrder())}; + + // Save this entry in the right position of the ordered entries array. + TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0); + OrderedEntries[E.getOrder()] = std::make_pair(&E, varInfo); + + // Add metadata to the named metadata node. + MD->addOperand(MDNode::get(C, Ops)); + }; + + OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo( + DeviceGlobalVarMetadataEmitter); + + for (const auto &E : OrderedEntries) { + assert(E.first && "All ordered entries must exist!"); + if (const auto *CE = + dyn_cast( + E.first)) { + if (!CE->getID() || !CE->getAddress()) { + // Do not blame the entry if the parent funtion is not emitted. + TargetRegionEntryInfo EntryInfo = E.second; + StringRef FnName = EntryInfo.ParentName; + if (!M.getNamedValue(FnName)) + continue; + ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo); + continue; + } + createOffloadEntry(IsTargetCodegen, CE->getID(), CE->getAddress(), + /*Size=*/0, CE->getFlags(), + GlobalValue::WeakAnyLinkage); + } else if (const auto *CE = dyn_cast< + OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>( + E.first)) { + OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags = + static_cast( + CE->getFlags()); + switch (Flags) { + case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo: { + if (IsEmbedded && HasRequiresUnifiedSharedMemory) + continue; + if (!CE->getAddress()) { + ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second); + continue; + } + // The vaiable has no definition - no need to add the entry. + if (CE->getVarSize() == 0) + continue; + break; + } + case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink: + assert(((IsEmbedded && !CE->getAddress()) || + (!IsEmbedded && CE->getAddress())) && + "Declaret target link address is set."); + if (IsEmbedded) + continue; + if (!CE->getAddress()) { + ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo()); + continue; + } + break; + } + + // Hidden or internal symbols on the device are not externally visible. + // We should not attempt to register them by creating an offloading + // entry. + if (auto *GV = dyn_cast(CE->getAddress())) + if (GV->hasLocalLinkage() || GV->hasHiddenVisibility()) + continue; + + createOffloadEntry(IsTargetCodegen, CE->getAddress(), CE->getAddress(), + CE->getVarSize(), Flags, CE->getLinkage()); + + } else { + llvm_unreachable("Unsupported entry kind."); + } + } +} + void TargetRegionEntryInfo::getTargetRegionEntryFnName( SmallVectorImpl &Name, StringRef ParentName, unsigned DeviceID, unsigned FileID, unsigned Line) {