diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -212,6 +212,7 @@ LANGOPT(OpenMPUseTLS , 1, 0, "Use TLS for threadprivates or runtime calls") LANGOPT(OpenMPIsDevice , 1, 0, "Generate code only for OpenMP target device") LANGOPT(OpenMPCUDAMode , 1, 0, "Generate code for OpenMP pragmas in SIMT/SPMD mode") +LANGOPT(OpenMPNewCodegen , 1, 0, "Use the experimental OpenMP-IR-Builder codegen path.") LANGOPT(OpenMPCUDAForceFullRuntime , 1, 0, "Force to use full runtime in all constructs when offloading to CUDA devices") LANGOPT(OpenMPCUDANumSMs , 32, 0, "Number of SMs for CUDA devices.") LANGOPT(OpenMPCUDABlocksPerSM , 32, 0, "Number of blocks per SM for CUDA devices.") diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1640,6 +1640,8 @@ Group, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>; def fopenmp_simd : Flag<["-"], "fopenmp-simd">, Group, Flags<[CC1Option, NoArgumentUnused]>, HelpText<"Emit OpenMP code only for SIMD-based constructs.">; +def fopenmp_new_codegen : Flag<["-"], "fopenmp-new-codegen">, Group, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>, + HelpText<"Use the experimental OpenMP-IR-Builder codegen path.">; def fno_openmp_simd : Flag<["-"], "fno-openmp-simd">, Group, Flags<[CC1Option, NoArgumentUnused]>; def fopenmp_cuda_mode : Flag<["-"], "fopenmp-cuda-mode">, Group, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>; diff --git a/llvm/include/llvm/IR/OpenMPConstants.h b/llvm/include/llvm/IR/OpenMPConstants.h --- a/llvm/include/llvm/IR/OpenMPConstants.h +++ b/llvm/include/llvm/IR/OpenMPConstants.h @@ -14,11 +14,13 @@ #ifndef LLVM_OPENMP_CONSTANTS_H #define LLVM_OPENMP_CONSTANTS_H +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/StringRef.h" namespace llvm { namespace omp { +LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); /// IDs for all OpenMP directives. enum class Directive { @@ -33,6 +35,26 @@ #define OMP_DIRECTIVE(Enum, ...) constexpr auto Enum = omp::Directive::Enum; #include "llvm/IR/OpenMPKinds.def" +/// IDs for all omp runtime library (RTL) functions. +enum class RuntimeFunction { +#define OMP_RTL(Enum, ...) Enum, +#include "llvm/IR/OpenMPKinds.def" +}; + +#define OMP_RTL(Enum, ...) constexpr auto Enum = omp::RuntimeFunction::Enum; +#include "llvm/IR/OpenMPKinds.def" + +/// IDs for all omp runtime library ident_t flag encodings (see +/// their defintion in openmp/runtime/src/kmp.h). +enum class IdentFlag { +#define OMP_IDENT_FLAG(Enum, Str, Value) Enum = Value, +#include "llvm/IR/OpenMPKinds.def" + LLVM_MARK_AS_BITMASK_ENUM(~0) +}; + +#define OMP_IDENT_FLAG(Enum, ...) constexpr auto Enum = omp::IdentFlag::Enum; +#include "llvm/IR/OpenMPKinds.def" + /// Parse \p Str and return the directive it matches or OMPD_unknown if none. Directive getOpenMPDirectiveKind(StringRef Str); diff --git a/llvm/include/llvm/IR/OpenMPIRBuilder.h b/llvm/include/llvm/IR/OpenMPIRBuilder.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/IR/OpenMPIRBuilder.h @@ -0,0 +1,122 @@ +//===- IR/OpenMPIRBuilder.h - OpenMP encoding builder for LLVM IR - C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the OpenMPIRBuilder class and helpers used as a convenient +// way to create LLVM instructions for OpenMP directives. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OPENMP_IR_IRBUILDER_H +#define LLVM_OPENMP_IR_IRBUILDER_H + +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/OpenMPConstants.h" + +namespace llvm { + +/// An interface to create LLVM-IR for OpenMP directives. +/// +/// Each OpenMP directive has a corresponding public generator method. +class OpenMPIRBuilder { +public: + /// Create a new OpenMPIRBuilder operating on the given module \p M. This will + /// not have an effect on \p M (see initialize). + OpenMPIRBuilder(Module &M) : M(M), Builder(M.getContext()) {} + + /// Initialize the internal state, this will put structures types and + /// potentially other helpers into the underlying module. Must be called + /// before any other method and only once! + void initialize(); + + /// Description of a LLVM-IR insertion point (IP) and (later) a source + /// location (filename, line, column, ...). + struct LocationDescription { + IRBuilder<>::InsertPoint IP; + }; + + /// Emitter methods for OpenMP directives. + /// + ///{ + + /// Generator for '#omp barrier' + /// + /// \param Loc The location where the barrier directive was encountered. + /// \param DK The kind of directive that caused the barrier. + /// \param CheckCancelFlag Flag to indicate a cancel barrier return value + /// should be checked and acted upon. + void createBarrier(const LocationDescription &Loc, omp::Directive DK, + bool CheckCancelFlag = true); + + ///} + +private: + /// Return the function declaration for the runtime function with \p FnID. + Function *getOrCreateRuntimeFunction(omp::RuntimeFunction FnID); + + /// Return the (LLVM-IR) string describing the source location \p LocStr. + Constant *getOrCreateSrcLocStr(StringRef LocStr); + + /// Return the (LLVM-IR) string describing the default source location. + Constant *getOrCreateDefaultSrcLocStr(); + + /// Return the (LLVM-IR) string describing the source location \p Loc. + Constant *getOrCreateSrcLocStr(const LocationDescription &Loc); + + /// Return an ident_t* encoding the source location \p SrcLocStr and \p Flags. + Value *getOrCreateIdent(Constant *SrcLocStr, + omp::IdentFlag Flags = omp::IdentFlag(0)); + + /// Generate a barrier runtime call. + /// + /// \param Loc The location at which the request originated and is fulfilled. + /// \param DK The directive which caused the barrier + /// \param CheckCancelFlag Flag to indicate a cancel barrier return value + /// should be checked and acted upon. + /// \param ForceSimpleCall Flag to force a simple (=non-cancelation) barrier + void emitBarrierImpl(const LocationDescription &Loc, omp::Directive DK, + bool CheckCancelFlag, bool ForceSimpleCall); + + /// Return the current thread ID. + /// + /// \param Ident The ident (ident_t*) describing the query origin. + Value *getOrCreateThreadID(Value *Ident); + + /// Declarations for LLVM-IR types (simple and structure) are generated below. + /// Their names are defined and used in OpenMPKinds.def. Here we provide the + /// declarations, the initialize method will provide the values. + /// + ///{ + +#define OMP_TYPE(VarName, InitValue) Type *VarName = nullptr; +#define OMP_STRUCT_TYPE(VarName, StrName, ...) Type *VarName = nullptr; +#include "llvm/IR/OpenMPKinds.def" + + ///} + + /// The underlying LLVM-IR module + Module &M; + + /// The LLVM-IR Builder used to create IR. + IRBuilder<> Builder; + + /// TODO: Stub for a cancelation block stack. + BasicBlock *CancelationBlock = nullptr; + + /// Map to remember the thread in a function. + DenseMap ThreadIDMap; + + /// Map to remember source location strings + StringMap SrcLocStrMap; + + /// Map to remember existing ident_t*. + DenseMap, GlobalVariable *> IdentMap; +}; + +} // end namespace llvm + +#endif // LLVM_IR_IRBUILDER_H diff --git a/llvm/include/llvm/IR/OpenMPKinds.def b/llvm/include/llvm/IR/OpenMPKinds.def --- a/llvm/include/llvm/IR/OpenMPKinds.def +++ b/llvm/include/llvm/IR/OpenMPKinds.def @@ -99,3 +99,108 @@ #undef OMP_DIRECTIVE ///} + +/// Types used in runtime structs or runtime functions +/// +///{ + +#ifndef OMP_TYPE +#define OMP_TYPE(VarName, InitValue) +#endif + +#define __OMP_TYPE(VarName) OMP_TYPE(VarName, Type::get##VarName##Ty(Ctx)) + +__OMP_TYPE(Void) +__OMP_TYPE(Int8) +__OMP_TYPE(Int32) +__OMP_TYPE(Int8Ptr) + +#undef __OMP_TYPE +#undef OMP_TYPE + +///} + +/// Struct types +/// +///{ + +#ifndef OMP_STRUCT_TYPE +#define OMP_STRUCT_TYPE(VarName, StructName, ...) +#endif + +#define __OMP_STRUCT_TYPE(VarName, Name, ...) \ + OMP_STRUCT_TYPE(VarName, "struct." #Name, __VA_ARGS__) + +__OMP_STRUCT_TYPE(IdentPtr, ident_t, Int32, Int32, Int32, Int32, Int8Ptr) + +#undef __OMP_STRUCT_TYPE +#undef OMP_STRUCT_TYPE + +///} + +/// Runtime library function (and their attributes) +/// +///{ + +#ifndef OMP_RTL +#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) +#endif + +#define __OMP_RTL(Name, IsVarArg, ReturnType, ...) \ + OMP_RTL(OMPRTL_##Name, #Name, IsVarArg, ReturnType, __VA_ARGS__) + +__OMP_RTL(__kmpc_barrier, false, Void, IdentPtr, Int32) +__OMP_RTL(__kmpc_cancel_barrier, false, Int32, IdentPtr, Int32) +__OMP_RTL(__kmpc_global_thread_num, false, Int32, IdentPtr) + +#undef __OMP_RTL +#undef OMP_RTL + +#ifndef OMP_RTL_ATTRS +#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) +#endif + +#define EnumAttr(Kind) Attribute::get(Ctx, Attribute::AttrKind::Kind) +#define AttributeSet(...) \ + AttributeSet::get(Ctx, ArrayRef({__VA_ARGS__})) + +#define __OMP_RTL_ATTRS(Name, FnAttrSet, RetAttrSet, ArgAttrSets) \ + OMP_RTL_ATTRS(OMPRTL_##Name, FnAttrSet, RetAttrSet, ArgAttrSets) + +__OMP_RTL_ATTRS(__kmpc_global_thread_num, + AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)), + AttributeSet(), {}) + +#undef __OMP_RTL_ATTRS +#undef OMP_RTL_ATTRS +#undef AttributeSet +#undef EnumAttr + +///} + +/// KMP ident_t bit flags +/// +/// In accordance with the values in `openmp/runtime/src/kmp.h`. +/// +///{ + +#ifndef OMP_IDENT_FLAG +#define OMP_IDENT_FLAG(Enum, Str, Value) +#endif + +#define __OMP_IDENT_FLAG(Name, Value) \ + OMP_IDENT_FLAG(OMP_IDENT_FLAG_##Name, #Name, Value) + +__OMP_IDENT_FLAG(KMPC, 0x02) +__OMP_IDENT_FLAG(BARRIER_EXPL, 0x20) +__OMP_IDENT_FLAG(BARRIER_IMPL, 0x0040) +__OMP_IDENT_FLAG(BARRIER_IMPL_MASK, 0x01C0) +__OMP_IDENT_FLAG(BARRIER_IMPL_FOR, 0x0040) +__OMP_IDENT_FLAG(BARRIER_IMPL_SECTIONS, 0x00C0) +__OMP_IDENT_FLAG(BARRIER_IMPL_SINGLE, 0x0140) +__OMP_IDENT_FLAG(BARRIER_IMPL_WORKSHARE, 0x01C0) + +#undef __OMP_IDENT_FLAG +#undef OMP_IDENT_FLAG + +///} diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt --- a/llvm/lib/IR/CMakeLists.txt +++ b/llvm/lib/IR/CMakeLists.txt @@ -39,6 +39,7 @@ Metadata.cpp Module.cpp ModuleSummaryIndex.cpp + OpenMPIRBuilder.cpp Operator.cpp OptBisect.cpp Pass.cpp diff --git a/llvm/lib/IR/OpenMPIRBuilder.cpp b/llvm/lib/IR/OpenMPIRBuilder.cpp --- a/llvm/lib/IR/OpenMPIRBuilder.cpp +++ b/llvm/lib/IR/OpenMPIRBuilder.cpp @@ -5,13 +5,20 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file +/// +/// This file implements the OpenMPIRBuilder class, which is used as a +/// convenient way to create LLVM instructions for OpenMP directives. +/// //===----------------------------------------------------------------------===// -#include "llvm/IR/OpenMPConstants.h" +#include "llvm/IR/OpenMPIRBuilder.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/IR/IRBuilder.h" + +#define DEBUG_TYPE "openmp-ir-builder" using namespace llvm; using namespace omp; @@ -32,3 +39,206 @@ } llvm_unreachable("Invalid OpenMP directive kind"); } + +Function *OpenMPIRBuilder::getOrCreateRuntimeFunction(RuntimeFunction FnID) { + Function *Fn = nullptr; + + // Try to find the declation in the module first. + switch (FnID) { +#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \ + case Enum: \ + Fn = M.getFunction(Str); \ + break; +#include "llvm/IR/OpenMPKinds.def" + } + + if (!Fn) { + + // Create a new declaration if we need one. + switch (FnID) { +#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \ + case Enum: \ + Fn = Function::Create(FunctionType::get(ReturnType, \ + ArrayRef{__VA_ARGS__}, \ + IsVarArg), \ + GlobalValue::ExternalLinkage, Str, M); \ + break; +#include "llvm/IR/OpenMPKinds.def" + } + + assert(Fn && "Failed to create OpenMP runtime function"); + + LLVMContext &Ctx = Fn->getContext(); + // Add attributes to the new declaration. + switch (FnID) { +#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \ + case Enum: \ + Fn->setAttributes( \ + AttributeList::get(Ctx, FnAttrSet, RetAttrSet, ArgAttrSets)); \ + break; +#include "llvm/IR/OpenMPKinds.def" + default: + // Attributes are optional. + break; + } + } + + return Fn; +} + +void OpenMPIRBuilder::initialize() { + LLVMContext &Ctx = M.getContext(); + + // Create all simple and struct types exposed by the runtime and remember the + // llvm::PointerTypes of them for easy access later. + Type *T; +#define OMP_TYPE(VarName, InitValue) this->VarName = InitValue; +#define OMP_STRUCT_TYPE(VarName, StructName, ...) \ + T = M.getTypeByName(StructName); \ + if (!T) \ + T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \ + this->VarName = PointerType::getUnqual(T); +#include "llvm/IR/OpenMPKinds.def" +} + +Value *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr, + IdentFlag LocFlags) { + // Enable "C-mode". + LocFlags |= OMP_IDENT_FLAG_KMPC; + + GlobalVariable *&DefaultIdent = IdentMap[{SrcLocStr, uint64_t(LocFlags)}]; + if (!DefaultIdent) { + Constant *I32Null = ConstantInt::getNullValue(Int32); + Constant *IdentData[] = {I32Null, + ConstantInt::get(Int32, uint64_t(LocFlags)), + I32Null, I32Null, SrcLocStr}; + Constant *Initializer = ConstantStruct::get( + cast(IdentPtr->getPointerElementType()), IdentData); + + // Look for existing encoding of the location + flags, not needed but + // minimizes the difference to the existing solution while we transition. + for (GlobalVariable &GV : M.getGlobalList()) + if (GV.getType() == IdentPtr && GV.hasInitializer()) + if (GV.getInitializer() == Initializer) + return DefaultIdent = &GV; + + DefaultIdent = new GlobalVariable(M, IdentPtr->getPointerElementType(), + /* isConstant = */ false, + GlobalValue::PrivateLinkage, Initializer); + DefaultIdent->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + DefaultIdent->setAlignment(Align(8)); + } + return DefaultIdent; +} + +Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr) { + Constant *&SrcLocStr = SrcLocStrMap[LocStr]; + if (!SrcLocStr) { + Constant *Initializer = + ConstantDataArray::getString(M.getContext(), LocStr); + + // Look for existing encoding of the location, not needed but minimizes the + // difference to the existing solution while we transition. + for (GlobalVariable &GV : M.getGlobalList()) + if (GV.isConstant() && GV.hasInitializer() && + GV.getInitializer() == Initializer) + return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr); + + SrcLocStr = Builder.CreateGlobalStringPtr(LocStr); + } + return SrcLocStr; +} + +Constant *OpenMPIRBuilder::getOrCreateDefaultSrcLocStr() { + return getOrCreateSrcLocStr(";unknown;unknown;0;0;;"); +} + +Constant * +OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc) { + // TODO: Support actual source locations. + return getOrCreateDefaultSrcLocStr(); +} + +Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) { + // TODO: It makes only so much sense to actually cache the global_thread_num + // calls in the front-end as we can do a better job later on. Once + // the middle-end combines global_thread_num calls (user calls and + // generated ones!) we can rethink having a caching scheme here. + Function *Fn = Builder.GetInsertBlock()->getParent(); + Value *&TID = ThreadIDMap[Fn]; + if (!TID) { + // Search the entry block, not needed once all thread id calls go through + // here and are cached in the OpenMPIRBuilder. + for (Instruction &I : Fn->getEntryBlock()) + if (CallInst *CI = dyn_cast(&I)) + if (CI->getCalledFunction() && + CI->getCalledFunction()->getName() == "__kmpc_global_thread_num") + return TID = CI; + + Function *FnDecl = + getOrCreateRuntimeFunction(OMPRTL___kmpc_global_thread_num); + Instruction *Call = + Builder.CreateCall(FnDecl, Ident, "omp_global_thread_num"); + if (Instruction *IdentI = dyn_cast(Ident)) + Call->moveAfter(IdentI); + else + Call->moveBefore(&*Fn->getEntryBlock().getFirstInsertionPt()); + TID = Call; + } + return TID; +} + +void OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, + Directive DK, bool CheckCancelFlag) { + // TODO: Do we really expect these create calls to happen at an invalid + // location and if so is ignoring them the right thing to do? This + // mimics Clang's behavior for now. + if (!Loc.IP.getBlock()) + return; + Builder.restoreIP(Loc.IP); + emitBarrierImpl(Loc, DK, CheckCancelFlag, + /* ForceSimpleCall = */ false); +} + +void OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, + Directive Kind, bool CheckCancelFlag, + bool ForceSimpleCall) { + // Build call __kmpc_cancel_barrier(loc, thread_id) or + // __kmpc_barrier(loc, thread_id); + + IdentFlag BarrierLocFlags; + switch (Kind) { + case OMPD_for: + BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR; + break; + case OMPD_sections: + BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS; + break; + case OMPD_single: + BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE; + break; + case OMPD_barrier: + BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL; + break; + default: + BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL; + break; + } + + // Set new insertion point for the internal builder. + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); + Value *Args[] = {getOrCreateIdent(SrcLocStr, BarrierLocFlags), + getOrCreateThreadID(getOrCreateIdent(SrcLocStr))}; + bool UseCancelBarrier = !ForceSimpleCall && CancelationBlock; + Value *Result = Builder.CreateCall( + getOrCreateRuntimeFunction(UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier + : OMPRTL___kmpc_barrier), + Args); + + if (UseCancelBarrier && CheckCancelFlag) { + Value *Cmp = Builder.CreateIsNotNull(Result); + // TODO Reimplement part of llvm::SplitBlockAndInsertIfThen in a helper and + // use it here. + (void)Cmp; + } +}