[OpenMP] Change OpenMPOpt to check openmp metadata

[jhuber6]

Summary

The metadata added in D102361 introduces a module flag that we can check
to determine if the module was compiled with -fopenmp enables. We can
now check for the precense of this instead of scanning the call graph
for OpenMP runtime functions.

Depends on D102361

[jhuber6]

Updating.

[jdoerfert]

LG, one nit.

llvm/lib/Transforms/IPO/OpenMPOpt.cpp

2748–2749

We don't need this anymore. Remove the entire function.

[jhuber6]

Removing unused call.

llvm/include/llvm/Transforms/IPO/OpenMPOpt.h

 
 namespace llvm {
 
 namespace omp {
 
 /// Summary of a kernel (=entry point for target offloading).
 using Kernel = Function *;
 
-/// Helper to remember if the module contains OpenMP (runtime calls), to be used
-/// foremost with containsOpenMP.
-struct OpenMPInModule {
-  OpenMPInModule &operator=(bool Found) {
-    if (Found)
-      Value = OpenMPInModule::OpenMP::FOUND;
-    else
-      Value = OpenMPInModule::OpenMP::NOT_FOUND;
-    return *this;
-  }
-  bool isKnown() { return Value != OpenMP::UNKNOWN; }
-  operator bool() { return Value != OpenMP::NOT_FOUND; }
-
-  /// Does this function \p F contain any OpenMP runtime calls?
-  bool containsOMPRuntimeCalls(Function *F) const {
-    return FuncsWithOMPRuntimeCalls.contains(F);
-  }
-
-  /// Return the known kernels (=GPU entry points) in the module.
-  SmallPtrSetImpl<Kernel> &getKernels() { return Kernels; }
-
-  /// Identify kernels in the module and populate the Kernels set.
-  void identifyKernels(Module &M);
-
-private:
-  enum class OpenMP { FOUND, NOT_FOUND, UNKNOWN } Value = OpenMP::UNKNOWN;
+/// Set of kernels in the module
+using KernelSet = SmallPtrSet<Kernel, 4>;
 
-  friend bool containsOpenMP(Module &M, OpenMPInModule &OMPInModule);
+/// Helper to determine if \p M contains OpenMP.
+bool containsOpenMP(Module &M);
 
-  /// In which functions are OpenMP runtime calls present?
-  SmallPtrSet<Function *, 32> FuncsWithOMPRuntimeCalls;
+/// Helper to determine if \p M is a OpenMP target offloading device module.
+bool isOpenMPDevice(Module &M);
 
-  /// Collection of known kernels (=GPU entry points) in the module.
-  SmallPtrSet<Kernel, 8> Kernels;
-};
-
-/// Helper to determine if \p M contains OpenMP (runtime calls).
-bool containsOpenMP(Module &M, OpenMPInModule &OMPInModule);
+/// Get OpenMP device kernels in \p M.
+KernelSet getDeviceKernels(Module &M);
 
 } // namespace omp
 
 /// OpenMP optimizations pass.
 class OpenMPOptPass : public PassInfoMixin<OpenMPOptPass> {
-  /// Helper to remember if the module contains OpenMP (runtime calls).
-  omp::OpenMPInModule OMPInModule;
-
 public:
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 class OpenMPOptCGSCCPass : public PassInfoMixin<OpenMPOptCGSCCPass> {
-  /// Helper to remember if the module contains OpenMP (runtime calls).
-  omp::OpenMPInModule OMPInModule;
-
 public:
   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
                         LazyCallGraph &CG, CGSCCUpdateResult &UR);
 };
 
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_IPO_OPENMPOPT_H

llvm/lib/Transforms/IPO/OpenMPOpt.cpp

void registerAAs() {
     };
     GlobalizationRFI.foreachUse(SCC, CreateAA);
 
     // Create an ExecutionDomain AA for every function and a HeapToStack AA for
     // every function if there is a device kernel.
     for (auto *F : SCC) {
       if (!F->isDeclaration())
         A.getOrCreateAAFor<AAExecutionDomain>(IRPosition::function(*F));
-      if (!OMPInfoCache.Kernels.empty())
+      if (isOpenMPDevice(M))
         A.getOrCreateAAFor<AAHeapToStack>(IRPosition::function(*F));
     }
   }
 };
 
 Kernel OpenMPOpt::getUniqueKernelFor(Function &F) {
   if (!OMPInfoCache.ModuleSlice.count(&F))
     return nullptr;
case IRPosition::IRP_FUNCTION:
     AA = new (A.Allocator) AAHeapToSharedFunction(IRP, A);
     break;
   }
 
   return *AA;
 }
 
 PreservedAnalyses OpenMPOptPass::run(Module &M, ModuleAnalysisManager &AM) {
-  if (!containsOpenMP(M, OMPInModule))
+  if (!containsOpenMP(M))
     return PreservedAnalyses::all();
-
   if (DisableOpenMPOptimizations)
     return PreservedAnalyses::all();
 
+  KernelSet Kernels = getDeviceKernels(M);
+
   // Create internal copies of each function if this is a kernel Module.
   DenseSet<const Function *> InternalizedFuncs;
-  if (!OMPInModule.getKernels().empty())
+  if (isOpenMPDevice(M))
     for (Function &F : M)
-      if (!F.isDeclaration() && !OMPInModule.getKernels().contains(&F))
+      if (!F.isDeclaration() && !Kernels.contains(&F))
         if (Attributor::internalizeFunction(F, /* Force */ true))
           InternalizedFuncs.insert(&F);
 
   // Look at every function definition in the Module that wasn't internalized.
   SmallVector<Function *, 16> SCC;
   for (Function &F : M)
     if (!F.isDeclaration() && !InternalizedFuncs.contains(&F))
       SCC.push_back(&F);
PreservedAnalyses OpenMPOptPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto OREGetter = [&FAM](Function *F) -> OptimizationRemarkEmitter & {
     return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*F);
   };
 
   BumpPtrAllocator Allocator;
   CallGraphUpdater CGUpdater;
 
   SetVector<Function *> Functions(SCC.begin(), SCC.end());
-  OMPInformationCache InfoCache(M, AG, Allocator, /*CGSCC*/ Functions,
-                                OMPInModule.getKernels());
+  OMPInformationCache InfoCache(M, AG, Allocator, /*CGSCC*/ Functions, Kernels);
 
-  unsigned MaxFixponitIterations = (!OMPInModule.getKernels().empty()) ? 64 : 32;
+  unsigned MaxFixponitIterations = (Kernels.empty()) ? 64 : 32;
   Attributor A(Functions, InfoCache, CGUpdater, nullptr, true, false, MaxFixponitIterations, OREGetter,
                DEBUG_TYPE);
 
   OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
   bool Changed = OMPOpt.run(true);
   if (Changed)
     return PreservedAnalyses::none();
 
   return PreservedAnalyses::all();
 }
 
 PreservedAnalyses OpenMPOptCGSCCPass::run(LazyCallGraph::SCC &C,
                                           CGSCCAnalysisManager &AM,
                                           LazyCallGraph &CG,
                                           CGSCCUpdateResult &UR) {
-  if (!containsOpenMP(*C.begin()->getFunction().getParent(), OMPInModule))
+  if (!containsOpenMP(*C.begin()->getFunction().getParent()))
     return PreservedAnalyses::all();
-
   if (DisableOpenMPOptimizations)
     return PreservedAnalyses::all();
 
   SmallVector<Function *, 16> SCC;
   // If there are kernels in the module, we have to run on all SCC's.
-  bool SCCIsInteresting = !OMPInModule.getKernels().empty();
   for (LazyCallGraph::Node &N : C) {
     Function *Fn = &N.getFunction();
     SCC.push_back(Fn);
-
-    // Do we already know that the SCC contains kernels,
-    // or that OpenMP functions are called from this SCC?
-    if (SCCIsInteresting)
-      continue;
-    // If not, let's check that.
-    SCCIsInteresting |= OMPInModule.containsOMPRuntimeCalls(Fn);
   }
 
-  if (!SCCIsInteresting || SCC.empty())
+  if (SCC.empty())
     return PreservedAnalyses::all();
 
+  Module &M = *C.begin()->getFunction().getParent();
+
+  KernelSet Kernels = getDeviceKernels(M);
+
   FunctionAnalysisManager &FAM =
       AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
 
   AnalysisGetter AG(FAM);
 
   auto OREGetter = [&FAM](Function *F) -> OptimizationRemarkEmitter & {
     return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*F);
   };
 
   BumpPtrAllocator Allocator;
   CallGraphUpdater CGUpdater;
   CGUpdater.initialize(CG, C, AM, UR);
 
   SetVector<Function *> Functions(SCC.begin(), SCC.end());
   OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG, Allocator,
-                                /*CGSCC*/ Functions, OMPInModule.getKernels());
+                                /*CGSCC*/ Functions, Kernels);
 
-  unsigned MaxFixponitIterations = (!OMPInModule.getKernels().empty()) ? 64 : 32;
+  unsigned MaxFixponitIterations = (isOpenMPDevice(M)) ? 64 : 32;
   Attributor A(Functions, InfoCache, CGUpdater, nullptr, false, true, MaxFixponitIterations, OREGetter,
                DEBUG_TYPE);
 
   OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
   bool Changed = OMPOpt.run(false);
   if (Changed)
     return PreservedAnalyses::none();
 
   return PreservedAnalyses::all();
 }
 
 namespace {
 
 struct OpenMPOptCGSCCLegacyPass : public CallGraphSCCPass {
   CallGraphUpdater CGUpdater;
-  OpenMPInModule OMPInModule;
   static char ID;
 
   OpenMPOptCGSCCLegacyPass() : CallGraphSCCPass(ID) {
     initializeOpenMPOptCGSCCLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     CallGraphSCCPass::getAnalysisUsage(AU);
   }
 
-  bool doInitialization(CallGraph &CG) override {
-    // Disable the pass if there is no OpenMP (runtime call) in the module.
-    containsOpenMP(CG.getModule(), OMPInModule);
-    return false;
-  }
-
   bool runOnSCC(CallGraphSCC &CGSCC) override {
-    if (!containsOpenMP(CGSCC.getCallGraph().getModule(), OMPInModule))
+    if (!containsOpenMP(CGSCC.getCallGraph().getModule()))
       return false;
     if (DisableOpenMPOptimizations || skipSCC(CGSCC))
       return false;
 
     SmallVector<Function *, 16> SCC;
     // If there are kernels in the module, we have to run on all SCC's.
-    bool SCCIsInteresting = !OMPInModule.getKernels().empty();
     for (CallGraphNode *CGN : CGSCC) {
       Function *Fn = CGN->getFunction();
       if (!Fn || Fn->isDeclaration())
         continue;
       SCC.push_back(Fn);
-
-      // Do we already know that the SCC contains kernels,
-      // or that OpenMP functions are called from this SCC?
-      if (SCCIsInteresting)
-        continue;
-      // If not, let's check that.
-      SCCIsInteresting |= OMPInModule.containsOMPRuntimeCalls(Fn);
     }
 
-    if (!SCCIsInteresting || SCC.empty())
+    if (SCC.empty())
       return false;
 
+    Module &M = CGSCC.getCallGraph().getModule();
+    KernelSet Kernels = getDeviceKernels(M);
+
     CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
     CGUpdater.initialize(CG, CGSCC);
 
     // Maintain a map of functions to avoid rebuilding the ORE
     DenseMap<Function *, std::unique_ptr<OptimizationRemarkEmitter>> OREMap;
     auto OREGetter = [&OREMap](Function *F) -> OptimizationRemarkEmitter & {
       std::unique_ptr<OptimizationRemarkEmitter> &ORE = OREMap[F];
       if (!ORE)
         ORE = std::make_unique<OptimizationRemarkEmitter>(F);
       return *ORE;
     };
 
     AnalysisGetter AG;
     SetVector<Function *> Functions(SCC.begin(), SCC.end());
     BumpPtrAllocator Allocator;
-    OMPInformationCache InfoCache(
-        *(Functions.back()->getParent()), AG, Allocator,
-        /*CGSCC*/ Functions, OMPInModule.getKernels());
+    OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG,
+                                  Allocator,
+                                  /*CGSCC*/ Functions, Kernels);
 
-    unsigned MaxFixponitIterations = (!OMPInModule.getKernels().empty()) ? 64 : 32;
+    unsigned MaxFixponitIterations = (isOpenMPDevice(M)) ? 64 : 32;
     Attributor A(Functions, InfoCache, CGUpdater, nullptr, false, true,
                  MaxFixponitIterations, OREGetter, DEBUG_TYPE);
 
     OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
     return OMPOpt.run(false);
   }
 
   bool doFinalization(CallGraph &CG) override { return CGUpdater.finalize(); }
 };
 
 } // end anonymous namespace
 
-void OpenMPInModule::identifyKernels(Module &M) {
-
+KernelSet llvm::omp::getDeviceKernels(Module &M) {
+  // TODO: Create a more cross-platform way of determining device kernels.
   NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
+  KernelSet Kernels;
+
   if (!MD)
-    return;
+    return Kernels;
 
   for (auto *Op : MD->operands()) {
     if (Op->getNumOperands() < 2)
       continue;
     MDString *KindID = dyn_cast<MDString>(Op->getOperand(1));
     if (!KindID || KindID->getString() != "kernel")
       continue;
 
     Function *KernelFn =
         mdconst::dyn_extract_or_null<Function>(Op->getOperand(0));
     if (!KernelFn)
       continue;
 
     ++NumOpenMPTargetRegionKernels;
 
     Kernels.insert(KernelFn);
   }
-}
 
-bool llvm::omp::containsOpenMP(Module &M, OpenMPInModule &OMPInModule) {
-  if (OMPInModule.isKnown())
-    return OMPInModule;
-
-  auto RecordFunctionsContainingUsesOf = [&](Function *F) {
-    for (User *U : F->users())
-      if (auto *I = dyn_cast<Instruction>(U))
-        OMPInModule.FuncsWithOMPRuntimeCalls.insert(I->getFunction());
-  };
-
-  // MSVC doesn't like long if-else chains for some reason and instead just
-  // issues an error. Work around it..
-  do {
-#define OMP_RTL(_Enum, _Name, ...)                                             \
-  if (Function *F = M.getFunction(_Name)) {                                    \
-    RecordFunctionsContainingUsesOf(F);                                        \
-    OMPInModule = true;                                                        \
-  }
-#include "llvm/Frontend/OpenMP/OMPKinds.def"
-  } while (false);
+  return Kernels;
 
-  // Identify kernels once. TODO: We should split the OMPInformationCache into a
-  // module and an SCC part. The kernel information, among other things, could
-  // go into the module part.
-  if (OMPInModule.isKnown() && OMPInModule) {
-    OMPInModule.identifyKernels(M);
-    return true;
-  }
+bool llvm::omp::containsOpenMP(Module &M) {
+  Metadata *MD = M.getModuleFlag("openmp");
+  if (!MD)
+    return false;
+
 
-  return OMPInModule = false;
+bool llvm::omp::isOpenMPDevice(Module &M) {
+  Metadata *MD = M.getModuleFlag("openmp-device");
+  if (!MD)
+    return false;
+
+  return true;
 }
 
 char OpenMPOptCGSCCLegacyPass::ID = 0;
 
 INITIALIZE_PASS_BEGIN(OpenMPOptCGSCCLegacyPass, "openmp-opt-cgscc",
                       "OpenMP specific optimizations", false, false)
 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
 INITIALIZE_PASS_END(OpenMPOptCGSCCLegacyPass, "openmp-opt-cgscc",
                     "OpenMP specific optimizations", false, false)
 
 Pass *llvm::createOpenMPOptCGSCCLegacyPass() {
   return new OpenMPOptCGSCCLegacyPass();
 }

llvm/test/Transforms/OpenMP/add_attributes.ll

 ; OPTIMISTIC-NEXT: declare noalias i8* @__kmpc_task_reduction_modifier_init(i8*, i32, i32, i32, i8*)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
 ; OPTIMISTIC-NEXT: declare void @__kmpc_proxy_task_completed_ooo(i8*)
 
 ; OPTIMISTIC: ; Function Attrs: convergent noinline nounwind 
 ; OPTIMISTIC-NEXT: declare void @__kmpc_barrier_simple_spmd(%struct.ident_t* nocapture nofree readonly, i32)
 
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/add_attributes_amdgcn.ll

 ; CHECK: ; Function Attrs: convergent nounwind
 ; CHECK-NEXT: declare void @__kmpc_syncwarp(i64)
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
 ; OPTIMISTIC-NEXT: declare i64 @__kmpc_warp_active_thread_mask()
 
 ; OPTIMISTIC: ; Function Attrs: convergent nounwind
 ; OPTIMISTIC-NEXT: declare void @__kmpc_syncwarp(i64)
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/deduplication.ll

entry:
   %tid3 = call i32 @omp_get_level()
   %tid4 = call i32 @omp_get_level()
   %tid5 = call i32 @omp_get_level()
   call void @useI32(i32 %tid3)
   call void @useI32(i32 %tid4)
   call void @useI32(i32 %tid5)
   ret void
 }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/deduplication_remarks.ll

 
 declare i32 @__kmpc_global_thread_num(%struct.ident_t*)
 
 declare !dbg !4 void @useI32(i32) local_unnamed_addr
 
 declare void @llvm.dbg.value(metadata, metadata, metadata)
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!8, !9, !10, !11, !12}
+!llvm.module.flags = !{!8, !9, !10, !11, !12, !29}
 !llvm.ident = !{!13}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "deduplication_remarks.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4}
 !4 = !DISubprogram(name: "useI32", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
 !5 = !DISubroutineType(types: !6)

 !21 = !DILocation(line: 5, column: 10, scope: !14)
 !22 = !DILocation(line: 0, scope: !14)
 !23 = !DILocation(line: 6, column: 2, scope: !14)
 !24 = !DILocation(line: 7, column: 10, scope: !14)
 !25 = !DILocation(line: 8, column: 2, scope: !14)
 !26 = !DILocation(line: 9, column: 10, scope: !14)
 !27 = !DILocation(line: 10, column: 2, scope: !14)
 !28 = !DILocation(line: 13, column: 1, scope: !14)
+!29 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/globalization_remarks.ll

 ; RUN: opt -passes=openmp-opt -pass-remarks-missed=openmp-opt -disable-output < %s 2>&1 | FileCheck %s
 ; ModuleID = 'declare_target_codegen_globalization.cpp'
 source_filename = "declare_target_codegen_globalization.cpp"
 target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
 target triple = "nvptx64"
 
+; CHECK: remark: globalization_remarks.c:5:7: Could not move globalized variable to the stack. Variable is potentially captured.
 ; CHECK: remark: globalization_remarks.c:5:7: Found thread data sharing on the GPU. Expect degraded performance due to data globalization.
 
 @S = external local_unnamed_addr global i8*
 
 define void @foo() {
 entry:
-  %0 = call i8* @__kmpc_alloc_shared(i64 4), !dbg !8
+  %0 = call i8* @__kmpc_alloc_shared(i64 4), !dbg !10
   %x_on_stack = bitcast i8* %0 to i32*
   %1 = bitcast i32* %x_on_stack to i8*
   call void @share(i8* %1)
   call void @__kmpc_free_shared(i8* %0)
   ret void
 }
 
 define void @share(i8* %x) {
 entry:
   store i8* %x, i8** @S
   ret void
 }
 
 declare i8* @__kmpc_alloc_shared(i64)
 
 declare void @__kmpc_free_shared(i8*)
 
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!nvvm.annotations = !{!7, !8}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "globalization_remarks.c", directory: "/tmp/globalization_remarks.c")
 !2 = !{}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !{i32 1, !"wchar_size", i32 4}
-!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !7, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!7 = !DISubroutineType(types: !2)
-!8 = !DILocation(line: 5, column: 7, scope: !6)
+!5 = !{i32 7, !"openmp", i32 50}
+!6 = !{i32 7, !"openmp-device", i32 50}
+!7 = !{void ()* @foo, !"kernel", i32 1}
+!8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !9, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!9 = !DISubroutineType(types: !2)
+!10 = !DILocation(line: 5, column: 7, scope: !8)

llvm/test/Transforms/OpenMP/gpu_kernel_detection_remarks.ll

 
 define void @non_kernel() {
   ret void
 }
 
 ; Needed to trigger the openmp-opt pass
 declare dso_local void @__kmpc_kernel_prepare_parallel(i8*)
 
+!llvm.module.flags = !{!4}
 !nvvm.annotations = !{!2, !0, !1, !3, !1, !2}
 
 !0 = !{void ()* @kernel1, !"kernel", i32 1}
 !1 = !{void ()* @non_kernel, !"non_kernel", i32 1}
 !2 = !{null, !"align", i32 1}
 !3 = !{void ()* @kernel2, !"kernel", i32 1}
+!4 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll

 declare void @__kmpc_barrier_simple_spmd(%struct.ident_t*, i32) 
 
 declare i1 @__kmpc_kernel_parallel(i8**)
 
 declare void @__kmpc_kernel_end_parallel()
 
 
 !nvvm.annotations = !{!1}
+!llvm.module.flags = !{!2, !3}
 
 !1 = !{void ()* @__omp_offloading_50_6dfa0f01_foo_l6, !"kernel", i32 1}
+!2 = !{i32 7, !"openmp", i32 50}
+!3 = !{i32 7, !"openmp-device", i32 50}

llvm/test/Transforms/OpenMP/hide_mem_transfer_latency.ll

 declare void @__tgt_target_data_begin_mapper(%struct.ident_t*, i64, i32, i8**, i8**, i64*, i64*, i8**, i8**)
 declare i32 @__tgt_target_teams_mapper(%struct.ident_t*, i64, i8*, i32, i8**, i8**, i64*, i64*, i8**, i8**, i32, i32)
 declare void @__tgt_target_data_end_mapper(%struct.ident_t*, i64, i32, i8**, i8**, i64*, i64*, i8**, i8**)
 
 declare dso_local i32 @rand(...)
 
 ; CHECK: declare void @__tgt_target_data_begin_mapper_issue(%struct.ident_t*, i64, i32, i8**, i8**, i64*, i64*, i8**, i8**, %struct.__tgt_async_info*)
 ; CHECK: declare void @__tgt_target_data_begin_mapper_wait(i64, %struct.__tgt_async_info*)
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/icv_remarks.ll

 attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
 attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #3 = { norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #4 = { argmemonly nounwind willreturn }
 attributes #5 = { nounwind readnone speculatable willreturn }
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!13, !14, !15}
+!llvm.module.flags = !{!13, !14, !15, !59}
 !llvm.ident = !{!16}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git 73cea83a6f5ab521edf3cccfc603534776d691ec)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "icv_remarks.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4, !9, !12}
 !4 = !DISubprogram(name: "omp_set_num_threads", scope: !5, file: !5, line: 57, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
 !5 = !DIFile(filename: "/usr/local/lib/clang/11.0.0/include/omp.h", directory: "")

 !51 = distinct !DILocation(line: 18, column: 1, scope: !33)
 !52 = !DILocation(line: 20, column: 5, scope: !49, inlinedAt: !51)
 !53 = !DILocation(line: 21, column: 16, scope: !49, inlinedAt: !51)
 !54 = !DILocation(line: 0, scope: !49, inlinedAt: !51)
 !55 = !DILocation(line: 22, column: 5, scope: !49, inlinedAt: !51)
 !56 = !DILocation(line: 18, column: 1, scope: !33)
 !57 = !{!58}
 !58 = !{i64 2, i64 -1, i64 -1, i1 true}
+!59 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/icv_tracking.ll

 16:                                               ; preds = %7, %13, %11, %3
   %17 = call i32 @omp_get_max_threads()
   %18 = call i32 @icv_free_use(i32 %17)
   ret i32 %18
 }
 
 declare i32 @__gxx_personality_v0(...)
 
+!llvm.module.flags = !{!2}
+
 !0 = !{!1}
 !1 = !{i64 2, i64 -1, i64 -1, i1 true}
+!2 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/parallel_deletion.ll

 declare !callback !2 void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...)
 
 declare void @unknown()
 
 declare void @readonly() readonly
 
 declare void @readnone() readnone
 
+!llvm.module.flags = !{!8}
+
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{!"clang"}
 !2 = !{!3}
 !3 = !{i64 2, i64 -1, i64 -1, i1 true}
 !4 = !{!5, !5, i64 0}
 !5 = !{!"int", !6, i64 0}
 !6 = !{!"omnipotent char", !7, i64 0}
 !7 = !{!"Simple C/C++ TBAA"}
+!8 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/parallel_deletion_remarks.ll

define internal void @.omp_outlined..6(i32* noalias nocapture %0, i32* noalias nocapture %1) !dbg !47 {
   ret void, !dbg !51
 }
 
 attributes #0 = { willreturn }
 attributes #1 = { readonly willreturn }
 attributes #2 = { readnone willreturn }
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!9, !10, !11, !12, !13}
+!llvm.module.flags = !{!9, !10, !11, !12, !13, !52}
 !llvm.ident = !{!14}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "parallel_deletion_remarks.c", directory: "/tmp")
 !2 = !{}
 !3 = !{!4, !7, !8}
 !4 = !DISubprogram(name: "unknown_willreturn", scope: !1, file: !1, line: 3, type: !5, spFlags: DISPFlagOptimized, retainedNodes: !2)
 !5 = !DISubroutineType(types: !6)

 !44 = !DILocalVariable(name: ".global_tid.", arg: 1, scope: !42, type: !28, flags: DIFlagArtificial)
 !45 = !DILocalVariable(name: ".bound_tid.", arg: 2, scope: !42, type: !28, flags: DIFlagArtificial)
 !46 = !DILocation(line: 13, column: 2, scope: !42)
 !47 = distinct !DISubprogram(name: ".omp_outlined..6", scope: !1, file: !1, line: 15, type: !26, scopeLine: 15, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !48)
 !48 = !{!49, !50}
 !49 = !DILocalVariable(name: ".global_tid.", arg: 1, scope: !47, type: !28, flags: DIFlagArtificial)
 !50 = !DILocalVariable(name: ".bound_tid.", arg: 2, scope: !47, type: !28, flags: DIFlagArtificial)
 !51 = !DILocation(line: 15, column: 2, scope: !47)
+!52 = !{i32 7, !"openmp", i32 50}

llvm/test/Transforms/OpenMP/parallel_region_merging.ll

 define internal void @.omp_outlined..39(i32* noalias nocapture readnone %.global_tid., i32* noalias nocapture readnone %.bound_tid., i32* nocapture nonnull readonly align 4 dereferenceable(4) %a)  {
 entry:
   %0 = load i32, i32* %a, align 4
   call void @use(i32 %0)
   ret void
 }
 
 
-!llvm.module.flags = !{!0}
+!llvm.module.flags = !{!0, !3}
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{!2}
 !2 = !{i64 2, i64 -1, i64 -1, i1 true}
+!3 = !{i32 7, !"openmp", i32 50}
 ; CHECK-LABEL: define {{[^@]+}}@merge
 ; CHECK-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 [[A]], i32* [[A_ADDR]], align 4
 ; CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @[[GLOB1:[0-9]+]])
 ; CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
 ; CHECK:       omp_parallel:

 ;
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..39
 ; CHECK2-SAME: (i32* noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], i32* noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], i32* nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A]], align 4
 ; CHECK2-NEXT:    call void @use(i32 [[TMP0]])
 ; CHECK2-NEXT:    ret void
 ;
+

llvm/test/Transforms/OpenMP/remove_globalization.ll

 define internal void @foo() {
 ; CHECK-LABEL: define {{[^@]+}}@foo
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i8, i64 4, align 1
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  %0 = call i8* @__kmpc_alloc_shared(i64 4), !dbg !9
+  %0 = call i8* @__kmpc_alloc_shared(i64 4), !dbg !11
   call void @use(i8* %0)
   call void @__kmpc_free_shared(i8* %0)
   ret void
 }
 
 define internal void @bar() {
 ; CHECK-LABEL: define {{[^@]+}}@bar
 ; CHECK-SAME: () #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i8* @__kmpc_alloc_shared(i64 noundef 4) #[[ATTR0]]
 ; CHECK-NEXT:    call void @share(i8* nofree writeonly [[TMP0]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    call void @__kmpc_free_shared(i8* [[TMP0]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  %0 = call i8* @__kmpc_alloc_shared(i64 4), !dbg !10
+  %0 = call i8* @__kmpc_alloc_shared(i64 4), !dbg !12
   call void @share(i8* %0)
   call void @__kmpc_free_shared(i8* %0)
   ret void
 }
 
 define internal void @use(i8* %x) {
 entry:
   ret void

 
 ; CHECK: declare i8* @__kmpc_alloc_shared(i64)
 declare i8* @__kmpc_alloc_shared(i64)
 
 ; CHECK: declare void @__kmpc_free_shared(i8* nocapture)
 declare void @__kmpc_free_shared(i8*)
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4}
+!llvm.module.flags = !{!3, !4, !6, !7}
 !nvvm.annotations = !{!5}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 13.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "remove_globalization.c", directory: "/tmp/remove_globalization.c")
 !2 = !{}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !{i32 1, !"wchar_size", i32 4}
 !5 = !{void ()* @kernel, !"kernel", i32 1}
-!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!7 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!8 = !DISubroutineType(types: !2)
-!9 = !DILocation(line: 2, column: 2, scope: !6)
-!10 = !DILocation(line: 4, column: 2, scope: !7)
+!6 = !{i32 7, !"openmp", i32 50}
+!7 = !{i32 7, !"openmp-device", i32 50}
+!8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!9 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!10 = !DISubroutineType(types: !2)
+!11 = !DILocation(line: 2, column: 2, scope: !8)
+!12 = !DILocation(line: 4, column: 2, scope: !9)

llvm/test/Transforms/OpenMP/replace_globalization.ll

 
 ; CHECK: %{{.*}} = bitcast i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([16 x i8], [16 x i8] addrspace(3)* [[SHARED_X]], i32 0, i32 0) to i8*) to [4 x i32]*
 define internal void @baz() {
 entry:
   %tid = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
   %cmp = icmp eq i32 %tid, 0
   br i1 %cmp, label %master, label %exit
 master:
-  %x = call i8* @__kmpc_alloc_shared(i64 16), !dbg !9
+  %x = call i8* @__kmpc_alloc_shared(i64 16), !dbg !11
   %x_on_stack = bitcast i8* %x to [4 x i32]*
   %0 = bitcast [4 x i32]* %x_on_stack to i8*
   call void @use(i8* %0)
   call void @__kmpc_free_shared(i8* %x)
   br label %exit
 exit:
   ret void
 }
 
 ; CHECK: %{{.*}} = bitcast i8* addrspacecast (i8 addrspace(3)* getelementptr inbounds ([4 x i8], [4 x i8] addrspace(3)* [[SHARED_Y]], i32 0, i32 0) to i8*) to [4 x i32]*
 define internal void @qux() {
 entry:
   %tid = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
   %ntid = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
   %warpsize = call i32 @llvm.nvvm.read.ptx.sreg.warpsize()
   %0 = sub nuw i32 %warpsize, 1
   %1 = sub nuw i32 %ntid, 1
   %2 = xor i32 %0, -1
   %master_tid = and i32 %1, %2
   %3 = icmp eq i32 %tid, %master_tid
   br i1 %3, label %master, label %exit
 master:
-  %y = call i8* @__kmpc_alloc_shared(i64 4), !dbg !10
+  %y = call i8* @__kmpc_alloc_shared(i64 4), !dbg !12
   %y_on_stack = bitcast i8* %y to [4 x i32]*
   %4 = bitcast [4 x i32]* %y_on_stack to i8*
   call void @use(i8* %4)
   call void @__kmpc_free_shared(i8* %y)
   br label %exit
 exit:
   ret void
 }

 declare i32 @llvm.nvvm.read.ptx.sreg.tid.x()
 
 declare i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
 
 declare i32 @llvm.nvvm.read.ptx.sreg.warpsize()
 
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4}
-!nvvm.annotations = !{!5, !6}
-
+!llvm.module.flags = !{!3, !4, !5, !6}
+!nvvm.annotations = !{!7, !8}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "replace_globalization.c", directory: "/tmp/replace_globalization.c")
 !2 = !{}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{void ()* @foo, !"kernel", i32 1}
-!6 = !{void ()* @bar, !"kernel", i32 1}
-!7 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!8 = !DISubroutineType(types: !2)
-!9 = !DILocation(line: 5, column: 7, scope: !7)
-!10 = !DILocation(line: 5, column: 14, scope: !7)
+!5 = !{i32 7, !"openmp", i32 50}
+!6 = !{i32 7, !"openmp-device", i32 50}
+!7 = !{void ()* @foo, !"kernel", i32 1}
+!8 = !{void ()* @bar, !"kernel", i32 1}
+!9 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!10 = !DISubroutineType(types: !2)
+!11 = !DILocation(line: 5, column: 7, scope: !9)
+!12 = !DILocation(line: 5, column: 14, scope: !9)

llvm/test/Transforms/OpenMP/rtf_type_checking.ll

 declare void @__kmpc_barrier(%struct.ident_t*, float)
 
 ; Proper use of runtime function.
 declare void @__kmpc_flush(%struct.ident_t*)
 
 ; Different return type.
 declare void @omp_get_thread_num()
 
-!llvm.module.flags = !{!0}
+!llvm.module.flags = !{!0, !4}
 !llvm.ident = !{!1}
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{!"clang"}
 !2 = !{!3}
 !3 = !{i64 2, i64 -1, i64 -1, i1 true}
+!4 = !{i32 7, !"openmp", i32 50}
 
 ; NPM: Running pass: OpenMPOptCGSCCPass on (.omp_outlined.)
 ; NPM-NOT: Running pass: OpenMPOptCGSCCPass on (.omp_outlined.)
 ; NPM: Running pass: OpenMPOptCGSCCPass on (main)
 ; NPM-NOT: Running pass: OpenMPOptCGSCCPass on (main)
 ; ===-------------------------------------------------------------------------===
 ;                         ... Statistics Collected ...
 ; ===-------------------------------------------------------------------------===
 ;
 ; LPM: 1 cgscc-passmgr - Maximum CGSCCPassMgr iterations on one SCC
 ; CHECK: 2 openmp-opt{{.*}}Number of OpenMP runtime functions identified
 ;
 ; There are two matches since the pass is run once per function.

llvm/test/Transforms/OpenMP/single_threaded_execution.ll

 
 declare i32 @llvm.nvvm.read.ptx.sreg.tid.x()
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
 declare void @__kmpc_kernel_init(i32, i16)
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4}
-!nvvm.annotations = !{!5}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!nvvm.annotations = !{!7}
 
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "single_threaded_execution.c", directory: "/tmp/single_threaded_execution.c")
 !2 = !{}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{void ()* @kernel, !"kernel", i32 1}
+!5 = !{i32 7, !"openmp", i32 50}
+!6 = !{i32 7, !"openmp-device", i32 50}
+!7 = !{void ()* @kernel, !"kernel", i32 1}

llvm/test/Transforms/OpenMP/values_in_offload_arrays.ll

entry:
   call void @__tgt_target_data_end_mapper(%struct.ident_t* @0, i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes., i64 0, i64 0), i8** null, i8** null)
   ret i32 %rem
 }
 
 declare void @__tgt_target_data_begin_mapper(%struct.ident_t*, i64, i32, i8**, i8**, i64*, i64*, i8**, i8**)
 declare void @__tgt_target_data_end_mapper(%struct.ident_t*, i64, i32, i8**, i8**, i64*, i64*, i8**, i8**)
 
 declare dso_local i32 @rand(...)
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 7, !"openmp", i32 50}
+

llvm/test/Transforms/PhaseOrdering/openmp-opt-module.ll

 ; RUN: opt -O2 -pass-remarks-missed=openmp-opt -enable-new-pm < %s 2>&1 | FileCheck %s --check-prefix=MODULE
 target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
 
 @.str = private unnamed_addr constant [13 x i8] c"Alloc Shared\00", align 1
 
 @S = external local_unnamed_addr global i8*
 
 ; MODULE: remark: openmp_opt_module.c:5:7: Found thread data sharing on the GPU. Expect degraded performance due to data globalization.
 
 define void @foo() {
 entry:
-  %x = call i8* @__kmpc_alloc_shared(i64 4), !dbg !7
+  %x = call i8* @__kmpc_alloc_shared(i64 4), !dbg !10
   %x_on_stack = bitcast i8* %x to i32*
   %0 = bitcast i32* %x_on_stack to i8*
   call void @use(i8* %0)
   call void @__kmpc_free_shared(i8* %x)
   ret void
 }
 
 define void @use(i8* %0) {

 }
 
 ; Function Attrs: convergent nounwind mustprogress
 declare i8* @_Z10SafeMallocmPKc(i64 %size, i8* nocapture readnone %msg)
 
 declare void @__kmpc_free_shared(i8*)
 
 !llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!nvvm.annotations = !{!7}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "openmp_opt_module.c", directory: "/tmp/openmp_opt_module.c")
 !2 = !{}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !{i32 1, !"wchar_size", i32 4}
-!5 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!6 = !DISubroutineType(types: !2)
-!7 = !DILocation(line: 5, column: 7, scope: !5)
+!5 = !{i32 7, !"openmp", i32 50}
+!6 = !{i32 7, !"openmp-device", i32 50}
+!7 = !{void ()* @foo, !"kernel", i32 1}
+!8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !9, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!9 = !DISubroutineType(types: !2)
+!10 = !DILocation(line: 5, column: 7, scope: !8)