[OpenMP] OpenMPOpt Support for Globalization Remarks

[jhuber6]

Summary

This patch add support for printing analysis messages relating to data globalization on the GPU. This occurs when data is shared between the threads in a GPU context and must be pushed to global or shared memory.

[jdoerfert]

Test?

[jhuber6]

In D88243#2293144, @jdoerfert wrote:

Test?

I tested it just using the example file to see if it works. I tried just calling opt on it but it was giving me some dumb error so I figured I'd just get it set up first.

clang++ -fopenmp clang/tests/OpenMP/declare_target_codegen_globalization.cpp -Rpass=openmp-opt -Rpass-analysis=openmp-opt -S -emit-llvm -fopenmp-targets=nvptx64-nvidia-cuda -O3

[jdoerfert]

llvm/lib/Transforms/IPO/OpenMPOpt.cpp

708

Can't we do foreachuse?

[jhuber6]

In D88243#2293147, @jhuber6 wrote:

In D88243#2293144, @jdoerfert wrote:

Test?

I tested it just using the example file to see if it works. I tried just calling opt on it but it was giving me some dumb error so I figured I'd just get it set up first.

clang++ -fopenmp clang/tests/OpenMP/declare_target_codegen_globalization.cpp -Rpass=openmp-opt -Rpass-analysis=openmp-opt -S -emit-llvm -fopenmp-targets=nvptx64-nvidia-cuda -O3

I think the problem is calling opt on the file since it's combined with the nvptx IR. Is there some opt command line option that does that correctly?

[jhuber6]

Adding test case and changing analysis to use forEachUse. The nvptx file doesn't have the line number information after compiling with debugging symbols so the remark just says "unknown." When you get the remarks from clang it seems to print the information more than necessary.

declare_target_codegen_globalization.cpp:7:5: remark: Found thread data sharing on the GPU. Expect degraded performance due to data globalization. [-Rpass-analysis=openmp-opt]
int bar() {
    ^
remark: Found thread data sharing on the GPU. Expect degraded performance due to data globalization. [-Rpass-analysis=openmp-opt]
remark: Found thread data sharing on the GPU. Expect degraded performance due to data globalization. [-Rpass-analysis=openmp-opt]

[jhuber6]

Manually adding missing debug info for stack pushing not being generated by Clang. A real solution will require modifying clang's code generation.

[jdoerfert]

LGTM

llvm/test/Transforms/OpenMP/globalization_remarks.ll

63 ↗

(On Diff #294151)

I think you want !31 but I guess it doesn't really matter.

llvm/include/llvm/Frontend/OpenMP/OMPKinds.def

 #define OMP_TYPE(VarName, InitValue)
 #endif
 
 #define __OMP_TYPE(VarName) OMP_TYPE(VarName, Type::get##VarName##Ty(Ctx))
 
 __OMP_TYPE(Void)
 __OMP_TYPE(Int1)
 __OMP_TYPE(Int8)
+__OMP_TYPE(Int16)
 __OMP_TYPE(Int32)
 __OMP_TYPE(Int64)
 __OMP_TYPE(Int8Ptr)
+__OMP_TYPE(Int16Ptr)
 __OMP_TYPE(Int32Ptr)
 __OMP_TYPE(Int64Ptr)
 
 OMP_TYPE(SizeTy, M.getDataLayout().getIntPtrType(Ctx))
 
 #define __OMP_PTR_TYPE(NAME, BASE) OMP_TYPE(NAME, BASE->getPointerTo())
 
 __OMP_PTR_TYPE(VoidPtr, Int8)

 __OMP_RTL(__tgt_target_data_update_nowait_mapper, false, Void, Int64, Int32,
           VoidPtrPtr, VoidPtrPtr, Int64Ptr, Int64Ptr, VoidPtrPtr)
 __OMP_RTL(__tgt_mapper_num_components, false, Int64, VoidPtr)
 __OMP_RTL(__tgt_push_mapper_component, false, Void, VoidPtr, VoidPtr, VoidPtr,
           Int64, Int64)
 __OMP_RTL(__kmpc_task_allow_completion_event, false, VoidPtr, IdentPtr,
           /* Int */ Int32, /* kmp_task_t */ VoidPtr)
 
+__OMP_RTL(__kmpc_data_sharing_init_stack, false, Void, )
+__OMP_RTL(__kmpc_data_sharing_init_stack_spmd, false, Void, ) 
+__OMP_RTL(__kmpc_data_sharing_coalesced_push_stack, false, VoidPtr, SizeTy,
+          Int16)
+__OMP_RTL(__kmpc_data_sharing_push_stack, false, VoidPtr, SizeTy, Int16)
+__OMP_RTL(__kmpc_data_sharing_pop_stack, false, Void, VoidPtr)
+
 /// Note that device runtime functions (in the following) do not necessarily
 /// need attributes as we expect to see the definitions.
 __OMP_RTL(__kmpc_kernel_parallel, false, Int1, VoidPtrPtr)
 __OMP_RTL(__kmpc_kernel_prepare_parallel, false, Void, VoidPtr)
 
 __OMP_RTL(__last, false, Void, )
 
 #undef __OMP_RTL

llvm/lib/Transforms/IPO/OpenMPOpt.cpp

using OptimizationRemarkGetter =
       function_ref<OptimizationRemarkEmitter &(Function *)>;
 
   OpenMPOpt(SmallVectorImpl<Function *> &SCC, CallGraphUpdater &CGUpdater,
             OptimizationRemarkGetter OREGetter,
             OMPInformationCache &OMPInfoCache, Attributor &A)
       : M(*(*SCC.begin())->getParent()), SCC(SCC), CGUpdater(CGUpdater),
         OREGetter(OREGetter), OMPInfoCache(OMPInfoCache), A(A) {}
 
+  /// Check if any remarks are enabled for openmp-opt
+  bool remarksEnabled() {
+    auto &Ctx = M.getContext();
+    return Ctx.getDiagHandlerPtr()->isAnyRemarkEnabled(DEBUG_TYPE);
+  }
+
   /// Run all OpenMP optimizations on the underlying SCC/ModuleSlice.
   bool run() {
     if (SCC.empty())
       return false;
 
     bool Changed = false;
 
     LLVM_DEBUG(dbgs() << TAG << "Run on SCC with " << SCC.size()
bool run() {
 
     // Recollect uses, in case Attributor deleted any.
     OMPInfoCache.recollectUses();
 
     Changed |= deduplicateRuntimeCalls();
     Changed |= deleteParallelRegions();
     if (HideMemoryTransferLatency)
       Changed |= hideMemTransfersLatency();
+    if (remarksEnabled())
+      analysisGlobalization();
 
     return Changed;
   }
 
   /// Print initial ICV values for testing.
   /// FIXME: This should be done from the Attributor once it is added.
   void printICVs() const {
     InternalControlVar ICVs[] = {ICV_nthreads, ICV_active_levels, ICV_cancel};
auto SplitMemTransfers = [&](Use &U, Function &Decl) {
       Changed |= WasSplit;
       return WasSplit;
     };
     RFI.foreachUse(SCC, SplitMemTransfers);
 
     return Changed;
   }
 
+  void analysisGlobalization() {
+    auto &RFI =
+      OMPInfoCache.RFIs[OMPRTL___kmpc_data_sharing_coalesced_push_stack];
+    for (Function *F : SCC) {
+      auto *UV = RFI.getUseVector(*F);
+      if (!UV)
+        return;
+      for (Use *U : *UV) {
+        if (CallInst *CI = getCallIfRegularCall(*U, &RFI)) {
+          auto Remark = [&](OptimizationRemarkAnalysis ORA) {
+            return ORA << "Found thread data sharing on the GPU. Expect "
+              << "degraded performance due to data globalization.";
+          };
+          emitRemark<OptimizationRemarkAnalysis>(CI, "OpenMPGlobalization",
+              Remark);
+        }
+      }
+    }
+    return;
+  }
+
   /// Maps the values stored in the offload arrays passed as arguments to
   /// \p RuntimeCall into the offload arrays in \p OAs.
   bool getValuesInOffloadArrays(CallInst &RuntimeCall,
                                 MutableArrayRef<OffloadArray> OAs) {
     assert(OAs.size() == 3 && "Need space for three offload arrays!");
 
     // A runtime call that involves memory offloading looks something like:
     // call void @__tgt_target_data_begin_mapper(arg0, arg1,