[libomptarget] [amdgpu] Ensure OMP_TEAMS_THREAD_LIMIT is not overwritten by the default workgroup size

[JonChesterfield]

This stuff would also benefit from unit testing.

Generalising slightly, this looks a lot like a discrete optimisation problem. Find values for launch parameters subject to N constraints that we change slowly over time. I think there are two outputs - the values written to the HSA packet - and a lot of inputs.

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

 #include <assert.h>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <elf.h>
 #include <fstream>
 #include <functional>
 #include <iostream>
 #include <libelf.h>
 #include <list>
 #include <memory>
 #include <mutex>
 #include <shared_mutex>
 #include <thread>
 #include <unordered_map>
 #include <vector>
 
int32_t __tgt_rtl_init_device(int device_id) {
   }
 
   if (enforce_upper_bound(&DeviceInfo.NumTeams[device_id],
                           DeviceInfo.GroupsPerDevice[device_id])) {
     DP("Default number of teams exceeds device limit, capping at %d\n",
        DeviceInfo.GroupsPerDevice[device_id]);
   }
 
-  // Adjust threads to the env variables
-  if (DeviceInfo.EnvTeamThreadLimit > 0 &&
-      (enforce_upper_bound(&DeviceInfo.NumThreads[device_id],
-                           DeviceInfo.EnvTeamThreadLimit))) {
-    DP("Capping max number of threads to OMP_TEAMS_THREAD_LIMIT=%d\n",
+  if (DeviceInfo.EnvTeamThreadLimit > 0) {
+    DeviceInfo.NumThreads[device_id] = DeviceInfo.EnvTeamThreadLimit;
+    DP("Number of threads set according to OMP_TEAMS_THREAD_LIMIT=%d\n",
        DeviceInfo.EnvTeamThreadLimit);
-  }
-
-  // Set default number of threads
+  } else {
     DeviceInfo.NumThreads[device_id] = RTLDeviceInfoTy::Default_WG_Size;
-  DP("Default number of threads set according to library's default %d\n",
+    DP("Number of threads set according to library's default %d\n",
        RTLDeviceInfoTy::Default_WG_Size);
+  }
+
   if (enforce_upper_bound(&DeviceInfo.NumThreads[device_id],
                           DeviceInfo.ThreadsPerGroup[device_id])) {
     DP("Default number of threads exceeds device limit, capping at %d\n",
        DeviceInfo.ThreadsPerGroup[device_id]);
   }
 
   DP("Device %d: default limit for groupsPerDevice %d & threadsPerGroup %d\n",
      device_id, DeviceInfo.GroupsPerDevice[device_id],
int32_t __tgt_rtl_run_target_team_region_locked(
 
   assert(arg_num == (int)KernelInfoEntry.num_args);
 
   /*
    * Set limit based on ThreadsPerGroup and GroupsPerDevice
    */
   int num_groups = 0;
 
-  int threadsPerGroup = RTLDeviceInfoTy::Default_WG_Size;
+  int threadsPerGroup = DeviceInfo.NumThreads[KernelInfo->device_id];
 
   getLaunchVals(threadsPerGroup, num_groups, KernelInfo->ConstWGSize,
                 KernelInfo->ExecutionMode, DeviceInfo.EnvTeamLimit,
                 DeviceInfo.EnvNumTeams,
                 num_teams,      // From run_region arg
                 thread_limit,   // From run_region arg
                 loop_tripcount, // From run_region arg
                 KernelInfo->device_id);