[libc] Add an option to use a job pool for GPU tests

[jhuber6]

Summary

Currently the GPU has restrictions on how many tests can be run in
parallel due to resource constraints. However, building these tests can
take a long time so we want to be able to build them in parallel. This
patch introduces the option LIBC_GPU_TEST_JOBS which is set to the
number of threads to run in parallel.

[tra]

LGTM.

Just a FYI -- in the past, running too many tests in parallel caused some tests to hang. I've empirically settled on - j4 on cuda bots. Things may have improved in recent CUDA and driver versions, but I didn't push parallel tests further.

[jhuber6]

In D153157#4428981, @tra wrote:

LGTM.

Just a FYI -- in the past, running too many tests in parallel caused some tests to hang. I've empirically settled on - j4 on cuda bots. Things may have improved in recent CUDA and driver versions, but I didn't push parallel tests further.

I've only had CUDA hang on me once in about 500 runs of the test suite due to parallelism, the AMD however stack fails about every other run. This should hopefully allow us to build in parallel, ever since I enabled the string conversion functions the test times have become more unreasonable. It may be worse on AMDGPU specifically because we run LTO on everything, but the strtof test alone takes about 44 seconds to build and run on my machine.

[jplehr]

FWICT this look good to me.
Do I see that right that in the AMDGPU buildbot, we want to set LIBC_GPU_TEST_JOBS=1?

libc/cmake/modules/LLVMLibCTestRules.cmake

function(add_libc_hermetic_test test_name)
   set(test_cmd ${HERMETIC_TEST_ENV}
       $<$<BOOL:${LIBC_TARGET_ARCHITECTURE_IS_GPU}>:${gpu_loader_exe}> ${HERMETIC_TEST_LOADER_ARGS}
       $<TARGET_FILE:${fq_build_target_name}> ${HERMETIC_TEST_ARGS})
   add_custom_target(
     ${fq_target_name}
     COMMAND ${test_cmd}
     COMMAND_EXPAND_LISTS
     COMMENT "Running hermetic test ${fq_target_name}"
+    ${LIBC_HERMETIC_TEST_JOB_POOL}
   )
 
   add_dependencies(${HERMETIC_TEST_SUITE} ${fq_target_name})
   add_dependencies(libc-hermetic-tests ${fq_target_name})
 endfunction(add_libc_hermetic_test)
 
 # A convenience function to add both a unit test as well as a hermetic test.
 function(add_libc_test test_name)

libc/cmake/modules/prepare_libc_gpu_build.cmake

 find_program(LIBC_CLANG_OFFLOAD_PACKAGER
              NAMES clang-offload-packager
              PATHS ${LLVM_BINARY_DIR}/bin)
 if(NOT LIBC_CLANG_OFFLOAD_PACKAGER)
   message(FATAL_ERROR "Cannot find the 'clang-offload-packager' for the GPU "
                       "build")
 endif()
 
+# Optionally set up a job pool to limit the number of GPU tests run in parallel.
+# This is sometimes necessary as running too many tests in parallel can cause
+# the GPU or driver to run out of resources.
+set(LIBC_GPU_TEST_JOBS "" CACHE STRING "Number of jobs to run in parallel for "
+                                       "GPU tests")
+if(LIBC_GPU_TEST_JOBS)
+  set_property(GLOBAL PROPERTY JOB_POOLS LIBC_GPU_TEST_POOL=${LIBC_GPU_TEST_JOBS})
+  set(LIBC_HERMETIC_TEST_JOB_POOL JOB_POOL LIBC_GPU_TEST_POOL)
+endif()
+
 set(LIBC_GPU_TEST_ARCHITECTURE "" CACHE STRING "Architecture for the GPU tests")
 
 set(gpu_test_architecture "")
 if(LIBC_GPU_TEST_ARCHITECTURE)
   set(gpu_test_architecture ${LIBC_GPU_TEST_ARCHITECTURE})
   message(STATUS "Using user-specified GPU architecture for testing: "
                  "'${gpu_test_architecture}'")
 elseif(detected_gpu_architectures)