[WIP][test-suite] Add HIP RCCL project tests

[ashi1]

Summary

Add ROCm RCCL project compilation and tests to HIP builder.
Support building RCCL with multiple ROCm installations.

[ashi1]

This is a work-in-progress. RCCL requires using hipcc for the linking step, however the CMake file would not allow me to override the CMAKE_LINKER as hipcc. The reason to use hipcc is to handle static libraries created by -fgpu-rdc.

Therefore, this patch requires upstream clang to support consuming static libraries (containing bundles). Currently, you will see this error on the link step with clang++ (running ninja hip-tests-rccl):

[81/82] Linking CXX executable External/HIP/RcclMultiTests-hip-4.2.0
FAILED: External/HIP/RcclMultiTests-hip-4.2.0
: && /root/llvm-project/build/bin/clang++ -O3 -DNDEBUG  External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllGatherMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllReduceGroupMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllReduceMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllToAllMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_BroadcastMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_CombinedCallsMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_GatherMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_GroupCallsMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_ReduceMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_ReduceScatterMultiProcess.cpp.o External/HIP/CMakeFiles/RcclMultiTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_ScatterMultiProcess.cpp.o -o External/HIP/RcclMultiTests-hip-4.2.0  -Wl,-rpath,/root/Externals/hip/rocm-4.2.0/lib  -L/root/Externals/hip/rocm-4.2.0/lib -lamdhip64  External/HIP/libRcclLib-hip-4.2.0.a  -fgpu-rdc  -ldl  /root/Externals/hip/rocm-4.2.0/lib/libamdhip64.so  /usr/local/lib/libgtest.a  /usr/local/lib/libgtest_main.a  -lhsa-runtime64  -lrt  -lpthread  -lamd_comgr && :
External/HIP/libRcclLib-hip-4.2.0.a(sendrecv.cpp.o):(.hipFatBinSegment+0x8): undefined reference to `__hip_fatbin'
clang-11: error: linker command failed with exit code 1 (use -v to see invocation)
[82/82] Linking CXX executable External/HIP/RcclSingleTests-hip-4.2.0
FAILED: External/HIP/RcclSingleTests-hip-4.2.0
: && /root/llvm-project/build/bin/clang++ -O3 -DNDEBUG  External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllGather.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllReduce.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllReduceAbort.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllReduceGroup.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllToAll.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_AllToAllv.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_Broadcast.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_BroadcastAbort.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_CombinedCalls.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_Gather.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_GroupCalls.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_Reduce.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_ReduceScatter.cpp.o External/HIP/CMakeFiles/RcclSingleTests-hip-4.2.0.dir/root/Externals/hip/rccl/test/test_Scatter.cpp.o -o External/HIP/RcclSingleTests-hip-4.2.0  -Wl,-rpath,/root/Externals/hip/rocm-4.2.0/lib  -L/root/Externals/hip/rocm-4.2.0/lib -lamdhip64  External/HIP/libRcclLib-hip-4.2.0.a  -fgpu-rdc  -ldl  /root/Externals/hip/rocm-4.2.0/lib/libamdhip64.so  /usr/local/lib/libgtest.a  /usr/local/lib/libgtest_main.a  -lhsa-runtime64  -lrt  -lpthread  -lamd_comgr && :
External/HIP/libRcclLib-hip-4.2.0.a(sendrecv.cpp.o):(.hipFatBinSegment+0x8): undefined reference to `__hip_fatbin'
clang-11: error: linker command failed with exit code 1 (use -v to see invocation)
ninja: build stopped: subcommand failed.

[ashi1]

Added context.

External/HIP/CMakeLists.txt

macro(create_local_hip_tests VariantSuffix)
   list(APPEND HIP_LOCAL_TESTS saxpy)
   foreach(_hip_test IN LISTS HIP_LOCAL_TESTS)
     create_one_local_test(${_hip_test} ${_hip_test}.hip
                           ${VariantOffload} ${VariantSuffix}
                           "${VariantCPPFLAGS}" "${VariantLibs}")
   endforeach()
 endmacro()
 
+macro(create_rccl_test TestTarget TestSources)
+  llvm_test_run()
+  llvm_test_executable(${TestTarget} ${TestSources})
+  target_link_libraries(${TestTarget} RcclLib-${VariantSuffix} ${RCCL_LDFLAGS})
+  target_link_libraries(${TestTarget} ${VariantLibs} ${GTEST_BOTH_LIBRARIES} hsa-runtime64 rt pthread amd_comgr)
+  target_compile_options(${TestTarget} PUBLIC ${RCCL_CPPFLAGS} ${VariantCPPFLAGS})
+  target_include_directories(${TestTarget} PUBLIC ${GTEST_INCLUDE_DIRS})
+endmacro()
+
+macro(create_rccl_build VariantSuffix)
+  # FIXME: rccl link step requires using hipcc, because clang does not natively support
+  # linking with .a files.
+  #set(PREV_CMAKE_CXX_COMPILER ${CMAKE_CXX_COMPILER})
+  #set(CMAKE_CXX_COMPILER ${hipcc_executable})
+  #set(PREV_CMAKE_C_LINK_EXECUTABLE ${CMAKE_C_LINK_EXECUTABLE})
+  message(STATUS "CXX: ${CMAKE_CXX_COMPILER} LINK: ${CMAKE_C_LINK_EXECUTABLE}")
+
+  add_library(RcclLib-${VariantSuffix} ${RcclLibSources})
+  target_compile_options(RcclLib-${VariantSuffix} PRIVATE ${RCCL_CPPFLAGS} ${VariantCPPFLAGS})
+  target_link_libraries(RcclLib-${VariantSuffix} ${VariantLibs} ${RCCL_LDFLAGS})
+  add_dependencies(RcclLib-${VariantSuffix} timeit-target fpcmp-target)
+  if(NOT GTest_FOUND)
+    message(STATUS "Gtest not found, skipping RCCL test build.")
+    add_dependencies(hip-tests-rccl RcclLib-${VariantSuffix})
+  else()
+    set(_RcclMainTarget hip-tests-rccl-${VariantSuffix})
+    create_rccl_test(RcclSingleTests-${VariantSuffix} "${RcclTestSingleSources}")
+    create_rccl_test(RcclMultiTests-${VariantSuffix} "${RcclTestMultiSources}")
+    add_custom_target(${_RcclMainTarget}
+      DEPENDS RcclSingleTests-${VariantSuffix} RcclMultiTests-${VariantSuffix})
+    add_dependencies(hip-tests-rccl ${_RcclMainTarget})
+  endif()
+
+#  set(CMAKE_CXX_COMPILER ${PREV_CMAKE_CXX_COMPILER})
+#  set(CMAKE_CXX_LINK_EXECUTABLE ${PREV_CMAKE_CXX_LINK_EXECUTABLE})
+endmacro()
+
 function(create_hip_test VariantSuffix)
   message(STATUS "Creating HIP test variant ${VariantSuffix}")
   add_custom_target(hip-tests-simple-${VariantSuffix}
     COMMENT "Build HIP test variant ${VariantSuffix}")
 
+  # FIXME: RCCL only links with hipcc, and clang++ requires a patch upstream first.
+  #set(CMAKE_C_COMPILER ${hipcc_executable})
+  #set(CMAKE_CXX_COMPILER ${hipcc_executable})
+  #set(CMAKE_C_COMPILE_OBJECT ${hipcc_executable})
+  #set(CMAKE_CXX_COMPILE_OBJECT ${hipcc_executable})
+  #set(CMAKE_C_LINK_EXECUTABLE "/root/test-suite-build-dir/tools/timeit --summary <TARGET>.link.time <CMAKE_C_COMPILER> <FLAGS> <CMAKE_C_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+  #set(CMAKE_CXX_LINK_EXECUTABLE "/root/test-suite-build-dir/tools/timeit --summary <TARGET>.link.time <CMAKE_C_COMPILER> <FLAGS> <CMAKE_C_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+  #set(CMAKE_LINKER ${hipcc_executable})
+
   set(VariantCPPFLAGS ${_HIP_CPPFLAGS})
   set(VariantLibs ${_HIP_Libs})
   list(APPEND LDFLAGS ${_HIP_LDFLAGS})
 
   create_local_hip_tests(${VariantSuffix})
   add_dependencies(hip-tests-simple hip-tests-simple-${VariantSuffix})
 
+  if(DEFINED RCCL_PATH)
+    create_rccl_build(${VariantSuffix})
+  endif()
+
   add_custom_target(check-hip-simple-${VariantSuffix}
     COMMAND ${TEST_SUITE_LIT} ${TEST_SUITE_LIT_FLAGS}
             ${VARIANT_SIMPLE_TEST_TARGETS}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     DEPENDS hip-tests-simple-${VariantSuffix}
     USES_TERMINAL)
   add_dependencies(check-hip-simple check-hip-simple-${VariantSuffix})
 endfunction(create_hip_test)
add_custom_target(hip-tests-simple
     COMMENT "Build all simple HIP tests")
   add_custom_target(check-hip-simple
     COMMENT "Run all simple HIP tests")
 
   if(NOT AMDGPU_ARCHS)
     list(APPEND AMDGPU_ARCHS "gfx906;gfx908")
   endif()
 
+  # Add RCCL build targets if present in External/hip
+  if(EXISTS "${TEST_SUITE_HIP_ROOT}/rccl")
+    set(RCCL_PATH "${TEST_SUITE_HIP_ROOT}/rccl" CACHE PATH "RCCL Path")
+    message(STATUS "Found rccl ${RCCL_PATH}")
+    add_custom_target(hip-tests-rccl COMMENT "rccl build.")
+    set(RCCL_CPPFLAGS
+      -fvisibility=hidden
+      -fgpu-rdc
+      -Wno-format-nonliteral)
+#      -parallel-jobs=8)
+    set(RCCL_LDFLAGS
+      -fgpu-rdc
+#      -parallel-jobs=8
+      -ldl)
+    set(NCCL_MAJOR 2)
+    set(NCCL_MINOR 7)
+    set(NCCL_PATCH 0)
+    set(NCCL_VERSION "${NCCL_MAJOR}${NCCL_MINOR}0${NCCL_PATCH}")
+    add_definitions(-DENABLE_COLLTRACE)
+
+    # RCCL Library Source Files
+    configure_file(${RCCL_PATH}/src/nccl.h.in ${PROJECT_BINARY_DIR}/rccl.h)
+    configure_file(${RCCL_PATH}/src/nccl.h.in ${PROJECT_BINARY_DIR}/nccl.h)
+    include_directories(${PROJECT_BINARY_DIR})
+    include_directories(${RCCL_PATH}/src)
+    include_directories(${RCCL_PATH}/src/include)
+    include_directories(${RCCL_PATH}/src/collectives)
+    include_directories(${RCCL_PATH}/src/collectives/device)
+    gpu_glob(RcclCuSources ${RCCL_PATH}/src/collectives/device/*.cu)
+    set(RcclCppSources)
+    foreach(filename ${RcclCuSources})
+      string(REPLACE ".cu" ".cpp" rccl_cpp_filename ${filename})
+      configure_file(${filename} ${rccl_cpp_filename} COPYONLY)
+      list(APPEND RcclCppSources ${rccl_cpp_filename})
+    endforeach(filename)
+    gpu_glob(RcclCcSources
+      ${RCCL_PATH}/src/*.cc
+      ${RCCL_PATH}/src/graph/*.cc
+      ${RCCL_PATH}/src/collectives/*.cc
+      ${RCCL_PATH}/src/clique/*.cc
+      ${RCCL_PATH}/src/misc/*.cc
+      ${RCCL_PATH}/src/transport/*.cc)
+    list(REMOVE_ITEM RcclCcSources ${RCCL_PATH}/src/misc/gdrwrap.cc)
+    list(REMOVE_ITEM RcclCcSources ${RCCL_PATH}/src/misc/nvmlwrap.cc)
+    list(APPEND RcclLibSources ${RcclCppSources} ${RcclCcSources})
+
+    # RCCL Tests require GTest
+    find_package(GTest 1.10)
+    if(NOT GTest_FOUND)
+      message(STATUS "GTest not found. Skipping RCCL tests.")
+    else()
+      # RCCL Test Source Files
+      include_directories(${RCCL_PATH}/test)
+      include_directories(${GTEST_INCLUDE_DIR})
+      gpu_glob(RcclTestSingleSources ${RCCL_PATH}/test/*.cpp)
+      gpu_glob(RcclTestMultiSources ${RCCL_PATH}/test/*MultiProcess.cpp)
+      # Remove all multi process tests from single process tests.
+      foreach(File IN LISTS RcclTestMultiSources)
+        list(REMOVE_ITEM RcclTestSingleSources ${File})
+      endforeach()
+    endif()
+  endif()
+
   foreach(_RocmPath ${ROCM_PATHS})
     get_version(_RocmVersion ${_RocmPath})
     set(_HIP_Suffix "hip-${_RocmVersion}")
     # Set up HIP test flags
     set(_HIP_CPPFLAGS -xhip --hip-device-lib-path=${_RocmPath}/amdgcn/bitcode
                       -I${_RocmPath}/include)
     set(_HIP_LDFLAGS -L${_RocmPath}/lib -lamdhip64)
     set(_HIP_Libs amdhip64-${RocmVersion})
 
+    # FIXME: rccl needs to use hipcc for linking. remove once clang++ supports rccl.
+    #find_program(hipcc_executable hipcc PATHS ${_RocmPath}/hip)
+    #message(STATUS "Found hipcc: ${hipcc_executable}")
+
     # Unset these for each iteration of rocm path.
     set(_ArchFlags)
     set(_ArchList)
     foreach(_AMDGPUArch IN LISTS AMDGPU_ARCHS)
       list(APPEND _ArchFlags --offload-arch=${_AMDGPUArch})
     endforeach()
     message(STATUS "Building ${_RocmPath} targets for ${AMDGPU_ARCHS}")
     list(APPEND _HIP_CPPFLAGS ${_ArchFlags})
 
     create_hip_test(${_HIP_Suffix})
   endforeach()
 
-  add_custom_target(hip-tests-all DEPENDS hip-tests-simple
+  add_custom_target(hip-tests-all DEPENDS hip-tests-simple hip-tests-rccl
     COMMENT "Build all HIP tests.")
 
   file(COPY lit.local.cfg DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
 endmacro(create_hip_tests)
 
 if(TEST_SUITE_HIP_ROOT)
   create_hip_tests()
 endif()