Index: External/HIP/CMakeLists.txt =================================================================== --- External/HIP/CMakeLists.txt +++ External/HIP/CMakeLists.txt @@ -15,11 +15,57 @@ endforeach() endmacro() +macro(create_rccl_test TestTarget TestSources) + llvm_test_run() + llvm_test_executable(${TestTarget} ${TestSources}) + target_link_libraries(${TestTarget} RcclLib-${VariantSuffix} ${RCCL_LDFLAGS}) + target_link_libraries(${TestTarget} ${VariantLibs} ${GTEST_BOTH_LIBRARIES} hsa-runtime64 rt pthread amd_comgr) + target_compile_options(${TestTarget} PUBLIC ${RCCL_CPPFLAGS} ${VariantCPPFLAGS}) + target_include_directories(${TestTarget} PUBLIC ${GTEST_INCLUDE_DIRS}) +endmacro() + +macro(create_rccl_build VariantSuffix) + # FIXME: rccl link step requires using hipcc, because clang does not natively support + # linking with .a files. + #set(PREV_CMAKE_CXX_COMPILER ${CMAKE_CXX_COMPILER}) + #set(CMAKE_CXX_COMPILER ${hipcc_executable}) + #set(PREV_CMAKE_C_LINK_EXECUTABLE ${CMAKE_C_LINK_EXECUTABLE}) + message(STATUS "CXX: ${CMAKE_CXX_COMPILER} LINK: ${CMAKE_C_LINK_EXECUTABLE}") + + add_library(RcclLib-${VariantSuffix} ${RcclLibSources}) + target_compile_options(RcclLib-${VariantSuffix} PRIVATE ${RCCL_CPPFLAGS} ${VariantCPPFLAGS}) + target_link_libraries(RcclLib-${VariantSuffix} ${VariantLibs} ${RCCL_LDFLAGS}) + add_dependencies(RcclLib-${VariantSuffix} timeit-target fpcmp-target) + if(NOT GTest_FOUND) + message(STATUS "Gtest not found, skipping RCCL test build.") + add_dependencies(hip-tests-rccl RcclLib-${VariantSuffix}) + else() + set(_RcclMainTarget hip-tests-rccl-${VariantSuffix}) + create_rccl_test(RcclSingleTests-${VariantSuffix} "${RcclTestSingleSources}") + create_rccl_test(RcclMultiTests-${VariantSuffix} "${RcclTestMultiSources}") + add_custom_target(${_RcclMainTarget} + DEPENDS RcclSingleTests-${VariantSuffix} RcclMultiTests-${VariantSuffix}) + add_dependencies(hip-tests-rccl ${_RcclMainTarget}) + endif() + +# set(CMAKE_CXX_COMPILER ${PREV_CMAKE_CXX_COMPILER}) +# set(CMAKE_CXX_LINK_EXECUTABLE ${PREV_CMAKE_CXX_LINK_EXECUTABLE}) +endmacro() + function(create_hip_test VariantSuffix) message(STATUS "Creating HIP test variant ${VariantSuffix}") add_custom_target(hip-tests-simple-${VariantSuffix} COMMENT "Build HIP test variant ${VariantSuffix}") + # FIXME: RCCL only links with hipcc, and clang++ requires a patch upstream first. + #set(CMAKE_C_COMPILER ${hipcc_executable}) + #set(CMAKE_CXX_COMPILER ${hipcc_executable}) + #set(CMAKE_C_COMPILE_OBJECT ${hipcc_executable}) + #set(CMAKE_CXX_COMPILE_OBJECT ${hipcc_executable}) + #set(CMAKE_C_LINK_EXECUTABLE "/root/test-suite-build-dir/tools/timeit --summary .link.time -o ") + #set(CMAKE_CXX_LINK_EXECUTABLE "/root/test-suite-build-dir/tools/timeit --summary .link.time -o ") + #set(CMAKE_LINKER ${hipcc_executable}) + set(VariantCPPFLAGS ${_HIP_CPPFLAGS}) set(VariantLibs ${_HIP_Libs}) list(APPEND LDFLAGS ${_HIP_LDFLAGS}) @@ -27,6 +73,10 @@ create_local_hip_tests(${VariantSuffix}) add_dependencies(hip-tests-simple hip-tests-simple-${VariantSuffix}) + if(DEFINED RCCL_PATH) + create_rccl_build(${VariantSuffix}) + endif() + add_custom_target(check-hip-simple-${VariantSuffix} COMMAND ${TEST_SUITE_LIT} ${TEST_SUITE_LIT_FLAGS} ${VARIANT_SIMPLE_TEST_TARGETS} @@ -66,6 +116,69 @@ list(APPEND AMDGPU_ARCHS "gfx906;gfx908") endif() + # Add RCCL build targets if present in External/hip + if(EXISTS "${TEST_SUITE_HIP_ROOT}/rccl") + set(RCCL_PATH "${TEST_SUITE_HIP_ROOT}/rccl" CACHE PATH "RCCL Path") + message(STATUS "Found rccl ${RCCL_PATH}") + add_custom_target(hip-tests-rccl COMMENT "rccl build.") + set(RCCL_CPPFLAGS + -fvisibility=hidden + -fgpu-rdc + -Wno-format-nonliteral) +# -parallel-jobs=8) + set(RCCL_LDFLAGS + -fgpu-rdc +# -parallel-jobs=8 + -ldl) + set(NCCL_MAJOR 2) + set(NCCL_MINOR 7) + set(NCCL_PATCH 0) + set(NCCL_VERSION "${NCCL_MAJOR}${NCCL_MINOR}0${NCCL_PATCH}") + add_definitions(-DENABLE_COLLTRACE) + + # RCCL Library Source Files + configure_file(${RCCL_PATH}/src/nccl.h.in ${PROJECT_BINARY_DIR}/rccl.h) + configure_file(${RCCL_PATH}/src/nccl.h.in ${PROJECT_BINARY_DIR}/nccl.h) + include_directories(${PROJECT_BINARY_DIR}) + include_directories(${RCCL_PATH}/src) + include_directories(${RCCL_PATH}/src/include) + include_directories(${RCCL_PATH}/src/collectives) + include_directories(${RCCL_PATH}/src/collectives/device) + gpu_glob(RcclCuSources ${RCCL_PATH}/src/collectives/device/*.cu) + set(RcclCppSources) + foreach(filename ${RcclCuSources}) + string(REPLACE ".cu" ".cpp" rccl_cpp_filename ${filename}) + configure_file(${filename} ${rccl_cpp_filename} COPYONLY) + list(APPEND RcclCppSources ${rccl_cpp_filename}) + endforeach(filename) + gpu_glob(RcclCcSources + ${RCCL_PATH}/src/*.cc + ${RCCL_PATH}/src/graph/*.cc + ${RCCL_PATH}/src/collectives/*.cc + ${RCCL_PATH}/src/clique/*.cc + ${RCCL_PATH}/src/misc/*.cc + ${RCCL_PATH}/src/transport/*.cc) + list(REMOVE_ITEM RcclCcSources ${RCCL_PATH}/src/misc/gdrwrap.cc) + list(REMOVE_ITEM RcclCcSources ${RCCL_PATH}/src/misc/nvmlwrap.cc) + list(APPEND RcclLibSources ${RcclCppSources} ${RcclCcSources}) + + # RCCL Tests require GTest + find_package(GTest 1.10) + if(NOT GTest_FOUND) + message(STATUS "GTest not found. Skipping RCCL tests.") + else() + # RCCL Test Source Files + include_directories(${RCCL_PATH}/test) + include_directories(${GTEST_INCLUDE_DIR}) + gpu_glob(RcclTestSingleSources ${RCCL_PATH}/test/*.cpp) + gpu_glob(RcclTestMultiSources ${RCCL_PATH}/test/*MultiProcess.cpp) + # Remove all multi process tests from single process tests. + foreach(File IN LISTS RcclTestMultiSources) + list(REMOVE_ITEM RcclTestSingleSources ${File}) + endforeach() + endif() + endif() + foreach(_RocmPath ${ROCM_PATHS}) get_version(_RocmVersion ${_RocmPath}) set(_HIP_Suffix "hip-${_RocmVersion}") @@ -75,6 +188,10 @@ set(_HIP_LDFLAGS -L${_RocmPath}/lib -lamdhip64) set(_HIP_Libs amdhip64-${RocmVersion}) + # FIXME: rccl needs to use hipcc for linking. remove once clang++ supports rccl. + #find_program(hipcc_executable hipcc PATHS ${_RocmPath}/hip) + #message(STATUS "Found hipcc: ${hipcc_executable}") + # Unset these for each iteration of rocm path. set(_ArchFlags) set(_ArchList) @@ -87,7 +204,7 @@ create_hip_test(${_HIP_Suffix}) endforeach() - add_custom_target(hip-tests-all DEPENDS hip-tests-simple + add_custom_target(hip-tests-all DEPENDS hip-tests-simple hip-tests-rccl COMMENT "Build all HIP tests.") file(COPY lit.local.cfg DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")