diff --git a/External/sollve_vv/CMakeLists.txt b/External/sollve_vv/CMakeLists.txt --- a/External/sollve_vv/CMakeLists.txt +++ b/External/sollve_vv/CMakeLists.txt @@ -3,10 +3,313 @@ include(External) +option(TEST_SUITE_SOLLVEVV_FORCE_ALL "Execute all tests, even those known to be unsupported by Clang" OFF) + set(TEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS "-fopenmp-targets=nvptx64-nvidia-cuda;-Xopenmp-target;-march=sm_70" CACHE STRING "Compiler arguments for OpenMP offloading") set(TEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS "-fopenmp-targets=nvptx64-nvidia-cuda;-Xopenmp-target;-march=sm_70;-lomptarget" CACHE STRING "Linker arguments for OpenMP offloading") -function (add_sollvevv LANG) +set(TEST_SUITE_SOLLVEVV_EXPECT_PASS + 4.5/offloading_success.cpp + 4.5/offloading_success.c + 4.5/application_kernels/alpaka_complex_template.cpp + 4.5/application_kernels/linked_list.c + 4.5/application_kernels/mmm_target.c + 4.5/application_kernels/mmm_target_parallel_for_simd.c + 4.5/application_kernels/omp_default_device.c + 4.5/application_kernels/qmcpack_target_math.c + 4.5/application_kernels/gemv_target.cpp + 4.5/application_kernels/gemv_target_many_matrices.cpp + 4.5/application_kernels/gemv_target_reduction.cpp + 4.5/application_kernels/gemv_target_teams_dist_par_for.cpp + 4.5/application_kernels/gridmini_map_struct_float_mul.cpp + 4.5/declare_target/test_declare_target_end_declare_target.c + 4.5/declare_target/test_declare_target_extended_list.c + 4.5/declare_target/test_declare_target_link_extended_list.c + 4.5/declare_target/test_declare_target_to_extended_list.c + 4.5/target/test_target_defaultmap.c + 4.5/target/test_target_firstprivate.c + 4.5/target/test_target_if.c + 4.5/target/test_target_map_global_arrays.c + 4.5/target/test_target_map_local_array.c + 4.5/target/test_target_map_pointer.c + 4.5/target/test_target_map_pointer_no_map_type_modifier.c + 4.5/target/test_target_map_scalar_no_map_type_modifier.c + 4.5/target/test_target_map_struct_default.c + 4.5/target/test_target_map_zero_length_pointer.c + 4.5/target/test_target_private.c + 4.5/target/test_target_device.c + 4.5/target_data/test_target_data_if.c + 4.5/target_data/test_target_data_map_alloc.c + 4.5/target_data/test_target_data_map_array_sections.c + 4.5/target_data/test_target_data_map_devices.c + 4.5/target_data/test_target_data_map_from.c + 4.5/target_data/test_target_data_map_to.c + 4.5/target_data/test_target_data_map_to_from.c + 4.5/target_data/test_target_data_pointer_swap.c + 4.5/target_data/test_target_data_use_device_ptr.c + 4.5/target_data/test_target_data_map_classes.cpp + 4.5/target_enter_data/test_target_enter_data_depend.c + 4.5/target_enter_data/test_target_enter_data_devices.c + 4.5/target_enter_data/test_target_enter_data_global_array.c + 4.5/target_enter_data/test_target_enter_data_if.c + 4.5/target_enter_data/test_target_enter_data_malloced_array.c + 4.5/target_enter_data/test_target_enter_data_struct.c + 4.5/target_enter_exit_data/test_target_enter_exit_data_depend.c + 4.5/target_enter_exit_data/test_target_enter_exit_data_devices.c + 4.5/target_enter_exit_data/test_target_enter_exit_data_if.c + 4.5/target_enter_exit_data/test_target_enter_exit_data_map_global_array.c + 4.5/target_enter_exit_data/test_target_enter_exit_data_map_malloced_array.c + 4.5/target_enter_exit_data/test_target_enter_exit_data_struct.c + 4.5/target_enter_exit_data/test_target_enter_exit_data_classes_simple.cpp + 4.5/target_parallel/test_target_parallel.c + 4.5/target_simd/test_target_simd.c + 4.5/target_simd/test_target_simd_collapse.c + 4.5/target_simd/test_target_simd_safelen.c + 4.5/target_simd/test_target_simd_simdlen.c + 4.5/target_teams_distribute/test_target_teams_distribute_collapse.c + 4.5/target_teams_distribute/test_target_teams_distribute_defaultmap.c + 4.5/target_teams_distribute/test_target_teams_distribute_depend_array_section.c + 4.5/target_teams_distribute/test_target_teams_distribute_depend_in_out.c + 4.5/target_teams_distribute/test_target_teams_distribute_depend_list.c + 4.5/target_teams_distribute/test_target_teams_distribute_depend_out_in.c + 4.5/target_teams_distribute/test_target_teams_distribute_depend_out_out.c + 4.5/target_teams_distribute/test_target_teams_distribute_depend_unused_data.c + 4.5/target_teams_distribute/test_target_teams_distribute_dist_schedule.c + 4.5/target_teams_distribute/test_target_teams_distribute_firstprivate.c + 4.5/target_teams_distribute/test_target_teams_distribute_if.c + 4.5/target_teams_distribute/test_target_teams_distribute_lastprivate.c + 4.5/target_teams_distribute/test_target_teams_distribute_num_teams.c + 4.5/target_teams_distribute/test_target_teams_distribute_private.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_defaultmap.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_devices.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_firstprivate.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_if_no_modifier.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_if_parallel_modifier.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_if_target_modifier.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_map_default.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_map_from.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_map_to.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_map_tofrom.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_num_teams.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_num_threads.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_private.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_schedule_private.c + 4.5/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_thread_limit.c + 4.5/target_update/test_target_update_depend.c + 4.5/target_update/test_target_update_devices.c + 4.5/target_update/test_target_update_from.c + 4.5/target_update/test_target_update_if.c + 4.5/target_update/test_target_update_to.c + 4.5/task/test_target_and_task_nowait.c + 4.5/task/test_task_target.c + 5.0/atomic/test_atomic_acquire_release.c + 5.0/atomic/test_atomic_hint.c + 5.0/atomic/test_atomic_num_hint.c + 5.0/atomic/test_atomic_num_hint_device.c + 5.0/declare_mapper/test_declare_mapper_target_struct.c + 5.0/declare_target/test_declare_target_device_type_host.c + 5.0/declare_target/test_declare_target_nested.c + 5.0/declare_target/test_declare_target_parallel_for.c + 5.0/declare_variant/test_declare_variant.c + 5.0/flush/test_flush_no_memory_order_clause.c + 5.0/master_taskloop/test_master_taskloop.c + 5.0/master_taskloop_simd/test_master_taskloop_simd.c + 5.0/parallel_for/test_parallel_for_allocate.c + 5.0/parallel_for/test_parallel_for_notequals.c + 5.0/parallel_for/test_parallel_for_order_concurrent.c + 5.0/parallel_for_simd/test_parallel_for_simd_atomic.c + 5.0/parallel_master/test_parallel_master.c + 5.0/parallel_master_taskloop/test_parallel_master_taskloop.c + 5.0/parallel_master_taskloop_simd/test_parallel_master_taskloop_simd.c + 5.0/program_control/test_capture_omp_affinity.c + 5.0/program_control/test_omp_get_supported_active_levels.c + 5.0/program_control/test_omp_target_offload_env_DEFAULT.c + 5.0/program_control/test_omp_target_offload_env_DISABLED.c + 5.0/program_control/test_omp_target_offload_env_MANDATORY.c + 5.0/program_control/test_set_and_get_omp_affinity.c + 5.0/requires/test_requires_atomic_default_mem_order_acq_rel.c + 5.0/requires/test_requires_atomic_default_mem_order_relaxed.c + 5.0/requires/test_requires_atomic_default_mem_order_seq_cst.c + 5.0/scan/test_scan.c + 5.0/simd/test_simd_if.c + 5.0/simd/test_simd_nontemporal.c + 5.0/simd/test_simd_order_concurrent.c + 5.0/target/test_target_allocate.c + 5.0/target/test_target_defaultmap_default.c + 5.0/target/test_target_defaultmap_firstprivate.c + 5.0/target/test_target_defaultmap_none.c + 5.0/target/test_target_defaultmap_to_from_tofrom.c + 5.0/target/test_target_imperfect_loop.c + 5.0/target/test_target_map_with_close_modifier.c + 5.0/target/test_target_mapping_before_alloc.c + 5.0/target/test_target_parallel_for_notequals.c + 5.0/target/test_target_uses_allocators_cgroup.c + 5.0/target/test_target_uses_allocators_const.c + 5.0/target/test_target_uses_allocators_default.c + 5.0/target/test_target_uses_allocators_high_bw.c + 5.0/target/test_target_uses_allocators_large_cap.c + 5.0/target/test_target_uses_allocators_low_lat.c + 5.0/target/test_target_uses_allocators_pteam.c + 5.0/target/test_target_uses_allocators_thread.c + 5.0/target_data/test_target_data_use_device_addr.c + 5.0/target_data/test_target_data_use_device_ptr.c + 5.0/target_simd/test_target_simd_if.c + 5.0/target_simd/test_target_simd_nontemporal.c + 5.0/target_simd/test_target_simd_order_concurrent.c + 5.0/target_teams_distribute_parallel_for/test_target_teams_distribute_parallel_for_collapse.c + 5.0/target_teams_distribute_parallel_for_simd/test_target_teams_distribute_parallel_for_simd_atomic.c + 5.0/target_update/test_target_update_mapper_to_discontiguous.c + 5.0/task/test_parallel_for_reduction_task.c + 5.0/task/test_parallel_for_reduction_task_device.c + 5.0/task/test_task_depend_mutexinoutset.c + 5.0/task/test_task_detach.c + 5.0/task/test_task_in_reduction.c + 5.0/task/test_task_in_reduction_dynamically_enclosed.c + 5.0/taskgroup/test_taskgroup_task_reduction.c + 5.0/taskloop/test_taskloop_in_reduction.c + 5.0/taskloop_simd/test_taskloop_simd_in_reduction.c + 5.0/teams/test_team_default_shared.c + 5.0/teams/test_teams.c + 5.0/application_kernels/gridmini_map_class.cpp + 5.0/application_kernels/gridmini_map_struct_array.cpp + 5.0/application_kernels/gridmini_map_struct_float_mul.cpp + 5.0/application_kernels/gridmini_map_template.cpp + 5.0/application_kernels/gridmini_map_template_array.cpp + 5.0/application_kernels/gridmini_ptr.cpp + 5.0/depobj/test_depobj_depend_update_destroy.cpp + 5.0/teams/test_teams_distribute_default_none.c + ) +set(TEST_SUITE_SOLLVEVV_EXPECT_FLAKY + 5.0/declare_target/test_declare_target_device_type_any.c + 5.0/declare_target/test_nested_declare_target.c + + 4.5/target/test_target_depends.c + 4.5/target/test_target_map_classes_default.cpp + 4.5/target/test_target_map_array_default.c + 4.5/target/test_target_is_device_ptr.c + 4.5/target_data/test_target_data_map_pointer_translation.c + 4.5/target_data/test_target_data_map_tofrom.c + 4.5/target_enter_data/test_target_enter_data_classes_inheritance.cpp + 4.5/target_enter_data/test_target_enter_data_classes_simple.cpp + 4.5/target_enter_exit_data/test_target_enter_exit_data_classes_complex.cpp + 4.5/target_enter_exit_data/test_target_enter_exit_data_map_pointer_translation.c + 4.5/target_simd/test_nested_target_simd.c + 5.0/target/test_target_map_classes_default.cpp + 5.0/task/test_task_depend_iterator.cpp + ) +set(TEST_SUITE_SOLLVEVV_EXPECT_FAIL_RUN + 4.5/target_teams_distribute/test_target_teams_distribute.c + 4.5/target_teams_distribute/test_target_teams_distribute_default_none.c + 4.5/target_teams_distribute/test_target_teams_distribute_default_shared.c + 4.5/target_teams_distribute/test_target_teams_distribute_depend_disjoint_section.c + 4.5/target_teams_distribute/test_target_teams_distribute_depend_in_in.c + 4.5/target_teams_distribute/test_target_teams_distribute_device.c + 4.5/target_teams_distribute/test_target_teams_distribute_is_device_ptr.c + 4.5/target_teams_distribute/test_target_teams_distribute_map.c + 4.5/target_teams_distribute/test_target_teams_distribute_nowait.c + 4.5/target_teams_distribute/test_target_teams_distribute_shared.c + 4.5/target_teams_distribute/test_target_teams_distribute_thread_limit.c + 5.0/target/test_target_task_depend_mutexinoutset.c + 5.0/master_taskloop/test_master_taskloop_device.c + 5.0/master_taskloop_simd/test_master_taskloop_simd_device.c + 5.0/parallel_master/test_parallel_master_device.c + 5.0/parallel_master_taskloop/test_parallel_master_taskloop_device.c + 5.0/parallel_master_taskloop_simd/test_parallel_master_taskloop_simd_device.c + 5.0/requires/test_requires_unified_address.c + 5.0/requires/test_requires_unified_shared_memory.c + 5.0/requires/test_requires_unified_shared_memory_heap.c + 5.0/requires/test_requires_unified_shared_memory_heap_is_device_ptr.c + 5.0/requires/test_requires_unified_shared_memory_heap_map.c + 5.0/requires/test_requires_unified_shared_memory_omp_target_alloc.c + 5.0/requires/test_requires_unified_shared_memory_omp_target_alloc_is_device_ptr.c + 5.0/requires/test_requires_unified_shared_memory_stack.c + 5.0/requires/test_requires_unified_shared_memory_stack_is_device_ptr.c + 5.0/requires/test_requires_unified_shared_memory_stack_map.c + 5.0/requires/test_requires_unified_shared_memory_static.c + 5.0/requires/test_requires_unified_shared_memory_static_is_device_ptr.c + 5.0/requires/test_requires_unified_shared_memory_static_map.c + ) +set(TEST_SUITE_SOLLVEVV_EXPECT_FAIL_LINK + 4.5/application_kernels/qmcpack_target_static_lib.c + 5.0/target/test_target_device.c + 5.0/program_control/test_omp_get_device_num.c + 5.0/task/test_task_in_reduction_device.c + 5.0/taskloop/test_taskloop_in_reduction_device.c + 5.0/taskloop_simd/test_taskloop_simd_in_reduction_device.c + 5.0/taskgroup/test_taskgroup_task_reduction_device.c + 5.0/metadirective/test_metadirective_arch_is_nvidia.c + 5.0/requires/test_requires_dynamic_allocators.c + ) +set(TEST_SUITE_SOLLVEVV_EXPECT_FAIL_COMPILE + 4.5/application_kernels/reduction_separated_directives.cpp + 4.5/application_kernels/reduction_separated_directives.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_add.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_bitor.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_bitxor.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_or.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_subtract.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_min.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_bitand.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_and.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_max.c + 4.5/target_teams_distribute/test_target_teams_distribute_reduction_multiply.c + + 5.0/allocate/test_allocate_allocator.c + 5.0/declare_target/test_declare_target_nested_functions.c + 5.0/declare_target/test_declare_target_device_type_nohost.c + 5.0/requires/test_requires_reverse_offload.c + 5.0/loop/test_loop_reduction_bitxor_device.c + 5.0/loop/test_loop_bind.c + 5.0/loop/test_loop_bind_device.c + 5.0/loop/test_loop_collapse.c + 5.0/loop/test_loop_collapse_device.c + 5.0/loop/test_loop_lastprivate.c + 5.0/loop/test_loop_lastprivate_device.c + 5.0/loop/test_loop_nested.c + 5.0/loop/test_loop_nested_device.c + 5.0/loop/test_loop_order_concurrent.c + 5.0/loop/test_loop_order_concurrent_device.c + 5.0/loop/test_loop_private.c + 5.0/loop/test_loop_private_device.c + 5.0/loop/test_loop_reduction_add.c + 5.0/loop/test_loop_reduction_add_device.c + 5.0/loop/test_loop_reduction_and.c + 5.0/loop/test_loop_reduction_and_device.c + 5.0/loop/test_loop_reduction_bitand.c + 5.0/loop/test_loop_reduction_bitand_device.c + 5.0/loop/test_loop_reduction_bitor.c + 5.0/loop/test_loop_reduction_bitor_device.c + 5.0/loop/test_loop_reduction_bitxor.c + 5.0/loop/test_loop_reduction_max.c + 5.0/loop/test_loop_reduction_max_device.c + 5.0/loop/test_loop_reduction_min.c + 5.0/loop/test_loop_reduction_min_device.c + 5.0/loop/test_loop_reduction_multiply.c + 5.0/loop/test_loop_reduction_multiply_device.c + 5.0/loop/test_loop_reduction_or.c + 5.0/loop/test_loop_reduction_or_device.c + 5.0/loop/test_loop_reduction_subtract.c + 5.0/loop/test_loop_reduction_subtract_device.c + 5.0/metadirective/test_metadirective_arch_nvidia_or_amd.c + 5.0/target/test_target_in_reduction.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_min.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_subtract.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_or.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_bitor.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_bitand.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_add.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_bitxor.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_multiply.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_and.c + 5.0/target_update/test_target_update_mapper_from_discontiguous.c + 5.0/taskwait/test_taskwait_depend.c + 5.0/target_teams_distribute/test_target_teams_distribute_reduction_max.c + 5.1/atomic/test_atomic_compare.c + ) + + +macro (add_sollvevv LANG) set(_includedir "${TEST_SUITE_SOLLVEVV_ROOT}/ompvv" ) if (NOT OPENMP_${LANG}_FOUND) @@ -36,9 +339,65 @@ string(REPLACE "." "" _directory "${_directory}") set(_name "omptargetvv-${_basename}-${_directory}-${_ext}") + if (TEST_SUITE_SOLLVEVV_FORCE_ALL OR NOT CMAKE_${LANG}_COMPILER_ID STREQUAL "Clang" OR "${_file}" IN_LIST TEST_SUITE_SOLLVEVV_EXPECT_PASS) + set(_expect_working_compile 1) + set(_expect_working_link 1) + set(_expect_working_run 1) + elseif ("${_file}" IN_LIST TEST_SUITE_SOLLVEVV_EXPECT_FAIL_COMPILE) + set(_expect_working_compile 0) + set(_expect_working_link 0) + set(_expect_working_run 0) + elseif ("${_file}" IN_LIST TEST_SUITE_SOLLVEVV_EXPECT_FAIL_LINK) + set(_expect_working_compile 1) + set(_expect_working_link 0) + set(_expect_working_run 0) + elseif ("${_file}" IN_LIST TEST_SUITE_SOLLVEVV_EXPECT_FAIL_RUN) + set(_expect_working_compile 1) + set(_expect_working_link 1) + set(_expect_working_run 0) + elseif ("${_file}" IN_LIST TEST_SUITE_SOLLVEVV_EXPECT_FLAKY) + set(_expect_working_compile 1) + set(_expect_working_link 1) + set(_expect_working_run 2) + else () + list(APPEND _unkown_tests "${_file}") + continue() + endif () + + if(NOT _expect_working_link AND ${CMAKE_VERSION} VERSION_LESS "3.21") + # Requires CMake 3.21 for _LINKER_LAUNCHER support. + continue() + endif () + llvm_test_run() + if (NOT _expect_working_run) + set(TESTSCRIPT "${TESTSCRIPT}XFAIL: *\n") + elseif (_expect_working_run EQUAL 2) + set(TESTSCRIPT "${TESTSCRIPT}FLAKY: *\n") + endif () llvm_test_executable(${_name} "${TEST_SUITE_SOLLVEVV_ROOT}/tests/${_file}") + if (NOT _expect_working_compile) + add_dependencies(${_name} build-timeit) + get_property(_prev_compiler_launcher + TARGET ${_name} + PROPERTY ${LANG}_COMPILER_LAUNCHER + ) + set_property(TARGET ${_name} + PROPERTY ${LANG}_COMPILER_LAUNCHER "${CMAKE_BINARY_DIR}/tools/timeit" --expect-fail --expect-crash ${_prev_compiler_launcher} + ) + endif () + if (NOT _expect_working_link) + add_dependencies(${_name} build-timeit) + get_property(_prev_linker_launcher + TARGET ${_name} + PROPERTY ${LANG}_LINKER_LAUNCHER + ) + set_property(TARGET ${_name} + PROPERTY ${LANG}_LINKER_LAUNCHER "${CMAKE_BINARY_DIR}/tools/timeit" --expect-fail --expect-crash ${_prev_linker_launcher} + ) + endif () + target_include_directories(${_name} PRIVATE "${_includedir}") target_link_libraries(${_name} PUBLIC OpenMP::OpenMP_${_lang}) @@ -49,7 +408,8 @@ target_compile_options(${_name} PUBLIC ${TEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS}) target_link_options(${_name} PUBLIC ${TEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS}) endforeach () -endfunction () +endmacro () + llvm_externals_find(TEST_SUITE_SOLLVEVV_ROOT "sollve_vv" "OpenMP Offloading Validation & Verification Suite") @@ -61,9 +421,16 @@ return () endif () + set(_unkown_tests) foreach (_lang in C CXX) if (CMAKE_${_lang}_COMPILER) add_sollvevv(${_lang}) endif() endforeach () + + if (_unkown_tests) + message(SEND_ERROR "Found the following unknown tests in SOLLVE V&V sources:\n" + "${_unkown_tests}\n" + "These tests need to be classified whether they are working with the latest clang. Add them to either TEST_SUITE_SOLLVEVV_EXPECT_PASS, TEST_SUITE_SOLLVEVV_EXPECT_FAIL_RUN, TEST_SUITE_SOLLVEVV_EXPECT_FAIL_LINK, TEST_SUITE_SOLLVEVV_EXPECT_FAIL_COMPILE, or TEST_SUITE_SOLLVEVV_EXPECT_FLAKY") + endif () endif () diff --git a/litsupport/test.py b/litsupport/test.py --- a/litsupport/test.py +++ b/litsupport/test.py @@ -20,9 +20,13 @@ try: NOCHANGE = lit.Test.ResultCode('NOCHANGE', 'Executable Unchanged', False) NOEXE = lit.Test.ResultCode('NOEXE', 'Executable Missing', True) + FLAKYPASS = lit.Test.ResultCode('FLAKYPASS', 'Flaky (passed)', False) + FLAKYFAIL = lit.Test.ResultCode('FLAKYFAIL', 'Flaky (failed)', False) except TypeError: NOCHANGE = lit.Test.ResultCode('NOCHANGE', False) NOEXE = lit.Test.ResultCode('NOEXE', True) + FLAKYPASS = lit.Test.ResultCode('FLAKYPASS', False) + FLAKYFAIL = lit.Test.ResultCode('FLAKYFAIL', False) class TestSuiteTest(lit.formats.ShTest): @@ -46,27 +50,40 @@ # Report missing test executables. if not os.path.exists(context.executable): - return lit.Test.Result(NOEXE, "Executable '%s' is missing" % + result = lit.Test.Result(NOEXE, "Executable '%s' is missing" % context.executable) + else: + # Skip unchanged tests + if config.previous_results: + litsupport.modules.hash.compute(context) + if litsupport.modules.hash.same_as_previous(context): + result = lit.Test.Result( + NOCHANGE, 'Executable identical to previous run') + val = lit.Test.toMetricValue(context.executable_hash) + result.addMetric('hash', val) + return result - # Skip unchanged tests - if config.previous_results: - litsupport.modules.hash.compute(context) - if litsupport.modules.hash.same_as_previous(context): - result = lit.Test.Result( - NOCHANGE, 'Executable identical to previous run') - val = lit.Test.toMetricValue(context.executable_hash) - result.addMetric('hash', val) - return result + # Let test modules modify the test plan. + for modulename in config.test_modules: + module = litsupport.modules.modules.get(modulename) + if module is None: + raise Exception("Unknown testmodule '%s'" % modulename) + module.mutatePlan(context, plan) - # Let test modules modify the test plan. - for modulename in config.test_modules: - module = litsupport.modules.modules.get(modulename) - if module is None: - raise Exception("Unknown testmodule '%s'" % modulename) - module.mutatePlan(context, plan) + # Execute Test plan + result = litsupport.testplan.executePlanTestResult(context, plan) - # Execute Test plan - result = litsupport.testplan.executePlanTestResult(context, plan) + + if context.xfail: + if result.code.isFailure: + result.code = lit.Test.XFAIL + else: + result.code = lit.Test.XPASS + + if context.flaky: + if result.code.isFailure: + result.code = FLAKYFAIL + else: + result.code = FLAKYPASS return result diff --git a/litsupport/testfile.py b/litsupport/testfile.py --- a/litsupport/testfile.py +++ b/litsupport/testfile.py @@ -26,10 +26,12 @@ runscript = [] verifyscript = [] metricscripts = {} + xfail = False + flaky = False # Note that we keep both "RUN" and "RUN:" in the list to stay compatible # with older lit versions. keywords = ['PREPARE:', 'PREPARE', 'RUN:', 'RUN', 'VERIFY:', 'VERIFY', - 'METRIC:', 'METRIC'] + 'METRIC:', 'METRIC', 'XFAIL', 'FLAKY'] for line_number, command_type, ln in \ parseIntegratedTestScriptCommands(filename, keywords): if command_type.startswith('PREPARE'): @@ -42,6 +44,10 @@ metric, ln = ln.split(':', 1) metricscript = metricscripts.setdefault(metric.strip(), list()) _parseShellCommand(metricscript, ln) + elif command_type.startswith('XFAIL'): + xfail = True + elif command_type.startswith('FLAKY'): + flaky = True else: raise ValueError("unknown script command type: %r" % ( command_type,)) @@ -75,3 +81,5 @@ context.executable = shellcommand.getMainExecutable(context) if not context.executable: logging.error("Could not determine executable name in %s" % filename) + context.xfail = xfail + context.flaky = flaky diff --git a/tools/timeit.c b/tools/timeit.c --- a/tools/timeit.c +++ b/tools/timeit.c @@ -7,6 +7,7 @@ |* *| \*===----------------------------------------------------------------------===*/ +#include #include #include #include @@ -43,6 +44,18 @@ /* \brief Record the child command name, for error messages. */ static const char *g_target_program = 0; +/* \brief Exit with nonzero exit code unless child fails. */ +static int g_expect_fail = 0; + +/* \brief Exit with nonzero exit code unless child crashes. */ +static int g_expect_crash = 0; + +/* \brief Exit with zero exit code even if child fails. */ +static int g_allow_fail = 0; + +/* \brief Exit with zero exit code even if child crashes. */ +static int g_allow_crash = 0; + /* \brief If given, report output in POSIX mode format. */ static int g_posix_mode = 0; @@ -220,19 +233,43 @@ fclose(fp); } + int timeit_retcode = exit_status; + bool child_success = exit_status != 0; + bool child_crashed = exit_status < 0 || 128 <= exit_status; + bool child_failed = !child_crashed && !child_success; + + if (g_allow_fail && child_failed) { + timeit_retcode = 0; + } else if (g_allow_crash && child_crashed) { + timeit_retcode = 0; + } else if (g_expect_fail || g_expect_crash) { + if (child_failed) { + timeit_retcode = !g_expect_fail; + } else if (child_crashed) { + timeit_retcode = !g_expect_crash; + } else { + // success + timeit_retcode = 1; + } + } + if (g_append_exitstats && g_target_program) { - FILE *fp_stdout = fopen(g_target_redirect_stdout, "a"); - if (!fp_stdout) { - perror("fopen"); - return EXITCODE_MONITORING_FAILURE; + if (!g_summary_file) { + fprintf(stderr, "exit %d\n", exit_status); + } else { + FILE *fp_stdout = fopen(g_target_redirect_stdout, "a"); + if (!fp_stdout) { + perror("fopen"); + return EXITCODE_MONITORING_FAILURE; + } + fprintf(fp_stdout, "exit %d\n", exit_status); + fclose(fp_stdout); } - fprintf(fp_stdout, "exit %d\n", exit_status); - fclose(fp_stdout); /* let timeit itself report success */ - exit_status = 0; + timeit_retcode = 0; } - return exit_status; + return timeit_retcode; } #define set_resource_limit(resource, value) \ @@ -408,6 +445,16 @@ fprintf(stderr, "Options:\n"); fprintf(stderr, " %-20s %s", "-h, --help", "Show this help text.\n"); + fprintf(stderr, " %-20s %s", "--expect-fail", + "Return successfully only if program fails.\n"); + fprintf( + stderr, " %-20s %s", "--expect-crash", + "Return successfully only if program crashes; if combined with " + "--expect-fail returns successfully if the program fails or crashes.\n"); + fprintf(stderr, " %-20s %s", "--allow-fail", + "Return successfully if program fails.\n"); + fprintf(stderr, " %-20s %s", "--allow-crash", + "Return successfully if program crashes.\n"); fprintf(stderr, " %-20s %s", "-p, --posix", "Report time in /usr/bin/time POSIX format.\n"); fprintf(stderr, " %-20s %s", "-t, --timeout ", @@ -525,6 +572,26 @@ continue; } + if (streq(arg, "--expect-fail")) { + g_expect_fail = 1; + continue; + } + + if (streq(arg, "--expect-crash")) { + g_expect_crash = 1; + continue; + } + + if (streq(arg, "--allow-fail")) { + g_allow_fail = 1; + continue; + } + + if (streq(arg, "--allow-crash")) { + g_allow_crash = 1; + continue; + } + if (streq(arg, "-c") || streq(arg, "--chdir")) { if (i + 1 == argc) { fprintf(stderr, "error: %s argument requires an option\n", arg);