diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -676,6 +676,9 @@ if (MSVC_IDE OR XCODE) set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar") endif() +if(LLVM_INDIVIDUAL_TEST_COVERAGE) + set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --per-test-coverage") +endif() set(LLVM_LIT_ARGS "${LIT_ARGS_DEFAULT}" CACHE STRING "Default options for lit") # On Win32 hosts, provide an option to specify the path to the GnuWin32 tools. @@ -797,24 +800,25 @@ option(LLVM_ADD_NATIVE_VISUALIZERS_TO_SOLUTION "Configure project to use Visual Studio native visualizers" TRUE) endif() -if (LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE OR - LLVM_ENABLE_IR_PGO) - if(NOT LLVM_PROFILE_MERGE_POOL_SIZE) - # A pool size of 1-2 is probably sufficient on a SSD. 3-4 should be fine - # for spining disks. Anything higher may only help on slower mediums. - set(LLVM_PROFILE_MERGE_POOL_SIZE "4") - endif() - if(NOT LLVM_PROFILE_FILE_PATTERN) - if(NOT LLVM_PROFILE_DATA_DIR) - file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/profiles" LLVM_PROFILE_DATA_DIR) +if(NOT LLVM_INDIVIDUAL_TEST_COVERAGE) + if(LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE OR LLVM_ENABLE_IR_PGO) + if(NOT LLVM_PROFILE_MERGE_POOL_SIZE) + # A pool size of 1-2 is probably sufficient on an SSD. 3-4 should be fine + # for spinning disks. Anything higher may only help on slower mediums. + set(LLVM_PROFILE_MERGE_POOL_SIZE "4") endif() - file(TO_NATIVE_PATH "${LLVM_PROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_PROFILE_FILE_PATTERN) - endif() - if(NOT LLVM_CSPROFILE_FILE_PATTERN) - if(NOT LLVM_CSPROFILE_DATA_DIR) - file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/csprofiles" LLVM_CSPROFILE_DATA_DIR) + if(NOT LLVM_PROFILE_FILE_PATTERN) + if(NOT LLVM_PROFILE_DATA_DIR) + file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/profiles" LLVM_PROFILE_DATA_DIR) + endif() + file(TO_NATIVE_PATH "${LLVM_PROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_PROFILE_FILE_PATTERN) + endif() + if(NOT LLVM_CSPROFILE_FILE_PATTERN) + if(NOT LLVM_CSPROFILE_DATA_DIR) + file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/csprofiles" LLVM_CSPROFILE_DATA_DIR) + endif() + file(TO_NATIVE_PATH "${LLVM_CSPROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_CSPROFILE_FILE_PATTERN) endif() - file(TO_NATIVE_PATH "${LLVM_CSPROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_CSPROFILE_FILE_PATTERN) endif() endif() diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake --- a/llvm/cmake/modules/HandleLLVMOptions.cmake +++ b/llvm/cmake/modules/HandleLLVMOptions.cmake @@ -1160,6 +1160,7 @@ endif() option(LLVM_BUILD_INSTRUMENTED_COVERAGE "Build LLVM and tools with Code Coverage instrumentation" Off) +option(LLVM_INDIVIDUAL_TEST_COVERAGE "Emit individual coverage file for each test case." OFF) mark_as_advanced(LLVM_BUILD_INSTRUMENTED_COVERAGE) append_if(LLVM_BUILD_INSTRUMENTED_COVERAGE "-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\" -fcoverage-mapping" CMAKE_CXX_FLAGS diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst --- a/llvm/docs/CMake.rst +++ b/llvm/docs/CMake.rst @@ -375,6 +375,12 @@ will limit code coverage summaries to just the listed directories. If unset, coverage reports will include all sources identified by the tooling. + **LLVM_INDIVIDUAL_TEST_COVERAGE**: BOOL + Enable individual test case coverage. When set to ON, code coverage data for + each test case will be generated and stored in a separate directory under the + config.test_exec_root path. This feature allows code coverage analysis of each + individual test case. Defaults to OFF. + **LLVM_BUILD_LLVM_DYLIB**:BOOL If enabled, the target for building the libLLVM shared library is added. This library contains all of LLVM's components in a single shared library. diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -181,6 +181,12 @@ Run the tests in a random order, not failing/slowest first. Deprecated, use :option:`--order` instead. +.. option:: --per-test-coverage + + Emit the necessary test coverage data, divided per test case (involves + setting a unique value to LLVM_PROFILE_FILE for each RUN). The coverage + data files will be emitted in the directory specified by `config.test_exec_root`. + .. option:: --max-failures N Stop execution after the given number ``N`` of failures. diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py --- a/llvm/utils/lit/lit/LitConfig.py +++ b/llvm/utils/lit/lit/LitConfig.py @@ -37,6 +37,7 @@ maxIndividualTestTime=0, parallelism_groups={}, echo_all_commands=False, + per_test_coverage=False, ): # The name of the test runner. self.progname = progname @@ -87,6 +88,7 @@ self.maxIndividualTestTime = maxIndividualTestTime self.parallelism_groups = parallelism_groups self.echo_all_commands = echo_all_commands + self.per_test_coverage = per_test_coverage @property def maxIndividualTestTime(self): @@ -128,6 +130,22 @@ elif self.maxIndividualTestTime < 0: self.fatal("The timeout per test must be >= 0 seconds") + @property + def per_test_coverage(self): + """ + Interface for getting the per_test_coverage value + """ + return self._per_test_coverage + + @per_test_coverage.setter + def per_test_coverage(self, value): + """ + Interface for setting the per_test_coverage value + """ + if not isinstance(value, bool): + self.fatal("per_test_coverage must set to a value of type bool.") + self._per_test_coverage = value + def load_config(self, config, path): """load_config(config, path) - Load a config object from an alternate path.""" diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py --- a/llvm/utils/lit/lit/TestRunner.py +++ b/llvm/utils/lit/lit/TestRunner.py @@ -1067,10 +1067,25 @@ def executeScript(test, litConfig, tmpBase, commands, cwd): bashPath = litConfig.getBashPath() isWin32CMDEXE = litConfig.isWindows and not bashPath + coverage_index = 0 # Counter for coverage file index script = tmpBase + ".script" if isWin32CMDEXE: script += ".bat" + # Set unique LLVM_PROFILE_FILE for each run command + for j, ln in enumerate(commands): + match = re.match(kPdbgRegex, ln) + if match: + command = match.group(2) + commands[j] = match.expand(": '\\1'; \\2" if command else ": '\\1'") + if litConfig.per_test_coverage: + # Extract the test case name from the test object + test_case_name = test.path_in_suite[-1] + test_case_name = test_case_name.rsplit(".", 1)[0] # Remove the file extension + llvm_profile_file = f"{test_case_name}{coverage_index}.profraw" + commands[j] = f"export LLVM_PROFILE_FILE={llvm_profile_file} && {commands[j]}" + coverage_index += 1 + # Write script file mode = "w" open_kwargs = {} diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -184,6 +184,12 @@ help="Do not fail the run if all tests are filtered out", action="store_true", ) + execution_group.add_argument( + "--per-test-coverage", + dest="per_test_coverage", + action="store_true", + help="Enable individual test case coverage", + ) execution_group.add_argument( "--ignore-fail", dest="ignoreFail", diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -41,6 +41,7 @@ params=params, config_prefix=opts.configPrefix, echo_all_commands=opts.echoAllCommands, + per_test_coverage=opts.per_test_coverage, ) discovered_tests = lit.discovery.find_tests_for_inputs( diff --git a/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/lit.cfg b/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/lit.cfg @@ -0,0 +1,8 @@ +import lit.formats +import os + +config.name = "per-test-coverage-by-lit-cfg" +config.suffixes = [".py"] +config.test_format = lit.formats.ShTest(execute_external=True) +lit_config.per_test_coverage = True +config.substitutions.append(("%{python}", '"%s"' % (sys.executable))) diff --git a/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py b/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py @@ -0,0 +1,11 @@ +# Check that the environment variable is set correctly +# RUN: %{python} %s | FileCheck %s + +# Python script to read the environment variable +# and print its value +import os + +llvm_profile_file = os.environ.get('LLVM_PROFILE_FILE') +print(llvm_profile_file) + +# CHECK: per-test-coverage-by-lit-cfg0.profraw diff --git a/llvm/utils/lit/tests/Inputs/per-test-coverage/lit.cfg b/llvm/utils/lit/tests/Inputs/per-test-coverage/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/per-test-coverage/lit.cfg @@ -0,0 +1,8 @@ +import lit.formats +import os + +config.name = "per-test-coverage" +config.suffixes = [".py"] +config.test_format = lit.formats.ShTest(execute_external=True) +config.substitutions.append(("%{python}", '"%s"' % (sys.executable))) + diff --git a/llvm/utils/lit/tests/Inputs/per-test-coverage/per-test-coverage.py b/llvm/utils/lit/tests/Inputs/per-test-coverage/per-test-coverage.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/per-test-coverage/per-test-coverage.py @@ -0,0 +1,11 @@ +# Check that the environment variable is set correctly +# RUN: %{python} %s | FileCheck %s + +# Python script to read the environment variable +# and print its value +import os + +llvm_profile_file = os.environ.get('LLVM_PROFILE_FILE') +print(llvm_profile_file) + +# CHECK: per-test-coverage0.profraw diff --git a/llvm/utils/lit/tests/per-test-coverage-by-lit-cfg.py b/llvm/utils/lit/tests/per-test-coverage-by-lit-cfg.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/per-test-coverage-by-lit-cfg.py @@ -0,0 +1,6 @@ +# Test if lit_config.per_test_coverage in lit.cfg sets individual test case coverage. + +# RUN: %{lit} -a -v %{inputs}/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py \ +# RUN: | FileCheck -match-full-lines %s +# +# CHECK: PASS: per-test-coverage-by-lit-cfg :: per-test-coverage-by-lit-cfg.py ({{[^)]*}}) diff --git a/llvm/utils/lit/tests/per-test-coverage.py b/llvm/utils/lit/tests/per-test-coverage.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/per-test-coverage.py @@ -0,0 +1,6 @@ +# Test LLVM_PROFILE_FILE is set when --per-test-coverage is passed to command line. + +# RUN: %{lit} -a -v --per-test-coverage %{inputs}/per-test-coverage/per-test-coverage.py \ +# RUN: | FileCheck -match-full-lines %s +# +# CHECK: PASS: per-test-coverage :: per-test-coverage.py ({{[^)]*}})