diff --git a/llvm/utils/git-check-coverage b/llvm/utils/git-check-coverage new file mode 100755 --- /dev/null +++ b/llvm/utils/git-check-coverage @@ -0,0 +1,463 @@ +#!/usr/bin/env python3 +# +# ===- git-check-coverage - CheckCoverage Git Integration ---------*- python -*--===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===------------------------------------------------------------------------===# + +r""" +code-coverage git integration +============================ + +This file provides a code-coverage integration for git. Put it somewhere in your +path and ensure that it is executable. Code coverage information will be +provided for the last commit/HEAD by runing `git code-coverage`. + +Requires Python 2.7 or Python 3 +""" + +import argparse +import os +import subprocess +import re +import sys +from unidiff import PatchSet + + +def create_patch_from_last_commit(output_path): + """Create a patch file from the last commit in the Git repository.""" + try: + # Get the hash of the last commit using `git rev-parse` command + git_hash_cmd = ["git", "rev-parse", "HEAD"] + commit_hash = subprocess.check_output(git_hash_cmd).strip().decode("utf-8") + + # Create the patch from the last commit using `git show` command + patch_cmd = ["git", "show", commit_hash] + patch_output = subprocess.check_output(patch_cmd).decode("utf-8") + + # Write the patch to the output file + with open(output_path, "w") as patch_file: + patch_file.write(patch_output) + + print("Patch file '{}' created successfully.".format(output_path)) + print() + + except subprocess.CalledProcessError as e: + print("Error while creating the patch from the last commit:", e) + sys.exit(1) + + +def extract_source_files_from_patch(patch_path): + """ "read the patch file and extract the names of .cpp and .h files that + have been modified or added in the patch.""" + try: + source_files = [] + with open(patch_path, "r") as patch_file: + patch_diff = patch_file.read() + + # Use regular expression to find .cpp files in the patch + source_file_matches = re.findall(r"\+{3} b/(\S+\.(?:cpp|h))", patch_diff) + + source_files.extend(source_file_matches) + + for source_file in source_files: + print("Source files in the patch:") + print(source_file) + print() + return source_files + + except Exception as ex: + print("Error while extracting .cpp files from patch:", ex) + return [] + + +def extract_modified_lines_from_patch(patch_path, tests): + """ "Extract the modified source lines from the patch.""" + + source_lines = {} # Dictionary for added lines in source code files + + try: + patchset = PatchSet.from_filename(patch_path) + + for patched_file in patchset: + current_file = patched_file.target_file + if current_file not in tests: + source_lines[current_file] = [] + + for hunk in patched_file: + for line in hunk: + if line.is_added: + if current_file not in tests: + source_lines[current_file].append( + (line.target_line_no, line.value[1:]) + ) + + return source_lines + + except Exception as ex: + print("Error while extracting modified lines from patch:", ex) + return {} + + +def build_llvm(build_dir, llvm_source_dir): + """ "Configure and build LLVM in the specified build directory.""" + try: + cwd = os.getcwd() + + # Change to the build directory + os.chdir(build_dir) + + # Run the cmake command to configure LLVM build + cmake_command = [ + "cmake", + "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON", + "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON", + "." + ] + + subprocess.check_call(cmake_command) + + # Run the ninja build command + subprocess.check_call(["ninja"]) + os.chdir(cwd) + + print("LLVM build completed successfully.") + + except subprocess.CalledProcessError as e: + print("Error during LLVM build:", e) + sys.exit(1) + + +def run_single_test_with_coverage(llvm_lit_path, test_path): + """ "Run a single test case using llvm-lit with coverage.""" + try: + lit_cmd = [llvm_lit_path, "--per-test-coverage", test_path] + subprocess.check_call(lit_cmd) + + print("Test case executed:", test_path) + + except subprocess.CalledProcessError as e: + print("Error while running test:", e) + sys.exit(1) + + except Exception as ex: + print("Error:", ex) + sys.exit(1) + + +def run_modified_added_tests(llvm_lit_path, patch_path, tests): + """ "Read the patch file, identify modified and added test cases, and + then execute each of these test cases""" + try: + # Get the list of modified and added test cases from the patch + with open(patch_path, "r") as patch_file: + patch_diff = patch_file.read() + + modified_tests = [] + added_tests = [] + + # Use regular expressions to find modified and added test cases with ".ll" extension + for match in re.finditer(r"^\+\+\+ [ab]/(.*\.(ll|c|cpp))$", patch_diff, re.MULTILINE): + test_file = match.group(1) + if test_file in tests: + if match.group(0).startswith("+++"): + added_tests.append(test_file) + else: + modified_tests.append(test_file) + + if not modified_tests and not added_tests: + print("No modified or added tests found in the patch.") + return + + # Run each modified test case + print("Running modified test cases:") + for test_file in modified_tests: + run_single_test_with_coverage(llvm_lit_path, test_file) + print() + + # Run each newly added test case + print("Running added test cases:") + for test_file in added_tests: + run_single_test_with_coverage(llvm_lit_path, test_file) + print() + + except subprocess.CalledProcessError as e: + print("Error while running modified and added tests:", e) + sys.exit(1) + + except Exception as ex: + print("Error:", ex) + sys.exit(1) + + +def convert_profraw_to_profdata_patch(cpp_files, build_dir): + """ "Convert profraw coverage data files to profdata format, generate human-readable + coverage information, and extract coverage data for specific files.""" + # Create a list to store the paths of the generated coverage data files + os.chdir(build_dir) + coverage_files = [] + try: + # Change to the build directory + for root, dirs, files in os.walk("."): + for file in files: + if os.path.basename(file) == "default.profraw": + continue + if file.endswith(".profraw"): + profraw_file = os.path.join(root, file) + profdata_output = os.path.splitext(profraw_file)[0] + ".profdata" + + print("Profraw File:", profraw_file) + print("Profdata File:", profdata_output) + + # Get the current working directory + current_directory = os.getcwd() + print("Current Working Directory:", current_directory) + + # Construct the llvm-profdata command + llvm_profdata_cmd = [ + "./bin/llvm-profdata", + "merge", + "-o", + profdata_output, + profraw_file, + ] + + # Run llvm-profdata to convert profraw to profdata + subprocess.check_call(llvm_profdata_cmd) + + print("Converted {} to {}".format(profraw_file, profdata_output)) + + # Construct the llvm-cov show command + llvm_cov_cmd = [ + "bin/llvm-cov", + "show", + "-instr-profile", + profdata_output, + "bin/opt", + ] + llvm_cov_output = os.path.splitext(profdata_output)[0] + ".txt" + + # Redirect the output of llvm-cov show to a text file + with open(llvm_cov_output, "w") as output_file: + subprocess.check_call(llvm_cov_cmd, stdout=output_file) + + print( + "Generated human-readable info for {} in {}".format( + profdata_output, llvm_cov_output + ) + ) + + # Process coverage data for specific files + parent_directory = os.path.abspath( + os.path.join(os.getcwd(), "..") + ) # Get the parent directory of the current working directory + + for file, cpp_file in zip(files, cpp_files): + output1_file = os.path.join(parent_directory, f"{cpp_file}") + escaped_cpp_file = output1_file.replace( + "/", r"\/" + ) # Manually escape slashes + output_file = ( + os.path.splitext(llvm_cov_output)[0] + + f"_{cpp_file.replace('/', '_')}.txt" + ) + + awk_command = f"awk '/^{escaped_cpp_file}/ {{ print; getline; while ($0 != \"\") {{ print; getline }} }}' {llvm_cov_output} > {output_file}" + + # Run the constructed awk command + subprocess.run(awk_command, shell=True, check=True) + + print("Processed file saved as:", output_file) + print() + coverage_files.append(output_file) + + print("Conversion of profraw files to human-readable form is completed.") + print("List of coverage files:", coverage_files) + print() + + return coverage_files + + except subprocess.CalledProcessError as e: + print("Error during profraw to profdata conversion:", e) + sys.exit(1) + + +def report_covered_and_uncovered_lines(llvm_cov_patch, modified_lines): + """ "Report Covered and uncovered source code lines.""" + try: + # Keep track of uncovered lines not tested by any of the added test case + uncovered_lines_not_tested = set() + + for coverage_file in llvm_cov_patch: + if coverage_file.startswith("./default"): + continue + print(f"Coverage File: {coverage_file}") + + # Create a set to store covered line numbers + covered_line_numbers = set() + uncovered_line_numbers = set() + + with open(coverage_file, "r") as cov_file: + for line in cov_file: + parts = line.strip().split("|") + if len(parts) >= 3: + line_number_str = parts[0].strip() + execution_count = parts[1].strip() + + if line_number_str.isdigit() and execution_count.isdigit(): + line_number = int(line_number_str) + if int(execution_count) > 0: + covered_line_numbers.add(line_number) + elif int(execution_count) == 0: + uncovered_line_numbers.add(line_number) + + for file, lines in modified_lines.items(): + for line_number_source, line_content in lines: + if line_number_source in covered_line_numbers: + print( + f"Covered Line: {line_number_source} : {line_content.strip()}" + ) + + print("=" * 40) + + for file, lines in modified_lines.items(): + for line_number_source, line_content in lines: + if line_number_source in uncovered_line_numbers: + print( + f"Uncovered Line: {line_number_source} : {line_content.strip()}" + ) + # uncovered_lines_not_tested.add(code_line) + uncovered_lines_not_tested.add( + (line_number_source, line_content.strip()) + ) + + print("=" * 40) + + if uncovered_lines_not_tested: + print() + print("Uncovered lines not tested by any test case:") + print("=" * 40) + for line_number, line in uncovered_lines_not_tested: + print(f"Line {line_number}: {line}") + sys.exit(1) + + except Exception as ex: + print("Error while reporting covered and uncovered lines:", ex) + sys.exit(1) + + +def parse_suite_info(s): + """ " Function that takes a string 's' as input and return res + dictionary containing suite information.""" + + curr_suite = None + res = {} + + # Iterate over each line in the decoded 's' split by lines. + for line in s.decode().splitlines(): + # Calculate the number of leading spaces in the line. + leading_spaces = len(line) - len(line.lstrip(" ")) + + # Check if there are 2 leading spaces, indicating a suite name. + if leading_spaces == 2: + # Extract the suite name and assign it to curr_suite. + curr_suite = line.split()[0] + # Check if curr_suite is not None and there are 4 leading spaces, and "Source Root:" is in the line. + elif curr_suite is not None and leading_spaces == 4 and "Source Root:" in line: + # Ensure that curr_suite is not already in the res dictionary. + assert(curr_suite not in res) + + # Add the suite name as a key and the last part of the line as its value in the res dictionary. + res[curr_suite] = line.split()[-1] + + # Return the res dictionary containing suite information. + return res + + +def find_lit_tests(lit_path, test_paths): + """ "Function that takes lit_path and test_paths as inputs and + return a list of test paths.""" + + # Create a command list for listing test suites using lit_path and test_paths. + suites_cmd = [lit_path, "--show-suites"] + test_paths + output = subprocess.check_output(suites_cmd) + + # Parse the output to extract suite information using parse_suite_info function. + test_suites = parse_suite_info(output) + + # Create a command list for listing individual tests using lit_path and test_paths. + tests_cmd = [lit_path, "--show-tests"] + test_paths + output = subprocess.check_output(tests_cmd) + + # Convert the output lines to strings and split them. + lines = [line.decode() for line in output.splitlines()] + + # Extract test information from lines where "::" is present. + test_info = [line.split() for line in lines if "::" in line] + + # Construct a list of test paths by combining suite source roots and test case names. + return [os.path.join(test_suites[suite], test_case) for (suite, sep, test_case) in test_info] + + +def parse_args(): + """ "Function to parse command line arguments.""" + + # Create an ArgumentParser object. + parser = argparse.ArgumentParser() + + # Add optional argument "-b" or "--build-dir" with a default value of "build". + parser.add_argument("-b", "--build-dir", dest="build_dir", default="build") + + # Add positional argument "test_path" which can have one or more values. + parser.add_argument("test_path", nargs="+") + + # Parse the command line arguments. + args = parser.parse_args() + + # Return a tuple containing build_dir and test_path. + return (args.build_dir, args.test_path) + + +def main(): + (build_dir, test_paths) = parse_args() + + if not os.path.exists(build_dir): + os.makedirs(build_dir) + + patch_path = os.path.join(build_dir, "patch.diff") # Path to the patch file (output) + + create_patch_from_last_commit(patch_path) + + llvm_source_dir = "../llvm" # Path to LLVM source directory + build_llvm(build_dir, llvm_source_dir) + + llvm_lit_path = os.path.join(build_dir, "bin/llvm-lit") # Path to llvm-lit executable + tests = frozenset(find_lit_tests(llvm_lit_path, test_paths)) + run_modified_added_tests(llvm_lit_path, patch_path, tests) + + modified_lines = extract_modified_lines_from_patch(patch_path, tests) + for file, lines in modified_lines.items(): + print(f"File: {file}") + for line_number, line_content in lines: + print(f"Line {line_number}: {line_content}") + print("=" * 40) + print() + + source_files = extract_source_files_from_patch(patch_path) + + llvm_cov_patch = convert_profraw_to_profdata_patch(source_files, build_dir) + + print("Coverage Files List:") + for coverage_file in llvm_cov_patch: + print(coverage_file) + print() + + report_covered_and_uncovered_lines(llvm_cov_patch, modified_lines) + + +if __name__ == "__main__": + main() +