diff --git a/llvm/utils/git-check-coverage b/llvm/utils/git-check-coverage new file mode 100755 --- /dev/null +++ b/llvm/utils/git-check-coverage @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +# +# ===- git-check-coverage - CheckCoverage Git Integration ---------*- python -*--===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===------------------------------------------------------------------------===# + +r""" +code-coverage git integration +============================ + +This file provides a code-coverage integration for git. Put it somewhere in your +path and ensure that it is executable. Then, code coverage information will be +emit for a specific commit. + +For further details, run: +git code-coverage -h + +Requires Python 2.7 or Python 3 +""" + +import os +import subprocess +import re + + +def create_patch_from_last_commit(output_path): + """Create a patch file from the last commit in the Git repository.""" + try: + # Get the hash of the last commit using `git rev-parse` command + git_hash_cmd = ["git", "rev-parse", "HEAD"] + commit_hash = subprocess.check_output(git_hash_cmd).strip().decode("utf-8") + + # Create the patch from the last commit using `git show` command + patch_cmd = ["git", "show", commit_hash] + patch_output = subprocess.check_output(patch_cmd).decode("utf-8") + + # Write the patch to the output file + with open(output_path, "w") as patch_file: + patch_file.write(patch_output) + + print("Patch file '{}' created successfully.".format(output_path)) + + except subprocess.CalledProcessError as e: + print("Error while creating the patch from the last commit:", e) + + +def extract_source_files_from_patch(patch_path): + """ "read the patch file and extract the names of .cpp and .h files that have been modified or added in the patch""" + patch_path = "patch.diff" # Path to the patch file (output) + try: + source_files = [] + with open(patch_path, "r") as patch_file: + patch_diff = patch_file.read() + + # Use regular expression to find .cpp files in the patch + source_file_matches = re.findall(r"\+{3} b/(\S+\.(?:cpp|h))", patch_diff) + + source_files.extend(source_file_matches) + + for source_file in source_files: + print(source_file) + return source_files + + except Exception as ex: + print("Error while extracting .cpp files from patch:", ex) + return [] + + +def extract_modified_lines_from_patch(patch_path): + """ "Extract the modified source lines from the patch.""" + patch_path = "patch.diff" # Path to the patch file (output) + source_lines = {} # Dictionary for added lines in source code files + testcase_lines = {} # Dictionary for added lines in test case files + current_file = None # Stores the current file being processed + + # Read the content of the patch file + with open(patch_path, "r") as patch_file: + patch_diff = patch_file.read() + + # Iterate through each line in the patch diff + for line in patch_diff.split("\n"): + # Check for the line indicating a file change + if line.startswith("+++ "): + current_file = line[ + 6: + ].strip() # Extract the filename, leaving first six characters + # Categorize the current file into source or test case category + if current_file.endswith(".cpp") or current_file.endswith(".h"): + source_lines[current_file] = [] + elif current_file.endswith(".ll"): + testcase_lines[current_file] = [] + # Check if the line is an added line + elif ( + current_file + and line.startswith("+") + and not line.startswith("+++") + and not line.startswith("++++") + ): + # Append the added line content to the appropriate category + if current_file.endswith(".cpp") or current_file.endswith(".h"): + source_lines[current_file].append(line[1:]) + elif current_file.endswith(".ll"): + testcase_lines[current_file].append(line[1:]) + + # Return the categorized dictionaries + return source_lines, testcase_lines + + +def build_llvm(build_dir, llvm_source_dir): + """ "Configure and build LLVM in the specified build directory.""" + try: + # Change to the build directory + os.chdir(build_dir) + + # Run the cmake command to configure LLVM build + cmake_command = [ + "cmake", + "-G", + "Ninja", + llvm_source_dir, + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_LINKER=lld", + "-DCMAKE_C_COMPILER=/usr/bin/clang", + "-DCMAKE_CXX_COMPILER=/usr/bin/clang++", + "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON", + "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON", + ] + subprocess.check_call(cmake_command) + + # Run the ninja build command + subprocess.check_call(["ninja"]) + + print("LLVM build completed successfully.") + + except subprocess.CalledProcessError as e: + print("Error during LLVM build:", e) + + +def run_single_test_with_coverage(llvm_lit_path, test_path): + """Run a single test case using llvm-lit with coverage.""" + try: + lit_cmd = [llvm_lit_path, "--per-test-coverage", "../" + test_path] + subprocess.check_call(lit_cmd) + + print("Test case executed:", test_path) + + except subprocess.CalledProcessError as e: + print("Error while running test:", e) + except Exception as ex: + print("Error:", ex) + + +def run_modified_added_tests(llvm_lit_path, build_dir): + """ "Read the patch file, identify modified and added test cases, and then execute each of these test cases""" + patch_path = "patch.diff" # Path to the patch file (output) + + try: + # Get the list of modified and added test cases from the patch + with open(patch_path, "r") as patch_file: + patch_diff = patch_file.read() + + modified_tests = [] + added_tests = [] + + # Use regular expressions to find modified and added test cases with ".ll" extension + for match in re.finditer(r"^\+\+\+ [ab]/(.*\.ll)$", patch_diff, re.MULTILINE): + test_file = match.group(1) + if "test" in test_file: + if match.group(0).startswith("+++"): + added_tests.append(test_file) + else: + modified_tests.append(test_file) + + if not modified_tests and not added_tests: + print("No modified or added tests found in the patch.") + return + + # Run each modified test case + print("Running modified test cases:") + for test_file in modified_tests: + run_single_test_with_coverage(llvm_lit_path, test_file) + + # Run each newly added test case + print("Running added test cases:") + for test_file in added_tests: + run_single_test_with_coverage(llvm_lit_path, test_file) + + except subprocess.CalledProcessError as e: + print("Error while running modified and added tests:", e) + except Exception as ex: + print("Error:", ex) + + +def convert_profraw_to_profdata_patch(build_dir, cpp_files): + """ "Convert profraw coverage data files to profdata format, generate human-readable + coverage information, and extract coverage data for specific files.""" + # Create a list to store the paths of the generated coverage data files + coverage_files = [] + try: + # Change to the build directory + for root, dirs, files in os.walk("."): + for file in files: + if file.endswith(".profraw"): + profraw_file = os.path.join(root, file) + profdata_output = os.path.splitext(profraw_file)[0] + ".profdata" + + print(profraw_file) + print(profdata_output) + + # Get the current working directory + current_directory = os.getcwd() + print("Current Working Directory:", current_directory) + + # Construct the llvm-profdata command + llvm_profdata_cmd = [ + "./bin/llvm-profdata", + "merge", + "-o", + profdata_output, + profraw_file, + ] + + # Run llvm-profdata to convert profraw to profdata + subprocess.check_call(llvm_profdata_cmd) + + print("Converted {} to {}".format(profraw_file, profdata_output)) + + # Construct the llvm-cov show command + llvm_cov_cmd = [ + "bin/llvm-cov", + "show", + "-instr-profile", + profdata_output, + "bin/opt", + ] + llvm_cov_output = ( + os.path.splitext(profdata_output)[0] + "_before" + ".txt" + ) + + # Redirect the output of llvm-cov show to a text file + with open(llvm_cov_output, "w") as output_file: + subprocess.check_call(llvm_cov_cmd, stdout=output_file) + + print( + "Generated human-readable info for {} in {}".format( + profdata_output, llvm_cov_output + ) + ) + + # Process coverage data for specific files + parent_directory = os.path.abspath( + os.path.join(os.getcwd(), "..") + ) # Get the parent directory of the current working directory + + + for file, cpp_file in zip(files, cpp_files): + output1_file = os.path.join(parent_directory, f"{cpp_file}") + escaped_cpp_file = output1_file.replace( + "/", r"\/" + ) # Manually escape slashes + output_file = ( + os.path.splitext(llvm_cov_output)[0] + + f"_{cpp_file.replace('/', '_')}.txt" + ) + + awk_command = f"awk '/^{escaped_cpp_file}/ {{ print; getline; while ($0 != \"\") {{ print; getline }} }}' {llvm_cov_output} > {output_file}" + + # Run the constructed awk command + subprocess.run(awk_command, shell=True, check=True) + + print("Processed file saved as:", output_file) + + coverage_files.append(output_file) + + print("Conversion of profraw files to human-readable form is completed.") + print("List of coverage files:", coverage_files) + + return coverage_files + + except subprocess.CalledProcessError as e: + print("Error during profraw to profdata conversion:", e) + + +def report_inadequate_testing(source_lines, llvm_cov_patch): + """Report inadequate testing for modified source lines based on coverage data.""" + try: + # Initialize an empty dictionary to store inadequately tested files and lines + inadequately_tested_files = {} + + # Iterate through each source file and its modified lines + for source_file, modified_lines in source_lines.items(): + # Check if the source file is present in the llvm_cov_patch dictionary + if source_file in llvm_cov_patch: + # Get the coverage file corresponding to the source file + coverage_file = llvm_cov_patch[source_file] + covered_lines = set() + + # Open the coverage file and process each line + with open(coverage_file, "r") as cov_file: + for line in cov_file: + parts = line.strip().split(":") + if len(parts) >= 3 and parts[0].isdigit(): + # Extract line number and execution count from coverage data + line_number = int(parts[0]) + execution_count = int(parts[1]) + if execution_count > 0: + # Add line number to the set if it was covered + covered_lines.add(line_number) + + # Find lines from the modified_lines that are not covered + inadequately_covered_lines = [ + line for line in modified_lines if line not in covered_lines + ] + + # If there are inadequately covered lines, add them to the dictionary + if inadequately_covered_lines: + inadequately_tested_files[source_file] = inadequately_covered_lines + + # Print the inadequately tested lines if any, else print a message + if inadequately_tested_files: + print("Inadequately tested lines:") + for file, lines in inadequately_tested_files.items(): + print(f"File: {file}") + print("Lines:") + for line in lines: + print(f" - Line {line}") + print("=" * 40) + else: + print("All modified lines are adequately covered by tests.") + + except Exception as ex: + print("Error while reporting inadequate testing:", ex) + + +if __name__ == "__main__": + build_dir = "build" # Path to LLVM build directory + if not os.path.exists(build_dir): + os.makedirs(build_dir) + + patch_path = "build/patch.diff" # Path to the patch file (output) + + repo_path = ( + os.getcwd() + ) # Assuming the script is run from the root of the Git repository + + create_patch_from_last_commit(patch_path) + + llvm_source_dir = "../llvm" # Path to LLVM source directory + build_llvm(build_dir, llvm_source_dir) + + llvm_lit_path = "bin/llvm-lit" # Path to llvm-lit executable + run_modified_added_tests(llvm_lit_path, build_dir) + + source_lines, testcase_lines = extract_modified_lines_from_patch(patch_path) + + source_files = extract_source_files_from_patch(patch_path) + + llvm_cov_patch = convert_profraw_to_profdata_patch( + build_dir, source_files + ) + + print("Coverage Files List:") + for coverage_file in llvm_cov_patch: + print(coverage_file) + + report_inadequate_testing(source_lines, llvm_cov_patch)