[analyzer] SATest: Move from csv to json project maps

[vsavchenko]

Summary

JSON format is a bit more verbose and easier to reason about
and extend. For this reason, before extending SATestBuild
functionality it is better to refactor the part of how we
configure the whole system.

[NoQ]

Sounds like a nice abstraction :)

clang/utils/analyzer/ProjectMap.py

+import json
+import os
+
+from typing import Any, Dict, List, NamedTuple, Optional
+
+
+JSON = Dict[str, Any]
+
+
+DEFAULT_MAP_FILE = "projects.json"
+
+
+class ProjectInfo(NamedTuple):
+    """
+    Information about a project to analyze.
+    """
+    name: str
+    mode: int
+    enabled: bool = True
+
+
+class ProjectMap:
+    """
+    Project map stores info about all the "registered" projects.
+    """
+    def __init__(self, path: Optional[str] = None, should_exist: bool = True):
+        """
+        :param path: optional path to a project JSON file, when None defaults
+                     to DEFAULT_MAP_FILE.
+        :param should_exist: flag to tell if it's an exceptional situation when
+                             the project file doesn't exist, creates an empty
+                             project list instead if we are not expecting it to
+                             exist.
+        """
+        if path is None:
+            path = os.path.join(os.path.abspath(os.curdir), DEFAULT_MAP_FILE)
+
+        if not os.path.exists(path):
+            if should_exist:
+                raise ValueError(
+                    f"Cannot find the project map file {path}"
+                    f"\nRunning script for the wrong directory?\n")
+            else:
+                self._create_empty(path)
+
+        self.path = path
+        self._load_projects()
+
+    def save(self):
+        """
+        Save project map back to its original file.
+        """
+        self._save(self.projects, self.path)
+
+    def _load_projects(self):
+        with open(self.path) as raw_data:
+            raw_projects = json.load(raw_data)
+
+            if not isinstance(raw_projects, list):
+                raise ValueError(
+                    "Project map should be a list of JSON objects")
+
+            self.projects = self._parse(raw_projects)
+
+    @staticmethod
+    def _parse(raw_projects: List[JSON]) -> List[ProjectInfo]:
+        return [ProjectMap._parse_project(raw_project)
+                for raw_project in raw_projects]
+
+    @staticmethod
+    def _parse_project(raw_project: JSON) -> ProjectInfo:
+        try:
+            name: str = raw_project["name"]
+            build_mode: int = raw_project["mode"]
+            enabled: bool = raw_project.get("enabled", True)
+            return ProjectInfo(name, build_mode, enabled)
+
+        except KeyError as e:
+            raise ValueError(
+                f"Project info is required to have a '{e.args[0]}' field")
+
+    @staticmethod
+    def _create_empty(path: str):
+        ProjectMap._save([], path)
+
+    @staticmethod
+    def _save(projects: List[ProjectInfo], path: str):
+        with open(path, "w") as output:
+            json.dump(ProjectMap._convert_infos_to_dicts(projects),
+                      output, indent=2)
+
+    @staticmethod
+    def _convert_infos_to_dicts(projects: List[ProjectInfo]) -> List[JSON]:
+        return [project._asdict() for project in projects]

clang/utils/analyzer/SATestAdd.py

The project should use the following files for set up:
                                      example, PatchedSource) and make any
                                      needed modifications to the copied
                                      source.
                                      Then run:
                                           diff -ur CachedSource PatchedSource \
                                               > changes_for_analyzer.patch
 """
 import SATestBuild
+from ProjectMap import ProjectMap, ProjectInfo
 
-import csv
 import os
 import sys
 
-from typing import IO
-
 
 def add_new_project(name: str, build_mode: int):
     """
     Add a new project for testing: build it and add to the Project Map file.
     :param name: is a short string used to identify a project.
     """
 
-    project_info = SATestBuild.ProjectInfo(name, build_mode,
-                                           is_reference_build=True)
-    tester = SATestBuild.ProjectTester(project_info)
+    project_info = ProjectInfo(name, build_mode)
+    test_info = SATestBuild.TestInfo(project_info,
+    tester = SATestBuild.ProjectTester(test_info)
 
     project_dir = tester.get_project_dir()
     if not os.path.exists(project_dir):
         print(f"Error: Project directory is missing: {project_dir}")
         sys.exit(-1)
 
     # Build the project.
     tester.test()
 
     # Add the project name to the project map.
-    project_map_path = SATestBuild.get_project_map_path(should_exist=False)
-
-    if os.path.exists(project_map_path):
-        file_mode = "r+"
-    else:
-        print("Warning: Creating the project map file!")
-        file_mode = "w+"
+    project_map = ProjectMap(should_exist=False)
 
-    with open(project_map_path, file_mode) as map_file:
-        if is_existing_project(map_file, name):
-            print(f"Warning: Project with name '{name}' already exists.",
-                  file=sys.stdout)
-            print("Reference output has been regenerated.", file=sys.stdout)
-        else:
-            map_writer = csv.writer(map_file)
-            map_writer.writerow((name, build_mode))
-            print(f"The project map is updated: {project_map_path}")
-
-
-def is_existing_project(map_file: IO, project_name: str) -> bool:
-    map_reader = csv.reader(map_file)
+    if is_existing_project(project_map, name):
+        project_map.projects.append(project_info)
+        project_map.save()
 
-    for raw_info in map_reader:
-        if project_name == raw_info[0]:
-            return True
 
-    return False
+def is_existing_project(project_map: ProjectMap, project_name: str) -> bool:
+    return any(existing_project.name == project_name
+               for existing_project in project_map.projects)
 
 
 # TODO: Use argparse
 # TODO: Add an option not to build.
 # TODO: Set the path to the Repository directory.
 if __name__ == "__main__":
     if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
         print("Add a new project for testing to the analyzer"

clang/utils/analyzer/SATestBuild.py

For more logging, set the  env variables:
    zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
 
 The list of checkers tested are hardcoded in the Checkers variable.
 For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment
 variable. It should contain a comma separated list.
 """
 import CmpRuns
 import SATestUtils
+from ProjectMap import ProjectInfo, ProjectMap
 
 import argparse
-import csv
 import glob
 import logging
 import math
 import multiprocessing
 import os
 import plistlib
 import shutil
 import sys
 import threading
 import time
 
 from queue import Queue
+# mypy has problems finding InvalidFileException in the module
+# and this is we can shush that false positive
+from plistlib import InvalidFileException  # type:ignore
 from subprocess import CalledProcessError, check_call
-from typing import (cast, Dict, Iterable, IO, List, NamedTuple, Optional,
-                    Tuple, TYPE_CHECKING)
+from typing import Dict, IO, List, NamedTuple, Optional, TYPE_CHECKING
 
 
 ###############################################################################
 # Helper functions.
 ###############################################################################
 
 LOCAL = threading.local()
 LOCAL.stdout = sys.stdout
if not cc_candidate:
     stderr("Error: cannot find 'clang' in PATH")
     sys.exit(1)
 
 CLANG = cc_candidate
 
 # Number of jobs.
 MAX_JOBS = int(math.ceil(multiprocessing.cpu_count() * 0.75))
 
-# Project map stores info about all the "registered" projects.
-PROJECT_MAP_FILE = "projectMap.csv"
-
 # Names of the project specific scripts.
 # The script that downloads the project.
 DOWNLOAD_SCRIPT = "download_project.sh"
 # The script that needs to be executed before the build can start.
 CLEANUP_SCRIPT = "cleanup_run_static_analyzer.sh"
 # This is a file containing commands for scan-build.
 BUILD_SCRIPT = "run_static_analyzer.cmd"
 
def fileno(self) -> int:
         return 0
 
 
 ###############################################################################
 # Test harness logic.
 ###############################################################################
 
 
-def get_project_map_path(should_exist: bool = True) -> str:
-    project_map_path = os.path.join(os.path.abspath(os.curdir),
-                                    PROJECT_MAP_FILE)
-
-    if should_exist and not os.path.exists(project_map_path):
-        stderr(f"Error: Cannot find the project map file {project_map_path}"
-               f"\nRunning script for the wrong directory?\n")
-        sys.exit(1)
-
-    return project_map_path
-
-
 def run_cleanup_script(directory: str, build_log_file: IO):
     """
     Run pre-processing script if any.
     """
     cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
     script_path = os.path.join(directory, CLEANUP_SCRIPT)
 
     SATestUtils.run_script(script_path, build_log_file, cwd,
try:
                    shell=True)
 
     except CalledProcessError:
         stderr(f"Error: Patch failed. "
                f"See {build_log_file.name} for details.\n")
         sys.exit(1)
 
 
-class ProjectInfo(NamedTuple):
+class TestInfo(NamedTuple):
     """
     Information about a project and settings for its analysis.
     """
-    name: str
-    build_mode: int
+    project: ProjectInfo
     override_compiler: bool = False
     extra_analyzer_config: str = ""
     is_reference_build: bool = False
     strictness: int = 0
 
 
 # typing package doesn't have a separate type for Queue, but has a generic stub
 # We still want to have a type-safe checked project queue, for this reason,
 # we specify generic type for mypy.
 #
 # It is a common workaround for this situation:
 # https://mypy.readthedocs.io/en/stable/common_issues.html#using-classes-that-are-generic-in-stubs-but-not-at-runtime
 if TYPE_CHECKING:
-    ProjectQueue = Queue[ProjectInfo]  # this is only processed by mypy
+    TestQueue = Queue[TestInfo]  # this is only processed by mypy
 else:
-    ProjectQueue = Queue  # this will be executed at runtime
+    TestQueue = Queue  # this will be executed at runtime
 
 
 class RegressionTester:
     """
     A component aggregating all of the project testing.
     """
     def __init__(self, jobs: int, override_compiler: bool,
                  extra_analyzer_config: str, regenerate: bool,
                  strictness: bool):
         self.jobs = jobs
         self.override_compiler = override_compiler
         self.extra_analyzer_config = extra_analyzer_config
         self.regenerate = regenerate
         self.strictness = strictness
 
     def test_all(self) -> bool:
-        projects_to_test: List[ProjectInfo] = []
+        projects_to_test: List[TestInfo] = []
 
-        with open(get_project_map_path(), "r") as map_file:
-            validate_project_file(map_file)
+        project_map = ProjectMap()
 
-            # Test the projects.
-            for proj_name, proj_build_mode in get_projects(map_file):
-                projects_to_test.append(
-                    ProjectInfo(proj_name, int(proj_build_mode),
-                                self.override_compiler,
-                                self.extra_analyzer_config,
-                                self.regenerate, self.strictness))
+        for project in project_map.projects:
+                TestInfo(project,
         if self.jobs <= 1:
             return self._single_threaded_test_all(projects_to_test)
         else:
             return self._multi_threaded_test_all(projects_to_test)
 
     def _single_threaded_test_all(self,
-                                  projects_to_test: List[ProjectInfo]) -> bool:
+                                  projects_to_test: List[TestInfo]) -> bool:
         """
         Run all projects.
         :return: whether tests have passed.
         """
         success = True
         for project_info in projects_to_test:
             tester = ProjectTester(project_info)
             success &= tester.test()
         return success
 
     def _multi_threaded_test_all(self,
-                                 projects_to_test: List[ProjectInfo]) -> bool:
+                                 projects_to_test: List[TestInfo]) -> bool:
         """
         Run each project in a separate thread.
 
         This is OK despite GIL, as testing is blocked
         on launching external processes.
 
         :return: whether tests have passed.
         """
-        tasks_queue = ProjectQueue()
+        tasks_queue = TestQueue()
 
         for project_info in projects_to_test:
             tasks_queue.put(project_info)
 
         results_differ = threading.Event()
         failure_flag = threading.Event()
 
         for _ in range(self.jobs):
             T = TestProjectThread(tasks_queue, results_differ, failure_flag)
             T.start()
 
         # Required to handle Ctrl-C gracefully.
         while tasks_queue.unfinished_tasks:
             time.sleep(0.1)  # Seconds.
             if failure_flag.is_set():
                 stderr("Test runner crashed\n")
                 sys.exit(1)
         return not results_differ.is_set()
 
 
 class ProjectTester:
     """
     A component aggregating testing for one project.
     """
-    def __init__(self, project_info: ProjectInfo):
-        self.project_name = project_info.name
-        self.build_mode = project_info.build_mode
-        self.override_compiler = project_info.override_compiler
-        self.extra_analyzer_config = project_info.extra_analyzer_config
-        self.is_reference_build = project_info.is_reference_build
-        self.strictness = project_info.strictness
+    def __init__(self, test_info: TestInfo):
+        self.project = test_info.project
+        self.override_compiler = test_info.override_compiler
+        self.extra_analyzer_config = test_info.extra_analyzer_config
+        self.is_reference_build = test_info.is_reference_build
+        self.strictness = test_info.strictness
 
     def test(self) -> bool:
         """
         Test a given project.
         :return tests_passed: Whether tests have passed according
         to the :param strictness: criteria.
         """
-        stdout(f" \n\n--- Building project {self.project_name}\n")
+        if not self.project.enabled:
+            stdout(f" \n\n--- Skipping disabled project {self.project.name}\n")
+            return True
+
+        stdout(f" \n\n--- Building project {self.project.name}\n")
 
         start_time = time.time()
 
         project_dir = self.get_project_dir()
         if VERBOSE >= 1:
             stdout(f"  Build directory: {project_dir}.\n")
 
         # Set the build results directory.
         output_dir = self.get_output_dir()
         output_dir = os.path.join(project_dir, output_dir)
 
         self.build(project_dir, output_dir)
         check_build(output_dir)
 
         if self.is_reference_build:
             cleanup_reference_results(output_dir)
             passed = True
         else:
             passed = run_cmp_results(project_dir, self.strictness)
 
-        stdout(f"Completed tests for project {self.project_name} "
+        stdout(f"Completed tests for project {self.project.name} "
                f"(time: {time.time() - start_time:.2f}).\n")
 
         return passed
 
     def get_project_dir(self) -> str:
-        return os.path.join(os.path.abspath(os.curdir), self.project_name)
+        return os.path.join(os.path.abspath(os.curdir), self.project.name)
 
     def get_output_dir(self) -> str:
         if self.is_reference_build:
             return REF_PREFIX + OUTPUT_DIR_NAME
         else:
             return OUTPUT_DIR_NAME
 
     def build(self, directory: str, output_dir: str):
def build(self, directory: str, output_dir: str):
 
             shutil.rmtree(output_dir)
 
         assert(not os.path.exists(output_dir))
         os.makedirs(os.path.join(output_dir, LOG_DIR_NAME))
 
         # Build and analyze the project.
         with open(build_log_path, "w+") as build_log_file:
-            if self.build_mode == 1:
+            if self.project.mode == 1:
                 download_and_patch(directory, build_log_file)
                 run_cleanup_script(directory, build_log_file)
                 self.scan_build(directory, output_dir, build_log_file)
             else:
                 self.analyze_preprocessed(directory, output_dir)
 
             if self.is_reference_build:
                 run_cleanup_script(directory, build_log_file)
                 normalize_reference_results(directory, output_dir,
-                                            self.build_mode)
+                                            self.project.mode)
 
         stdout(f"Build complete (time: {time.time() - time_start:.2f}). "
                f"See the log for more details: {build_log_path}\n")
 
     def scan_build(self, directory: str, output_dir: str, build_log_file: IO):
         """
         Build the project with scan-build by reading in the commands and
         prefixing them with the scan-build options.
def analyze_preprocessed(self, directory: str, output_dir: str):
         prefix = CLANG + " --analyze "
 
         prefix += "--analyzer-output plist "
         prefix += " -Xclang -analyzer-checker=" + CHECKERS
         prefix += " -fcxx-exceptions -fblocks "
         prefix += " -Xclang -analyzer-config "
         prefix += f"-Xclang {self.generate_config()} "
 
-        if self.build_mode == 2:
+        if self.project.mode == 2:
             prefix += "-std=c++11 "
 
         plist_path = os.path.join(directory, output_dir, "date")
         fail_path = os.path.join(plist_path, "failures")
         os.makedirs(fail_path)
 
         for full_file_name in glob.glob(directory + "/*"):
             file_name = os.path.basename(full_file_name)
def generate_config(self) -> str:
 
         if self.extra_analyzer_config:
             out += "," + self.extra_analyzer_config
 
         return out
 
 
 class TestProjectThread(threading.Thread):
-    def __init__(self, tasks_queue: ProjectQueue,
+    def __init__(self, tasks_queue: TestQueue,
                  results_differ: threading.Event,
                  failure_flag: threading.Event):
         """
         :param results_differ: Used to signify that results differ from
                the canonical ones.
         :param failure_flag: Used to signify a failure during the run.
         """
         self.args = args
         self.tasks_queue = tasks_queue
         self.results_differ = results_differ
         self.failure_flag = failure_flag
         super().__init__()
 
         # Needed to gracefully handle interrupts with Ctrl-C
         self.daemon = True
 
     def run(self):
         while not self.tasks_queue.empty():
             try:
-                project_info = self.tasks_queue.get()
+                test_info = self.tasks_queue.get()
 
-                Logger = logging.getLogger(project_info.name)
+                Logger = logging.getLogger(test_info.project.name)
                 LOCAL.stdout = StreamToLogger(Logger, logging.INFO)
                 LOCAL.stderr = StreamToLogger(Logger, logging.ERROR)
 
-                tester = ProjectTester(project_info)
+                tester = ProjectTester(test_info)
                 if not tester.test():
                     self.results_differ.set()
 
                 self.tasks_queue.task_done()
 
             except BaseException:
                 self.failure_flag.set()
                 raise
for plist in glob.glob(output_dir + "/*/*.plist"):
         try:
             with open(plist, "rb") as plist_file:
                 data = plistlib.load(plist_file)
             # Delete empty reports.
             if not data['files']:
                 os.remove(plist)
                 continue
 
-        except plistlib.InvalidFileException as e:
+        except InvalidFileException as e:
             stderr(f"Error parsing plist file {plist}: {str(e)}")
             continue
 
 
 def clean_up_empty_folders(output_dir: str):
     """
     Remove empty folders from results, as git would not store them.
     """
     subdirs = glob.glob(output_dir + "/*")
     for subdir in subdirs:
         if not os.listdir(subdir):
             os.removedirs(subdir)
 
 
-def get_projects(map_file: IO) -> Iterable[Tuple[str, str]]:
-    """
-    Iterate over all projects defined in the project file handler `map_file`
-    from the start.
-    """
-    map_file.seek(0)
-    # TODO: csv format is not very readable, change it to JSON
-    for project_info in csv.reader(map_file):
-        if SATestUtils.is_comment_csv_line(project_info):
-            continue
-        # suppress mypy error
-        yield cast(Tuple[str, str], project_info)
-
-
-def validate_project_file(map_file: IO):
-    """
-    Validate project file.
-    """
-    for project_info in get_projects(map_file):
-        if len(project_info) != 2:
-            stderr("Error: Rows in the project map file "
-                   "should have 2 entries.")
-            raise Exception()
-
-        if project_info[1] not in ('0', '1', '2'):
-            stderr("Error: Second entry in the project map file should be 0"
-                   " (single file), 1 (project), or 2(single file c++11).")
-            raise Exception()
-
-
 if __name__ == "__main__":
     # Parse command line arguments.
     parser = argparse.ArgumentParser(
         description="Test the Clang Static Analyzer.")
 
     parser.add_argument("--strictness", dest="strictness", type=int, default=0,
                         help="0 to fail on runtime errors, 1 to fail when the "
                         "number of found bugs are different from the "

clang/utils/analyzer/SATestUpdateDiffs.py

 #!/usr/bin/env python
 
 """
 Update reference results for static analyzer.
 """
 import SATestBuild
+from ProjectMap import ProjectInfo, ProjectMap
 
 import os
 import shutil
 import sys
 
 from subprocess import check_call
 
 Verbose = 0
 
 
-def update_reference_results(project_name: str, build_mode: int):
-    project_info = SATestBuild.ProjectInfo(project_name, build_mode)
-    tester = SATestBuild.ProjectTester(project_info)
+def update_reference_results(project: ProjectInfo):
+    test_info = SATestBuild.TestInfo(project)
+    tester = SATestBuild.ProjectTester(test_info)
     project_dir = tester.get_project_dir()
 
     tester.is_reference_build = True
     ref_results_path = os.path.join(project_dir, tester.get_output_dir())
 
     tester.is_reference_build = False
     created_results_path = os.path.join(project_dir, tester.get_output_dir())
 
with open(build_log_path, "w+") as build_log_file:
 
         # Replace reference results with a freshly computed once.
         shutil.copytree(created_results_path, ref_results_path, symlinks=True)
 
         # Run cleanup script.
         SATestBuild.run_cleanup_script(project_dir, build_log_file)
 
         SATestBuild.normalize_reference_results(
-            project_dir, ref_results_path, build_mode)
+            project_dir, ref_results_path, project.mode)
 
         # Clean up the generated difference results.
         SATestBuild.cleanup_reference_results(ref_results_path)
 
         run_cmd(f"git add '{ref_results_path}'")
 
 
+# TODO: use argparse
 def main(argv):
     if len(argv) == 2 and argv[1] in ("-h", "--help"):
         print("Update static analyzer reference results based "
               "\non the previous run of SATestBuild.py.\n"
               "\nN.B.: Assumes that SATestBuild.py was just run",
               file=sys.stderr)
         sys.exit(1)
 
-    with open(SATestBuild.get_project_map_path(), "r") as f:
-        for project_name, build_mode in SATestBuild.get_projects(f):
-            update_reference_results(project_name, int(build_mode))
+    project_map = ProjectMap()
+    for project in project_map.projects:
+        update_reference_results(project)
 
 
 if __name__ == '__main__':
     main(sys.argv)