[analyzer] SATest: Add posibility to download source from git and zip

[NoQ]

*keeps ranting that there are no lit tests so it's much harder to figure out what the code is even supposed to do as there are no examples*

[xazax.hun]

clang/utils/analyzer/ProjectMap.py

14

I was wondering what is the point of inheriting from str. I am not well-versed in Python so it is more of a curiosity.

[vsavchenko]

clang/utils/analyzer/ProjectMap.py

14

It helps with direct assignments from strings, cross-comparisons between Enum values and strings, and most importantly - serialization.

clang/utils/analyzer/ProjectMap.py

 import json
 import os
 
-from typing import Any, Dict, List, NamedTuple, Optional
+from enum import Enum
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple
 
 
 JSON = Dict[str, Any]
 
 
 DEFAULT_MAP_FILE = "projects.json"
 
 
+class DownloadType(str, Enum):
+    GIT = "git"
+    ZIP = "zip"
+    SCRIPT = "script"
+
+
 class ProjectInfo(NamedTuple):
     """
     Information about a project to analyze.
     """
     name: str
     mode: int
+    source: DownloadType = DownloadType.SCRIPT
+    origin: str = ""
+    commit: str = ""
     enabled: bool = True
 
 
 class ProjectMap:
     """
     Project map stores info about all the "registered" projects.
     """
     def __init__(self, path: Optional[str] = None, should_exist: bool = True):
def _parse(raw_projects: List[JSON]) -> List[ProjectInfo]:
                 for raw_project in raw_projects]
 
     @staticmethod
     def _parse_project(raw_project: JSON) -> ProjectInfo:
         try:
             name: str = raw_project["name"]
             build_mode: int = raw_project["mode"]
             enabled: bool = raw_project.get("enabled", True)
-            return ProjectInfo(name, build_mode, enabled)
+            source: DownloadType = raw_project.get("source", "zip")
+
+            if source == DownloadType.GIT:
+                origin, commit = ProjectMap._get_git_params(raw_project)
+            else:
+                origin, commit = "", ""
+
+            return ProjectInfo(name, build_mode, source, origin, commit,
+                               enabled)
 
         except KeyError as e:
             raise ValueError(
                 f"Project info is required to have a '{e.args[0]}' field")
 
     @staticmethod
+    def _get_git_params(raw_project: JSON) -> Tuple[str, str]:
+        try:
+            return raw_project["origin"], raw_project["commit"]
+        except KeyError as e:
+            raise ValueError(
+                f"Profect info is required to have a '{e.args[0]}' field "
+                f"if it has a 'git' source")
+
+    @staticmethod
     def _create_empty(path: str):
         ProjectMap._save([], path)
 
     @staticmethod
     def _save(projects: List[ProjectInfo], path: str):
         with open(path, "w") as output:
             json.dump(ProjectMap._convert_infos_to_dicts(projects),
                       output, indent=2)
 
     @staticmethod
     def _convert_infos_to_dicts(projects: List[ProjectInfo]) -> List[JSON]:
         return [project._asdict() for project in projects]

clang/utils/analyzer/SATestBuild.py

For more logging, set the  env variables:
    zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
 
 The list of checkers tested are hardcoded in the Checkers variable.
 For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment
 variable. It should contain a comma separated list.
 """
 import CmpRuns
 import SATestUtils
-from ProjectMap import ProjectInfo, ProjectMap
+from ProjectMap import DownloadType, ProjectInfo, ProjectMap
 
 import argparse
 import glob
 import logging
 import math
 import multiprocessing
 import os
 import plistlib
 import shutil
 import sys
 import threading
 import time
+import zipfile
 
 from queue import Queue
 # mypy has problems finding InvalidFileException in the module
 # and this is we can shush that false positive
 from plistlib import InvalidFileException  # type:ignore
 from subprocess import CalledProcessError, check_call
 from typing import Dict, IO, List, NamedTuple, Optional, TYPE_CHECKING
 
def run_cleanup_script(directory: str, build_log_file: IO):
     cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
     script_path = os.path.join(directory, CLEANUP_SCRIPT)
 
     SATestUtils.run_script(script_path, build_log_file, cwd,
                            out=LOCAL.stdout, err=LOCAL.stderr,
                            verbose=VERBOSE)
 
 
-def download_and_patch(directory: str, build_log_file: IO):
-    """
-    Download the project and apply the local patchfile if it exists.
-    """
-    cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME)
-
-    # If the we don't already have the cached source, run the project's
-    # download script to download it.
-    if not os.path.exists(cached_source):
-        download(directory, build_log_file)
-        if not os.path.exists(cached_source):
-            stderr(f"Error: '{cached_source}' not found after download.\n")
-            exit(1)
-
-    patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
-
-    # Remove potentially stale patched source.
-    if os.path.exists(patched_source):
-        shutil.rmtree(patched_source)
-
-    # Copy the cached source and apply any patches to the copy.
-    shutil.copytree(cached_source, patched_source, symlinks=True)
-    apply_patch(directory, build_log_file)
-
-
-def download(directory: str, build_log_file: IO):
-    """
-    Run the script to download the project, if it exists.
-    """
-    script_path = os.path.join(directory, DOWNLOAD_SCRIPT)
-    SATestUtils.run_script(script_path, build_log_file, directory,
-                           out=LOCAL.stdout, err=LOCAL.stderr,
-                           verbose=VERBOSE)
-
-
-def apply_patch(directory: str, build_log_file: IO):
-    patchfile_path = os.path.join(directory, PATCHFILE_NAME)
-    patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
-
-    if not os.path.exists(patchfile_path):
-        stdout("  No local patches.\n")
-        return
-
-    stdout("  Applying patch.\n")
-    try:
-        check_call(f"patch -p1 < '{patchfile_path}'",
-                   cwd=patched_source,
-                   stderr=build_log_file,
-                   stdout=build_log_file,
-                   shell=True)
-
-    except CalledProcessError:
-        stderr(f"Error: Patch failed. "
-               f"See {build_log_file.name} for details.\n")
-        sys.exit(1)
-
-
 class TestInfo(NamedTuple):
     """
     Information about a project and settings for its analysis.
     """
     project: ProjectInfo
     override_compiler: bool = False
     extra_analyzer_config: str = ""
     is_reference_build: bool = False
def build(self, directory: str, output_dir: str):
             shutil.rmtree(output_dir)
 
         assert(not os.path.exists(output_dir))
         os.makedirs(os.path.join(output_dir, LOG_DIR_NAME))
 
         # Build and analyze the project.
         with open(build_log_path, "w+") as build_log_file:
             if self.project.mode == 1:
-                download_and_patch(directory, build_log_file)
+                self._download_and_patch(directory, build_log_file)
                 run_cleanup_script(directory, build_log_file)
                 self.scan_build(directory, output_dir, build_log_file)
             else:
                 self.analyze_preprocessed(directory, output_dir)
 
             if self.is_reference_build:
                 run_cleanup_script(directory, build_log_file)
                 normalize_reference_results(directory, output_dir,
class ProjectTester:
     def generate_config(self) -> str:
         out = "serialize-stats=true,stable-report-filename=true"
 
         if self.extra_analyzer_config:
             out += "," + self.extra_analyzer_config
 
         return out
 
+    def _download_and_patch(self, directory: str, build_log_file: IO):
+        """
+        Download the project and apply the local patchfile if it exists.
+        """
+        cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME)
+
+        # If the we don't already have the cached source, run the project's
+        # download script to download it.
+        if not os.path.exists(cached_source):
+            self._download(directory, build_log_file)
+            if not os.path.exists(cached_source):
+                stderr(f"Error: '{cached_source}' not found after download.\n")
+                exit(1)
+
+        patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
+
+        # Remove potentially stale patched source.
+        if os.path.exists(patched_source):
+            shutil.rmtree(patched_source)
+
+        # Copy the cached source and apply any patches to the copy.
+        shutil.copytree(cached_source, patched_source, symlinks=True)
+        self._apply_patch(directory, build_log_file)
+
+    def _download(self, directory: str, build_log_file: IO):
+        """
+        Run the script to download the project, if it exists.
+        """
+        if self.project.source == DownloadType.GIT:
+            self._download_from_git(directory, build_log_file)
+        elif self.project.source == DownloadType.ZIP:
+            self._unpack_zip(directory, build_log_file)
+        elif self.project.source == DownloadType.SCRIPT:
+            self._run_download_script(directory, build_log_file)
+        else:
+            raise ValueError(
+                f"Unknown source type '{self.project.source}' is found "
+                f"for the '{self.project.name}' project")
+
+    def _download_from_git(self, directory: str, build_log_file: IO):
+        cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME)
+        check_call(f"git clone {self.project.origin} {cached_source}",
+                   cwd=directory, stderr=build_log_file,
+                   stdout=build_log_file, shell=True)
+        check_call(f"git checkout --quiet {self.project.commit}",
+                   cwd=cached_source, stderr=build_log_file,
+                   stdout=build_log_file, shell=True)
+
+    def _unpack_zip(self, directory: str, build_log_file: IO):
+        zip_files = list(glob.glob(os.path.join(directory, "/*.zip")))
+
+        if len(zip_files) == 0:
+            raise ValueError(
+                f"Couldn't find any zip files to unpack for the "
+                f"'{self.project.name}' project")
+
+        if len(zip_files) > 1:
+            raise ValueError(
+                f"Couldn't decide which of the zip files ({zip_files}) "
+                f"for the '{self.project.name}' project to unpack")
+
+        with zipfile.ZipFile(zip_files[0], "r") as zip_file:
+            zip_file.extractall(os.path.join(directory,
+                                             CACHED_SOURCE_DIR_NAME))
+
+    @staticmethod
+    def _run_download_script(directory: str, build_log_file: IO):
+        script_path = os.path.join(directory, DOWNLOAD_SCRIPT)
+        SATestUtils.run_script(script_path, build_log_file, directory,
+                               out=LOCAL.stdout, err=LOCAL.stderr,
+                               verbose=VERBOSE)
+
+    @staticmethod
+    def _apply_patch(directory: str, build_log_file: IO):
+        patchfile_path = os.path.join(directory, PATCHFILE_NAME)
+        patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
+
+        if not os.path.exists(patchfile_path):
+            stdout("  No local patches.\n")
+            return
+
+        stdout("  Applying patch.\n")
+        try:
+            check_call(f"patch -p1 < '{patchfile_path}'",
+                       cwd=patched_source,
+                       stderr=build_log_file,
+                       stdout=build_log_file,
+                       shell=True)
+
+        except CalledProcessError:
+            stderr(f"Error: Patch failed. "
+                   f"See {build_log_file.name} for details.\n")
+            sys.exit(1)
+
 
 class TestProjectThread(threading.Thread):
     def __init__(self, tasks_queue: TestQueue,
                  results_differ: threading.Event,
                  failure_flag: threading.Event):
         """
         :param results_differ: Used to signify that results differ from
                the canonical ones.