[analyzer] SATestUtils.py: Refactor and add type annotations

[vsavchenko]

Modify forgotten parts

[NoQ]

Sure thanks!

clang/utils/analyzer/SATestUtils.py

47

Oof this list is weird.

clang/utils/analyzer/SATestBuild.py

 import plistlib
 import shutil
 import sys
 import threading
 import time
 
 from queue import Queue
 from subprocess import CalledProcessError, check_call
-from typing import (cast, Dict, Iterable, IO, List, NamedTuple, Tuple,
-                    TYPE_CHECKING)
+from typing import (cast, Dict, Iterable, IO, List, NamedTuple, Optional,
+                    Tuple, TYPE_CHECKING)
 
 
 ###############################################################################
 # Helper functions.
 ###############################################################################
 
 LOCAL = threading.local()
 LOCAL.stdout = sys.stdout

 
 ###############################################################################
 # Configuration setup.
 ###############################################################################
 
 
 # Find Clang for static analysis.
 if 'CC' in os.environ:
-    CLANG = os.environ['CC']
+    cc_candidate: Optional[str] = os.environ['CC']
 else:
-    CLANG = SATestUtils.which("clang", os.environ['PATH'])
-if not CLANG:
+    cc_candidate = SATestUtils.which("clang", os.environ['PATH'])
+if not cc_candidate:
     stderr("Error: cannot find 'clang' in PATH")
     sys.exit(1)
 
+CLANG = cc_candidate
+
 # Number of jobs.
 MAX_JOBS = int(math.ceil(multiprocessing.cpu_count() * 0.75))
 
 # Project map stores info about all the "registered" projects.
 PROJECT_MAP_FILE = "projectMap.csv"
 
 # Names of the project specific scripts.
 # The script that downloads the project.

 
 def run_cleanup_script(directory: str, build_log_file: IO):
     """
     Run pre-processing script if any.
     """
     cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
     script_path = os.path.join(directory, CLEANUP_SCRIPT)
 
-    SATestUtils.runScript(script_path, build_log_file, cwd,
-                          Stdout=LOCAL.stdout, Stderr=LOCAL.stderr)
+    SATestUtils.run_script(script_path, build_log_file, cwd,
+                           out=LOCAL.stdout, err=LOCAL.stderr,
+                           verbose=VERBOSE)
 
 
 def download_and_patch(directory: str, build_log_file: IO):
     """
     Download the project and apply the local patchfile if it exists.
     """
     cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME)
 
def download_and_patch(directory: str, build_log_file: IO):
     apply_patch(directory, build_log_file)
 
 
 def download(directory: str, build_log_file: IO):
     """
     Run the script to download the project, if it exists.
     """
     script_path = os.path.join(directory, DOWNLOAD_SCRIPT)
-    SATestUtils.runScript(script_path, build_log_file, directory,
-                          Stdout=LOCAL.stdout, Stderr=LOCAL.stderr)
+    SATestUtils.run_script(script_path, build_log_file, directory,
+                           out=LOCAL.stdout, err=LOCAL.stderr,
+                           verbose=VERBOSE)
 
 
 def apply_patch(directory: str, build_log_file: IO):
     patchfile_path = os.path.join(directory, PATCHFILE_NAME)
     patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
 
     if not os.path.exists(patchfile_path):
         stdout("  No local patches.\n")
def analyze_preprocessed(self, directory: str, output_dir: str):
         fail_path = os.path.join(plist_path, "failures")
         os.makedirs(fail_path)
 
         for full_file_name in glob.glob(directory + "/*"):
             file_name = os.path.basename(full_file_name)
             failed = False
 
             # Only run the analyzes on supported files.
-            if SATestUtils.hasNoExtension(file_name):
+            if SATestUtils.has_no_extension(file_name):
                 continue
-            if not SATestUtils.isValidSingleInputFile(file_name):
+            if not SATestUtils.is_valid_single_input_file(file_name):
                 stderr(f"Error: Invalid single input file {full_file_name}.\n")
                 raise Exception()
 
             # Build and call the analyzer command.
             plist_basename = os.path.join(plist_path, file_name)
             output_option = f"-o '{plist_basename}.plist' "
             command = f"{prefix}{output_option}'{file_name}'"
 

 def get_projects(map_file: IO) -> Iterable[Tuple[str, str]]:
     """
     Iterate over all projects defined in the project file handler `map_file`
     from the start.
     """
     map_file.seek(0)
     # TODO: csv format is not very readable, change it to JSON
     for project_info in csv.reader(map_file):
-        if (SATestUtils.isCommentCSVLine(project_info)):
+        if SATestUtils.is_comment_csv_line(project_info):
             continue
         # suppress mypy error
         yield cast(Tuple[str, str], project_info)
 
 
 def validate_project_file(map_file: IO):
     """
     Validate project file.

clang/utils/analyzer/SATestUtils.py

 import os
-from subprocess import check_call
 import sys
 
+from subprocess import CalledProcessError, check_call
+from typing import List, IO, Optional
 
-Verbose = 1
 
-
-def which(command, paths=None):
+def which(command: str, paths: Optional[str] = None) -> Optional[str]:
     """which(command, [paths]) - Look up the given command in the paths string
     (or the PATH environment variable, if unspecified)."""
 
     if paths is None:
         paths = os.environ.get('PATH', '')
 
     # Check for absolute match first.
     if os.path.exists(command):
for path in paths.split(os.pathsep):
         for ext in pathext:
             p = os.path.join(path, command + ext)
             if os.path.exists(p):
                 return p
 
     return None
 
 
-def hasNoExtension(FileName):
-    (Root, Ext) = os.path.splitext(FileName)
-    return (Ext == "")
+def has_no_extension(file_name: str) -> bool:
+    root, ext = os.path.splitext(file_name)
+    return ext == ""
 
 
-def isValidSingleInputFile(FileName):
-    (Root, Ext) = os.path.splitext(FileName)
-    return Ext in (".i", ".ii", ".c", ".cpp", ".m", "")
+def is_valid_single_input_file(file_name: str) -> bool:
+    root, ext = os.path.splitext(file_name)
+    return ext in (".i", ".ii", ".c", ".cpp", ".m", "")
 
 
-def runScript(ScriptPath, PBuildLogFile, Cwd, Stdout=sys.stdout,
-              Stderr=sys.stderr):
+def run_script(script_path: str, build_log_file: IO, cwd: str,
+               out=sys.stdout, err=sys.stderr, verbose: int = 0):
     """
     Run the provided script if it exists.
     """
-    if os.path.exists(ScriptPath):
+    if os.path.exists(script_path):
         try:
-            if Verbose == 1:
-                Stdout.write("  Executing: %s\n" % (ScriptPath,))
-            check_call("chmod +x '%s'" % ScriptPath, cwd=Cwd,
-                       stderr=PBuildLogFile,
-                       stdout=PBuildLogFile,
+            if verbose == 1:
+                out.write(f"  Executing: {script_path}\n")
+
+            check_call(f"chmod +x '{script_path}'", cwd=cwd,
+                       stderr=build_log_file,
+                       stdout=build_log_file,
                        shell=True)
-            check_call("'%s'" % ScriptPath, cwd=Cwd,
-                       stderr=PBuildLogFile,
-                       stdout=PBuildLogFile,
+
+            check_call(f"'{script_path}'", cwd=cwd,
+                       stderr=build_log_file,
+                       stdout=build_log_file,
                        shell=True)
-        except:
-            Stderr.write("Error: Running %s failed. See %s for details.\n" % (
-                         ScriptPath, PBuildLogFile.name))
+
+        except CalledProcessError:
+            err.write(f"Error: Running {script_path} failed. "
+                      f"See {build_log_file.name} for details.\n")
             sys.exit(-1)
 
 
-def isCommentCSVLine(Entries):
+def is_comment_csv_line(entries: List[str]) -> bool:
     """
     Treat CSV lines starting with a '#' as a comment.
     """
-    return len(Entries) > 0 and Entries[0].startswith("#")
+    return len(entries) > 0 and entries[0].startswith("#")