[analyzer][tests] Measure peak memory consumption for every project

[NoQ]

clang/utils/analyzer/SATestUtils.py

113

Do i understand correctly that this basically adds roughly 0.25 second delay to every invocation of the analyzer? If so, that may add up to a lot of seconds for a project with many small translation units which can potentially screw our wall clock measurements. I guess the ideal solution would be some sort of "select" that waits on process termination with timeout, or we could simply decrease the sleep interval and hope it won't consume too much cpu time.

clang/utils/analyzer/requirements.txt

1–2

Dunno, should we bother putting every utility into its own directory with its own requirements? Like, exploded-graph-rewriter.py certainly has different requirements.

[vsavchenko]

clang/utils/analyzer/SATestUtils.py

113

Not really, the process starts when we call Popen and instead of waiting for it to finish like check_call does, we poll the memory.

Another point (even though it is not very useful information in the context of my previous statement) is that this happens on every scan-build invocation and not for every clang invocation.

clang/utils/analyzer/requirements.txt

1–2

Maybe we should simply put it here as well?

[NoQ]

clang/utils/analyzer/SATestUtils.py

113

Oh right!

clang/utils/analyzer/requirements.txt

1–2

Maybe. Dunno, you're the boss!

[vsavchenko]

Add matplotlib and graphviz to requirements.txt

clang/utils/analyzer/Dockerfile

 VOLUME /llvm-project
 VOLUME /build
 VOLUME /scripts
 
 ENV PATH="/analyzer/bin:${PATH}"
 
 ADD entrypoint.py /entrypoint.py
 
-# Uncomment in case of requirements
-# ADD requirements.txt /requirements.txt
-# RUN pip3 install -r /requirements.txt
+ADD requirements.txt /requirements.txt
+RUN pip3 install -r /requirements.txt
 
 ENTRYPOINT ["python", "/entrypoint.py"]

clang/utils/analyzer/SATestBuild.py

For more logging, set the  env variables:
    zaks:TI zaks$ export CCC_ANALYZER_LOG=1
    zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
 
 The list of checkers tested are hardcoded in the Checkers variable.
 For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment
 variable. It should contain a comma separated list.
 """
 import CmpRuns
-import SATestUtils
+import SATestUtils as utils
 from ProjectMap import DownloadType, ProjectInfo
 
 import glob
 import logging
 import math
 import multiprocessing
 import os
 import plistlib
 import shutil
 import sys
 import threading
 import time
 import zipfile
 
 from queue import Queue
 # mypy has problems finding InvalidFileException in the module
 # and this is we can shush that false positive
 from plistlib import InvalidFileException  # type:ignore
 from subprocess import CalledProcessError, check_call
-from typing import Dict, IO, List, NamedTuple, Optional, TYPE_CHECKING
+from typing import Dict, IO, List, NamedTuple, Optional, TYPE_CHECKING, Tuple
 
 
 ###############################################################################
 # Helper functions.
 ###############################################################################
 
 class StreamToLogger:
     def __init__(self, logger: logging.Logger,

 # Configuration setup.
 ###############################################################################
 
 
 # Find Clang for static analysis.
 if 'CC' in os.environ:
     cc_candidate: Optional[str] = os.environ['CC']
 else:
-    cc_candidate = SATestUtils.which("clang", os.environ['PATH'])
+    cc_candidate = utils.which("clang", os.environ['PATH'])
 if not cc_candidate:
     stderr("Error: cannot find 'clang' in PATH")
     sys.exit(1)
 
 CLANG = cc_candidate
 
 # Number of jobs.
 MAX_JOBS = int(math.ceil(multiprocessing.cpu_count() * 0.75))

 
 def run_cleanup_script(directory: str, build_log_file: IO):
     """
     Run pre-processing script if any.
     """
     cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
     script_path = os.path.join(directory, CLEANUP_SCRIPT)
 
-    SATestUtils.run_script(script_path, build_log_file, cwd,
-                           out=LOCAL.stdout, err=LOCAL.stderr,
-                           verbose=VERBOSE)
+    utils.run_script(script_path, build_log_file, cwd,
 
 
 class TestInfo(NamedTuple):
     """
     Information about a project and settings for its analysis.
     """
     project: ProjectInfo
     override_compiler: bool = False
class ProjectTester:
 
     def get_output_dir(self) -> str:
         if self.is_reference_build:
             return REF_PREFIX + OUTPUT_DIR_NAME
         else:
             return OUTPUT_DIR_NAME
 
     def build(self, directory: str, output_dir: str):
-        time_start = time.time()
-
         build_log_path = get_build_log_path(output_dir)
 
         stdout(f"Log file: {build_log_path}\n")
         stdout(f"Output directory: {output_dir}\n")
 
         remove_log_file(output_dir)
 
         # Clean up scan build results.
         if os.path.exists(output_dir):
             if VERBOSE >= 1:
                 stdout(f"  Removing old results: {output_dir}\n")
 
             shutil.rmtree(output_dir)
 
         assert(not os.path.exists(output_dir))
         os.makedirs(os.path.join(output_dir, LOG_DIR_NAME))
 
         # Build and analyze the project.
         with open(build_log_path, "w+") as build_log_file:
             if self.project.mode == 1:
                 self._download_and_patch(directory, build_log_file)
                 run_cleanup_script(directory, build_log_file)
-                self.scan_build(directory, output_dir, build_log_file)
+                build_time, memory = self.scan_build(directory, output_dir,
+                                               build_log_file)
             else:
-                self.analyze_preprocessed(directory, output_dir)
+                build_time, memory = self.analyze_preprocessed(directory,
+                                                               output_dir)
 
             if self.is_reference_build:
                 run_cleanup_script(directory, build_log_file)
                 normalize_reference_results(directory, output_dir,
                                             self.project.mode)
 
-        stdout(f"Build complete (time: {time.time() - time_start:.2f}). "
+        stdout(f"Build complete (time: {utils.time_to_str(build_time)}, "
+               f"peak memory: {utils.memory_to_str(memory)}). "
                f"See the log for more details: {build_log_path}\n")
 
-    def scan_build(self, directory: str, output_dir: str, build_log_file: IO):
+    def scan_build(self, directory: str, output_dir: str,
+                   build_log_file: IO) -> Tuple[float, int]:
         """
         Build the project with scan-build by reading in the commands and
         prefixing them with the scan-build options.
         """
         build_script_path = os.path.join(directory, BUILD_SCRIPT)
         if not os.path.exists(build_script_path):
             stderr(f"Error: build script is not defined: "
                    f"{build_script_path}\n")
def scan_build(self, directory: str, output_dir: str,
         options += f"-enable-checker {all_checkers} "
         options += "--keep-empty "
         options += f"-analyzer-config '{self.generate_config()}' "
 
         if self.override_compiler:
             options += "--override-compiler "
 
         extra_env: Dict[str, str] = {}
+
+        execution_time = 0.0
+        peak_memory = 0
+
         try:
             command_file = open(build_script_path, "r")
             command_prefix = "scan-build " + options + " "
 
             for command in command_file:
                 command = command.strip()
 
                 if len(command) == 0:
def scan_build(self, directory: str, output_dir: str,
                         "-j" not in command:
                     command += f" -j{MAX_JOBS}"
 
                 command_to_run = command_prefix + command
 
                 if VERBOSE >= 1:
                     stdout(f"  Executing: {command_to_run}\n")
 
-                check_call(command_to_run, cwd=cwd,
-                           stderr=build_log_file,
-                           stdout=build_log_file,
-                           env=dict(os.environ, **extra_env),
-                           shell=True)
+                time, mem = utils.check_and_measure_call(
+                    command_to_run, cwd=cwd,
 
+                execution_time += time
+                peak_memory = max(peak_memory, mem)
+
         except CalledProcessError:
             stderr("Error: scan-build failed. Its output was: \n")
             build_log_file.seek(0)
             shutil.copyfileobj(build_log_file, LOCAL.stderr)
             sys.exit(1)
 
-    def analyze_preprocessed(self, directory: str, output_dir: str):
+        return execution_time, peak_memory
+
+    def analyze_preprocessed(self, directory: str,
+                             output_dir: str) -> Tuple[float, int]:
         """
         Run analysis on a set of preprocessed files.
         """
         if os.path.exists(os.path.join(directory, BUILD_SCRIPT)):
             stderr(f"Error: The preprocessed files project "
                    f"should not contain {BUILD_SCRIPT}\n")
             raise Exception()
 
         prefix = CLANG + " --analyze "
 
         prefix += "--analyzer-output plist "
         prefix += " -Xclang -analyzer-checker=" + CHECKERS
         prefix += " -fcxx-exceptions -fblocks "
         prefix += " -Xclang -analyzer-config "
         prefix += f"-Xclang {self.generate_config()} "
 
         if self.project.mode == 2:
             prefix += "-std=c++11 "
 
         plist_path = os.path.join(directory, output_dir, "date")
         fail_path = os.path.join(plist_path, "failures")
         os.makedirs(fail_path)
 
+        execution_time = 0.0
+        peak_memory = 0
+
         for full_file_name in glob.glob(directory + "/*"):
             file_name = os.path.basename(full_file_name)
             failed = False
 
             # Only run the analyzes on supported files.
-            if SATestUtils.has_no_extension(file_name):
+            if utils.has_no_extension(file_name):
                 continue
-            if not SATestUtils.is_valid_single_input_file(file_name):
+            if not utils.is_valid_single_input_file(file_name):
                 stderr(f"Error: Invalid single input file {full_file_name}.\n")
                 raise Exception()
 
             # Build and call the analyzer command.
             plist_basename = os.path.join(plist_path, file_name)
             output_option = f"-o '{plist_basename}.plist' "
             command = f"{prefix}{output_option}'{file_name}'"
 
             log_path = os.path.join(fail_path, file_name + ".stderr.txt")
             with open(log_path, "w+") as log_file:
                 try:
                     if VERBOSE >= 1:
                         stdout(f"  Executing: {command}\n")
 
-                    check_call(command, cwd=directory, stderr=log_file,
-                               stdout=log_file, shell=True)
+                    time, mem = utils.check_and_measure_call(
+                        command, cwd=directory, stderr=log_file,
 
+                    execution_time += time
+                    peak_memory = max(peak_memory, mem)
+
                 except CalledProcessError as e:
                     stderr(f"Error: Analyzes of {full_file_name} failed. "
                            f"See {log_file.name} for details. "
                            f"Error code {e.returncode}.\n")
                     failed = True
 
                 # If command did not fail, erase the log file.
                 if not failed:
                     os.remove(log_file.name)
 
+        return execution_time, peak_memory
+
     def generate_config(self) -> str:
         out = "serialize-stats=true,stable-report-filename=true"
 
         if self.extra_analyzer_config:
             out += "," + self.extra_analyzer_config
 
         return out
 
def _unpack_zip(self, directory: str, build_log_file: IO):
 
         with zipfile.ZipFile(zip_files[0], "r") as zip_file:
             zip_file.extractall(os.path.join(directory,
                                              CACHED_SOURCE_DIR_NAME))
 
     @staticmethod
     def _run_download_script(directory: str, build_log_file: IO):
         script_path = os.path.join(directory, DOWNLOAD_SCRIPT)
-        SATestUtils.run_script(script_path, build_log_file, directory,
-                               out=LOCAL.stdout, err=LOCAL.stderr,
-                               verbose=VERBOSE)
+        utils.run_script(script_path, build_log_file, directory,
 
     @staticmethod
     def _apply_patch(directory: str, build_log_file: IO):
         patchfile_path = os.path.join(directory, PATCHFILE_NAME)
         patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
 
         if not os.path.exists(patchfile_path):
             stdout("  No local patches.\n")

clang/utils/analyzer/SATestUtils.py

 import os
 import sys
+import time
 
 from subprocess import CalledProcessError, check_call
-from typing import List, IO, Optional
+from typing import List, IO, Optional, Tuple
 
 
 def which(command: str, paths: Optional[str] = None) -> Optional[str]:
     """which(command, [paths]) - Look up the given command in the paths string
     (or the PATH environment variable, if unspecified)."""
 
     if paths is None:
         paths = os.environ.get('PATH', '')
def has_no_extension(file_name: str) -> bool:
     return ext == ""
 
 
 def is_valid_single_input_file(file_name: str) -> bool:
     root, ext = os.path.splitext(file_name)
     return ext in (".i", ".ii", ".c", ".cpp", ".m", "")
 
 
+def time_to_str(time: float) -> str:
+    """
+    Convert given time in seconds into a human-readable string.
+    """
+    return f"{time:.2f}s"
+
+
+def memory_to_str(memory: int) -> str:
+    """
+    Convert given number of bytes into a human-readable string.
+    """
+    if memory:
+        try:
+            import humanize
+            return humanize.naturalsize(memory, gnu=True)
+        except ImportError:
+            # no formatter installed, let's keep it in bytes
+            return f"{memory}B"
+
+    # If memory is 0, we didn't succeed measuring it.
+    return "N/A"
+
+
+def check_and_measure_call(*popenargs, **kwargs) -> Tuple[float, int]:
+    """
+    Run command with arguments.  Wait for command to complete and measure
+    execution time and peak memory consumption.
+    If the exit code was zero then return, otherwise raise
+    CalledProcessError.  The CalledProcessError object will have the
+    return code in the returncode attribute.
+
+    The arguments are the same as for the call and check_call functions.
+
+    Return a tuple of execution time and peak memory.
+    """
+    peak_mem = 0
+    start_time = time.time()
+
+    try:
+        import psutil as ps
+
+        def get_memory(process: ps.Process) -> int:
+            mem = 0
+
+            # we want to gather memory usage from all of the child processes
+            descendants = list(process.children(recursive=True))
+            descendants.append(process)
+
+            for subprocess in descendants:
+                try:
+                    mem += subprocess.memory_info().rss
+                except (ps.NoSuchProcess, ps.AccessDenied):
+                    continue
+
+            return mem
+
+        with ps.Popen(*popenargs, **kwargs) as process:
+            # while the process is running calculate resource utilization.
+            while (process.is_running() and
+                   process.status() != ps.STATUS_ZOMBIE):
+                # track the peak utilization of the process
+                peak_mem = max(peak_mem, get_memory(process))
+                time.sleep(.5)
+
+            if process.is_running():
+                process.kill()
+
+        if process.returncode != 0:
+            cmd = kwargs.get("args")
+            if cmd is None:
+                cmd = popenargs[0]
+            raise CalledProcessError(process.returncode, cmd)
+
+    except ImportError:
+        # back off to subprocess if we don't have psutil installed
+        peak_mem = 0
+        check_call(*popenargs, **kwargs)
+
+    return time.time() - start_time, peak_mem
+
+
 def run_script(script_path: str, build_log_file: IO, cwd: str,
                out=sys.stdout, err=sys.stderr, verbose: int = 0):
     """
     Run the provided script if it exists.
     """
     if os.path.exists(script_path):
         try:
             if verbose == 1:

clang/utils/analyzer/requirements.txt

+humanize
+psutil