[lldb/Scripts] Add script to replay multiple reproducers

[JDevlieghere]

Summary

Script to replay reproducers with the command line driver in parallel. This is used for stage 1 as described in the RFC on lldb-dev [1].

[1] http://lists.llvm.org/pipermail/lldb-dev/2020-April/016100.html

[JDevlieghere]

lldb/scripts/reproducer-replay.py

2

This is needed for the timeout. I'd rather not rely on an external package for this unless anyone objects.

[labath]

lldb/scripts/reproducer-replay.py

2

SGTM. Hopefully will get rid of py2 soon, and this isn't a very critical piece of infrastructure.

51

it would probably be slightly cleaner to make this a generator and use yield os.path.join(root, dir)

87

I'm not sure how useful is this now that we don't put reproducers in /tmp

lldb/scripts/reproducer-replay.py

+#! /usr/bin/env python3
+
+from multiprocessing import Pool
+import multiprocessing
+import argparse
+import tempfile
+import logging
+import os
+import subprocess
+
+
+def run_reproducer(path):
+    proc = subprocess.Popen([LLDB, '--replay', path],
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+    reason = None
+    try:
+        outs, errs = proc.communicate(timeout=TIMEOUT)
+        result = 'PASSED' if proc.returncode == 0 else 'FAILED'
+        if proc.returncode != 0:
+            outs = outs.decode()
+            errs = errs.decode()
+            # Do some pattern matching to find out the cause of the failure.
+            if 'Encountered unexpected packet during replay' in errs:
+                reason = 'Unexpected packet'
+            elif 'Assertion failed' in errs:
+                reason = 'Assertion failed'
+            elif 'UNREACHABLE' in errs:
+                reason = 'Unreachable executed'
+            elif 'Segmentation fault' in errs:
+                reason = 'Segmentation fault'
+            elif 'Illegal instruction' in errs:
+                reason = 'Illegal instruction'
+            else:
+                reason = f'Exit code {proc.returncode}'
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        outs, errs = proc.communicate()
+        result = 'TIMEOUT'
+
+    reason_str = f' ({reason})' if reason else ''
+    print(f'{result}: {path}{reason_str}')
+
+
+def find_reproducers(path):
+    for root, dirs, files in os.walk(path):
+        for dir in dirs:
+            _, extension = os.path.splitext(dir)
+            if dir.startswith('Test') and extension == '.py':
+                yield os.path.join(root, dir)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='LLDB API Test Replay Driver. '
+        'Replay one or more reproducers in parallel using the specified LLDB driver. '
+        'The script will look for reproducers generated by the API lit test suite. '
+        'To generate the reproducers, pass --param \'lldb-run-with-repro=capture\' to lit.'
+    )
+    parser.add_argument(
+        '-j',
+        '--threads',
+        type=int,
+        default=multiprocessing.cpu_count(),
+        help='Number of threads. The number of CPU threads if not specified.')
+    parser.add_argument(
+        '-t',
+        '--timeout',
+        type=int,
+        default=60,
+        help='Replay timeout in seconds. 60 seconds if not specified.')
+    parser.add_argument(
+        '-p',
+        '--path',
+        type=str,
+        default=os.getcwd(),
+        help=
+        'Path to the directory containing the reproducers. The current working directory if not specified.'
+    )
+    parser.add_argument('-l',
+                        '--lldb',
+                        type=str,
+                        required=True,
+                        help='Path to the LLDB command line driver')
+    args = parser.parse_args()
+
+    global LLDB
+    global TIMEOUT
+    LLDB = args.lldb
+    TIMEOUT = args.timeout
+
+    print(
+        f'Replaying reproducers in {args.path} with {args.threads} threads and a {args.timeout} seconds timeout'
+    )
+
+    try:
+        pool = Pool(args.threads)
+        pool.map(run_reproducer, find_reproducers(args.path))
+    except KeyboardInterrupt:
+        print('Interrupted')