[opt-viewer] Accept directories that are searched for opt.yaml files

[anemet]

Summary

This allows to pass the build directory where all the opt.yaml files are
rather than find | xargs which may invoke opt-viewer multiple times producing
incomplete html output.

The patch generalizes the same functionality from opt-diff.

[davide]

tools/opt-viewer/opt-diff.py

28 ↗

(On Diff #106891)

typo: optmization

32 ↗

(On Diff #106891)

Ditto.

tools/opt-viewer/optrecord.py

240–250 ↗

(On Diff #106891)

this seems like something that can walk the whole filesystem if you end up passing the wrogn directory. Not sure there's a better way though.

[anemet]

tools/opt-viewer/optrecord.py

240–250 ↗

(On Diff #106891)

Do you mean like stuff mounted and so on?

[davide]

tools/opt-viewer/optrecord.py

240–250 ↗

(On Diff #106891)

Yes, exactly.

[modocache]

Excellent, thanks @anemet! I had been using xargs and was encountering exactly this issue. I'll accept assuming you fix the optmization typos before committing. Thanks!

[davide]

LGTM with the small nits addressed.

[anemet]

@davide, this version seems to correctly exclude mounted dirs, what do you
think?

[davide]

sgtm

llvm/trunk/tools/opt-viewer/opt-diff.py

try:
     from yaml import CLoader as Loader
 except ImportError:
     from yaml import Loader
 
 import optrecord
 import argparse
 from collections import defaultdict
 from multiprocessing import cpu_count, Pool
-import os, os.path
-import fnmatch
-
-def find_files(dir_or_file):
-    if os.path.isfile(dir_or_file):
-        return [dir_or_file]
-
-    all = []
-    for dir, subdirs, files in os.walk(dir_or_file):
-        for file in files:
-            if fnmatch.fnmatch(file, "*.opt.yaml"):
-                all.append( os.path.join(dir, file))
-    return all
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description=desc)
-    parser.add_argument('yaml_dir_or_file_1')
-    parser.add_argument('yaml_dir_or_file_2')
+    parser.add_argument(
+        'yaml_dir_or_file_1',
+        help='An optimization record file or a directory searched for optimization '
+             'record files that are used as the old version for the comparison')
+    parser.add_argument(
+        'yaml_dir_or_file_2',
+        help='An optimization record file or a directory searched for optimization '
+             'record files that are used as the new version for the comparison')
     parser.add_argument(
         '--jobs',
         '-j',
         default=cpu_count(),
         type=int,
         help='Max job count (defaults to %(default)s, the current CPU count)')
     parser.add_argument(
         '--no-progress-indicator',
         '-n',
         action='store_true',
         default=False,
         help='Do not display any indicator of how many YAML files were read.')
     parser.add_argument('--output', '-o', default='diff.opt.yaml')
     args = parser.parse_args()
 
-    files1 = find_files(args.yaml_dir_or_file_1)
-    files2 = find_files(args.yaml_dir_or_file_2)
+    files1 = optrecord.find_opt_files([args.yaml_dir_or_file_1])
+    files2 = optrecord.find_opt_files([args.yaml_dir_or_file_2])
 
     print_progress = not args.no_progress_indicator
     all_remarks1, _, _ = optrecord.gather_results(files1, args.jobs, print_progress)
     all_remarks2, _, _ = optrecord.gather_results(files2, args.jobs, print_progress)
 
     added = set(all_remarks2.values()) - set(all_remarks1.values())
     removed = set(all_remarks1.values()) - set(all_remarks2.values())
 
     for r in added:
         r.Added = True
     for r in removed:
         r.Added = False
     with open(args.output, 'w') as stream:
         yaml.dump_all(added | removed, stream)

llvm/trunk/tools/opt-viewer/opt-stats.py

 import optrecord
 import argparse
 import operator
 from collections import defaultdict
 from multiprocessing import cpu_count, Pool
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description=desc)
-    parser.add_argument('yaml_files', nargs='+')
+    parser.add_argument(
+        'yaml_dirs_or_files',
+        nargs='+',
+        help='List of optimization record files or directories searched '
+             'for optimization record files.')
     parser.add_argument(
         '--jobs',
         '-j',
         default=cpu_count(),
         type=int,
         help='Max job count (defaults to %(default)s, the current CPU count)')
     parser.add_argument(
         '--no-progress-indicator',
         '-n',
         action='store_true',
         default=False,
         help='Do not display any indicator of how many YAML files were read.')
     args = parser.parse_args()
 
     print_progress = not args.no_progress_indicator
+
+    files = optrecord.find_opt_files(args.yaml_dirs_or_files)
+    if not files:
+        parser.error("No *.opt.yaml files found")
+        sys.exit(1)
+
     all_remarks, file_remarks, _ = optrecord.gather_results(
-        args.yaml_files, args.jobs, print_progress)
+        files, args.jobs, print_progress)
     if print_progress:
         print('\n')
 
     bypass = defaultdict(int)
     byname = defaultdict(int)
     for r in optrecord.itervalues(all_remarks):
         bypass[r.Pass] += 1
         byname[r.Pass + "/" + r.Name] += 1

llvm/trunk/tools/opt-viewer/opt-viewer.py

def generate_report(all_remarks,
     IndexRenderer(args.output_dir).render(sorted_remarks)
 
     shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
             "style.css"), output_dir)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description=desc)
-    parser.add_argument('yaml_files', nargs='+')
+    parser.add_argument(
+        'yaml_dirs_or_files',
+        nargs='+',
+        help='List of optimization record files or directories searched '
+             'for optimization record files.')
     parser.add_argument(
         '--output-dir',
         '-o',
         default='html',
         help='Path to a directory where generated HTML files will be output. '
              'If the directory does not already exist, it will be created. '
              '"%(default)s" by default.')
     parser.add_argument(
parser.add_argument(
         '-n',
         action='store_true',
         default=False,
         help='Do not display any indicator of how many YAML files were read '
              'or rendered into HTML.')
     args = parser.parse_args()
 
     print_progress = not args.no_progress_indicator
+
+    files = optrecord.find_opt_files(args.yaml_dirs_or_files)
+    if not files:
+        parser.error("No *.opt.yaml files found")
+        sys.exit(1)
+
     all_remarks, file_remarks, should_display_hotness = \
-        optrecord.gather_results(args.yaml_files, args.jobs, print_progress)
+        optrecord.gather_results(files, args.jobs, print_progress)
 
     map_remarks(all_remarks)
 
     generate_report(all_remarks,
                     file_remarks,
                     args.source_dir,
                     args.output_dir,
                     should_display_hotness,
                     args.jobs,
                     print_progress)

llvm/trunk/tools/opt-viewer/optrecord.py

 #!/usr/bin/env python2.7
 
 from __future__ import print_function
 
 import yaml
 # Try to use the C parser.
 try:
     from yaml import CLoader as Loader
 except ImportError:
     print("For faster parsing, you may want to install libYAML for PyYAML")
     from yaml import Loader
 
 import cgi
 from collections import defaultdict
+import fnmatch
 import functools
 from multiprocessing import Lock
+import os, os.path
 import subprocess
 
 import optpmap
 
 
 p = subprocess.Popen(['c++filt', '-n'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 p_lock = Lock()
 
def gather_results(filenames, num_jobs, should_print_progress):
 
     all_remarks = dict()
     file_remarks = defaultdict(functools.partial(defaultdict, list))
     for _, all_remarks_job, file_remarks_job in remarks:
         merge_file_remarks(file_remarks_job, all_remarks, file_remarks)
         all_remarks.update(all_remarks_job)
 
     return all_remarks, file_remarks, max_hotness != 0
+
+
+def find_opt_files(dirs_or_files):
+    all = []
+    for dir_or_file in dirs_or_files:
+        if os.path.isfile(dir_or_file):
+            all.append(dir_or_file)
+        else:
+            for dir, subdirs, files in os.walk(dir_or_file):
+                # Exclude mounted directories and symlinks (os.walk default).
+                subdirs[:] = [d for d in subdirs
+                              if not os.path.ismount(os.path.join(dir, d))]
+                for file in files:
+                    if fnmatch.fnmatch(file, "*.opt.yaml"):
+                        all.append(os.path.join(dir, file))
+    return all