[opt-viewer] Reduce memory consumption by another 20-25%

[anemet]

Summary

The Args field of the remark which consists of a list of mappings in YAML is
translated into a list of (small) dicts on Python. An empty dict is 280 bytes
on my system so we can save memory by using a tuple of tuples instead.

Making a tuple of tuples rather than a list of tuples allows Args to be shared
with the key of the remark. This is actually an even greater saving. (Keys
are alive throughout the entire run in all_remarks.)

Here are a few opt-stats runs with different input sizes while measuring heap
usage with heapy. Avg remark size is simply estimated as
heap-size / # of remarks:

| # of files             |   60 |  114 |  308 |  605 | 1370 |
| # of remarks           |  20K |  37K | 146K | 180K | 640K |
| total file size (MB)   |   22 |   51 |  219 |  202 | 1034 |
|------------------------+------+------+------+------+------|
| Avg remark size before | 4339 | 4792 | 4761 | 4096 | 4607 |
| Avg remark size after  | 3446 | 3641 | 3567 | 3146 | 3347 |
| Rate                   | 0.79 | 0.76 | 0.75 | 0.77 | 0.73 |

[anemet]

Before writing out the difference YAML file in opt-diff, we need to recover
the YAML-friendly data structure for the remarks.

[modocache]

tools/opt-viewer/optrecord.py

69

Great comment, thanks!

84

Typo: s/handels/handles/g.

tools/opt-viewer/opt-diff.py

if __name__ == '__main__':
 
     added = set(all_remarks2.values()) - set(all_remarks1.values())
     removed = set(all_remarks1.values()) - set(all_remarks2.values())
 
     for r in added:
         r.Added = True
     for r in removed:
         r.Added = False
+
+    result = added | removed
+    for r in result:
+        r.recover_yaml_structure()
+
     with open(args.output, 'w') as stream:
-        yaml.dump_all(added | removed, stream)
+        yaml.dump_all(result, stream)

tools/opt-viewer/opt-viewer.py

</tr>'''.format(**locals()), file=self.stream)
 
             for remark in line_remarks.get(linenum, []):
                 self.render_inline_remarks(remark, html_line)
 
     def render_inline_remarks(self, r, line):
         inlining_context = r.DemangledFunctionName
         dl = context.caller_loc.get(r.Function)
         if dl:
-            link = optrecord.make_link(dl['File'], dl['Line'] - 2)
+            dl_dict = dict(list(dl))
+            link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
             inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
 
         # Column is the number of characters *including* tabs, keep those and
         # replace everything else with spaces.
         indent = line[:max(r.Column, 1) - 1]
         indent = re.sub('\S', ' ', indent)
 
         print('''

 
 
 def map_remarks(all_remarks):
     # Set up a map between function names and their source location for
     # function where inlining happened
     for remark in optrecord.itervalues(all_remarks):
         if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
             for arg in remark.Args:
-                caller = arg.get('Caller')
+                arg_dict = dict(list(arg))
+                caller = arg_dict.get('Caller')
                 if caller:
                     try:
-                        context.caller_loc[caller] = arg['DebugLoc']
+                        context.caller_loc[caller] = arg_dict['DebugLoc']
                     except KeyError:
                         pass
 
 
 def generate_report(all_remarks,
                     file_remarks,
                     source_dir,
                     output_dir,

tools/opt-viewer/optrecord.py

 def make_link(File, Line):
     return "\"{}#L{}\"".format(html_file_name(File), Line)
 
 
 class Remark(yaml.YAMLObject):
     # Work-around for http://pyyaml.org/ticket/154.
     yaml_loader = Loader
 
-    def _intern_strings(self):
+    # Intern all strings since we have lot of duplication across filenames,
+    # remark text.
+    #
+    # Change Args from a list of dicts to a tuple of tuples.  This saves
+    # memory in two ways.  One, a small tuple is significantly smaller than a
+    # small dict.  Two, using tuple instead of list allows Args to be directly
+    # used as part of the key (in Python only immutable types are hashable).
+    def _reduce_memory(self):
         self.Pass = intern(self.Pass)
         self.Name = intern(self.Name)
         self.Function = intern(self.Function)
 
-        # Intern key and value if string and recurse if value is a dictionary.
-        # This handels [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
-        def _intern_dict(old_dict):
+        def _reduce_memory_dict(old_dict):
             new_dict = dict()
             for (k, v) in old_dict.iteritems():
                 if type(k) is str:
                     k = intern(k)
 
                 if type(v) is str:
                     v = intern(v)
                 elif type(v) is dict:
-                    v = _intern_dict(v)
+                    # This handels [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
+                    v = _reduce_memory_dict(v)
                 new_dict[k] = v
-            return new_dict
+            return tuple(new_dict.items())
+
+        self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args])
+
+    # The inverse operation of the dictonary-related memory optimization in
+    # _reduce_memory_dict.  E.g.
+    #     (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}]
+    def recover_yaml_structure(self):
+        def tuple_to_dict(t):
+            d = dict()
+            for (k, v) in t:
+                if type(v) is tuple:
+                    v = tuple_to_dict(v)
+                d[k] = v
+            return d
 
-        self.Args = [_intern_dict(arg_dict) for arg_dict in self.Args]
+        self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args]
 
     def canonicalize(self):
         if not hasattr(self, 'Hotness'):
             self.Hotness = 0
         if not hasattr(self, 'Args'):
             self.Args = []
-        self._intern_strings()
+        self._reduce_memory()
 
     @property
     def File(self):
         return self.DebugLoc['File']
 
     @property
     def Line(self):
         return int(self.DebugLoc['Line'])
class Remark(yaml.YAMLObject):
     def DemangledFunctionName(self):
         return demangle(self.Function)
 
     @property
     def Link(self):
         return make_link(self.File, self.Line)
 
     def getArgString(self, mapping):
-        mapping = mapping.copy()
+        mapping = dict(list(mapping))
         dl = mapping.get('DebugLoc')
         if dl:
             del mapping['DebugLoc']
 
         assert(len(mapping) == 1)
         (key, value) = mapping.items()[0]
 
         if key == 'Caller' or key == 'Callee':
             value = cgi.escape(demangle(value))
 
         if dl and key != 'Caller':
+            dl_dict = dict(list(dl))
             return "<a href={}>{}</a>".format(
-                make_link(dl['File'], dl['Line']), value)
+                make_link(dl_dict['File'], dl_dict['Line']), value)
         else:
             return value
 
     def getDiffPrefix(self):
         if hasattr(self, 'Added'):
             if self.Added:
                 return '+'
             else:
class Remark(yaml.YAMLObject):
     def RelativeHotness(self):
         if self.max_hotness:
             return "{}%".format(int(round(self.Hotness * 100 / self.max_hotness)))
         else:
             return ''
 
     @property
     def key(self):
-        k = (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, self.Line, self.Column, self.Function)
-        for arg in self.Args:
-            for (key, value) in iteritems(arg):
-                if type(value) is dict:
-                    value = tuple(value.items())
-                k += (key, value)
-        return k
+        return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File,
+                self.Line, self.Column, self.Function, self.Args)
 
     def __hash__(self):
         return hash(self.key)
 
     def __eq__(self, other):
         return self.key == other.key
 
     def __repr__(self):