diagnose_bloat.py: Grouped diffs.

tools/binary_size/diagnose_bloat.py

 #!/usr/bin/env python
 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Tool for finding the cause of binary size bloat.
 
 See //tools/binary_size/README.md for example usage.
 
 Note: this tool will perform gclient sync/git checkout on your local repo if
@@ skipping 11 matching lines @@
 import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 import zipfile
 
 _COMMIT_COUNT_WARN_THRESHOLD = 15
 _ALLOWED_CONSECUTIVE_FAILURES = 2
-_DIFF_DETAILS_LINES_THRESHOLD = 100
 _SRC_ROOT = os.path.abspath(
     os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
 _DEFAULT_ARCHIVE_DIR = os.path.join(_SRC_ROOT, 'out', 'binary-size-results')
 _DEFAULT_OUT_DIR = os.path.join(_SRC_ROOT, 'out', 'binary-size-build')
 _DEFAULT_ANDROID_TARGET = 'monochrome_public_apk'
 _BINARY_SIZE_DIR = os.path.join(_SRC_ROOT, 'tools', 'binary_size')
 
 
 _DiffResult = collections.namedtuple('DiffResult', ['name', 'value', 'units'])
 
 
 class BaseDiff(object):
   """Base class capturing binary size diffs."""
   def __init__(self, name):
     self.name = name
     self.banner = '\n' + '*' * 30 + name + '*' * 30
 
   def AppendResults(self, logfile):
     """Print and write diff results to an open |logfile|."""
     _PrintAndWriteToFile(logfile, self.banner)
-    _PrintAndWriteToFile(logfile, 'Summary:')
-    _PrintAndWriteToFile(logfile, self.Summary())
-    _PrintAndWriteToFile(logfile, '\nDetails:')
-    _PrintAndWriteToFile(logfile, self.DetailedResults())
+    for s in self.Summary():
+      print s
+    print
+    for s in self.DetailedResults():
+      logfile.write(s + '\n')
 
   @property
   def summary_stat(self):
     return None
 
   def Summary(self):
     """A short description that summarizes the source of binary size bloat."""
     raise NotImplementedError()
 
   def DetailedResults(self):
     """An iterable description of the cause of binary size bloat."""
     raise NotImplementedError()
 
   def ProduceDiff(self, before_dir, after_dir):
     """Prepare a binary size diff with ready to print results."""
     raise NotImplementedError()
 
   def RunDiff(self, logfile, before_dir, after_dir):
     logging.info('Creating: %s', self.name)
     self.ProduceDiff(before_dir, after_dir)
     self.AppendResults(logfile)
 
 
 class NativeDiff(BaseDiff):
-  _RE_SUMMARY = re.compile(r'Section Sizes .*?\n\n.*?(?=\n\n)', flags=re.DOTALL)
   _RE_SUMMARY_STAT = re.compile(
       r'Section Sizes \(Total=(?P<value>\d+) (?P<units>\w+)\)')
   _SUMMARY_STAT_NAME = 'Native Library Delta'
 
   def __init__(self, size_name, supersize_path):
     self._size_name = size_name
     self._supersize_path = supersize_path
     self._diff = []
     super(NativeDiff, self).__init__('Native Diff')
 
   @property
   def summary_stat(self):
     m = NativeDiff._RE_SUMMARY_STAT.search(self._diff)
     if m:
       return _DiffResult(
           NativeDiff._SUMMARY_STAT_NAME, m.group('value'), m.group('units'))
     return None
 
   def DetailedResults(self):
     return self._diff.splitlines()
 
   def Summary(self):
-    return NativeDiff._RE_SUMMARY.search(self._diff).group()
+    return self.DetailedResults()[:100]
 
   def ProduceDiff(self, before_dir, after_dir):
     before_size = os.path.join(before_dir, self._size_name)
     after_size = os.path.join(after_dir, self._size_name)
     cmd = [self._supersize_path, 'diff', before_size, after_size]
     self._diff = _RunCmd(cmd)[0].replace('{', '{{').replace('}', '}}')
 
 
 class ResourceSizesDiff(BaseDiff):
   _RESOURCE_SIZES_PATH = os.path.join(
       _SRC_ROOT, 'build', 'android', 'resource_sizes.py')
+  _SUMMARY_SECTIONS = ('Breakdown', 'Specifics')
+  _AGGREGATE_SECTIONS = (
+      'InstallBreakdown', 'Breakdown', 'MainLibInfo', 'Uncompressed')
 
   def __init__(self, apk_name, slow_options=False):
     self._apk_name = apk_name
     self._slow_options = slow_options
     self._diff = None  # Set by |ProduceDiff()|
     super(ResourceSizesDiff, self).__init__('Resource Sizes Diff')
 
   @property
   def summary_stat(self):
-    for s in self._diff:
-      if 'normalized' in s.name:
-        return s
+    for section_name, results in self._diff.iteritems():
+      for sub_section_name, value, units in results:
+        if 'normalized' in sub_section_name:
+          full_name = '{} {}'.format(section_name, sub_section_name)
+          return _DiffResult(full_name, value, units)
     return None
 
   def DetailedResults(self):
-    return ['{:>+10,} {} {}'.format(value, units, name)
-            for name, value, units in self._diff]
+    return self._ResultLines()
 
   def Summary(self):
-    return 'Normalized APK size: {:+,} {}'.format(
-        self.summary_stat.value, self.summary_stat.units)
+    return self._ResultLines(
+        include_sections=ResourceSizesDiff._SUMMARY_SECTIONS)
 
   def ProduceDiff(self, before_dir, after_dir):
     before = self._RunResourceSizes(before_dir)
     after = self._RunResourceSizes(after_dir)
-    diff = []
+    self._diff = collections.defaultdict(list)
     for section, section_dict in after.iteritems():
       for subsection, v in section_dict.iteritems():
         # Ignore entries when resource_sizes.py chartjson format has changed.
         if (section not in before or
             subsection not in before[section] or
             v['units'] != before[section][subsection]['units']):
           logging.warning(
               'Found differing dict structures for resource_sizes.py, '
               'skipping %s %s', section, subsection)
         else:
-          diff.append(
-              _DiffResult(
-                  '%s %s' % (section, subsection),
-                  v['value'] - before[section][subsection]['value'],
-                  v['units']))
-    self._diff = sorted(diff, key=lambda x: abs(x.value), reverse=True)
+          self._diff[section].append(_DiffResult(
+              subsection,
+              v['value'] - before[section][subsection]['value'],
+              v['units']))
+
+  def _ResultLines(self, include_sections=None):
+    ret = []
+    for section_name, section_results in self._diff.iteritems():
+      section_no_target = re.sub(r'^.*_', '', section_name)
+      if not include_sections or section_no_target in include_sections:
+        sub_section_lines = []
+        section_sum = 0
+        units = ''
+        for name, value, units in section_results:
+          if value == 0 and include_sections:
+            continue
+          section_sum += value
+          sub_section_lines.append('{:>+10,} {} {}'.format(value, units, name))
+        section_header = section_name
+        if section_no_target in ResourceSizesDiff._AGGREGATE_SECTIONS:
+          section_header += ' ({:+,} {})'.format(section_sum, units)
+        if sub_section_lines:
+          ret.append(section_header)
+          ret.extend(sub_section_lines)
+    if not ret:
+      ret = ['Empty ' + self.name]
+    return ret
 
   def _RunResourceSizes(self, archive_dir):
     apk_path = os.path.join(archive_dir, self._apk_name)
     chartjson_file = os.path.join(archive_dir, 'results-chart.json')
     cmd = [self._RESOURCE_SIZES_PATH, apk_path,'--output-dir', archive_dir,
            '--no-output-dir', '--chartjson']
     if self._slow_options:
       cmd += ['--estimate-patch-size', '--dump-static-initializers']
     _RunCmd(cmd)
     with open(chartjson_file) as f:
@@ skipping 487 matching lines @@
   return os.path.join(dst, output_dir)
 
 
 def _PrintAndWriteToFile(logfile, s, *args, **kwargs):
   """Write and print |s| thottling output if |s| is a large list."""
   if isinstance(s, basestring):
     s = s.format(*args, **kwargs)
     print s
     logfile.write('%s\n' % s)
   else:
-    for l in s[:_DIFF_DETAILS_LINES_THRESHOLD]:
+    for l in s:
       print l
-    if len(s) > _DIFF_DETAILS_LINES_THRESHOLD:
-      print '\nOutput truncated, see %s for more.' % logfile.name
     logfile.write('\n'.join(s))

[agrieve, 2017/06/23 02:40:30]

nit: might be simpler to do:

if isinstance(s, basestring):
  data = s.format(*args, **kwargs) + '\n'
else:
  data = '\n'.join(s) + '\n'
sys.stdout.write(data)
logfile.write(data)


That way you'll ensure to write the same thing to the log as you do to stdout
(right now logfile.write is missing a trailing \n in the 2nd case I think.

[estevenson, 2017/06/23 17:23:27]

Done.

 
 
 @contextmanager
 def _TmpCopyBinarySizeDir():
   """Recursively copy files to a temp dir and yield supersize path."""
   # Needs to be at same level of nesting as the real //tools/binary_size
   # since supersize uses this to find d3 in //third_party.
   tmp_dir = tempfile.mkdtemp(dir=_SRC_ROOT)
   try:
     bs_dir = os.path.join(tmp_dir, 'binary_size')
@@ skipping 139 matching lines @@
 
       if i != 0:
         diff_mngr.MaybeDiff(i - 1, i)
 
     diff_mngr.Summarize()
 
 
 if __name__ == '__main__':
   sys.exit(main())