New binary size tool visualization options.

tools/binary_size/run_binary_size_analysis.py

 #!/usr/bin/python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generate a spatial analysis against an arbitrary library.
 
 To use, build the 'binary_size_tool' target. Then run this tool, passing
 in the location of the library to be analyzed along with any other options
 you desire.
 """
 
 import collections
 import fileinput
 import json
 import optparse
 import os
 import pprint
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 
 
+# TODO(andrewhayden): Only used for legacy reports. Delete.
 def FormatBytes(bytes):
   """Pretty-print a number of bytes."""
   if bytes > 1e6:
     bytes = bytes / 1.0e6
     return '%.1fm' % bytes
   if bytes > 1e3:
     bytes = bytes / 1.0e3
     return '%.1fk' % bytes
   return str(bytes)
 
 
+# TODO(andrewhayden): Only used for legacy reports. Delete.
 def SymbolTypeToHuman(type):
   """Convert a symbol type as printed by nm into a human-readable name."""
   return {'b': 'bss',
           'd': 'data',
           'r': 'read-only data',
           't': 'code',
           'w': 'weak symbol',
           'v': 'weak symbol'}[type]
 
 
@@ skipping 16 matching lines @@
   addr_re = re.compile(r'^[0-9a-f]{8} (.) ([^\t]+)(?:\t.*)?$')
   # Match lines that don't have an address at all -- typically external symbols.
   noaddr_re = re.compile(r'^ {8} (.) (.*)$')
 
   for line in input:
     line = line.rstrip()
     match = sym_re.match(line)
     if match:
       size, type, sym = match.groups()[0:3]
       size = int(size, 16)
-      type = type.lower()
-      if type == 'v':
-        type = 'w'  # just call them all weak
-      if type == 'b':
+      if type.lower() == 'b':
         continue  # skip all BSS for now
       path = match.group(4)
       yield sym, type, size, path
       continue
     match = addr_re.match(line)
     if match:
       type, sym = match.groups()[0:2]
       # No size == we don't care.
       continue
     match = noaddr_re.match(line)
     if match:
       type, sym = match.groups()
       if type in ('U', 'w'):
         # external or weak symbol
         continue
 
     print >>sys.stderr, 'unparsed:', repr(line)
 

[bulach, 2014/04/16 16:32:16]

nit: another \n

also, _MkChuld

[Andrew Hayden (chromium.org), 2014/04/16 17:53:21]

Done.

+def _mk_child(node, name):
+  child = None
+  for test in node['children']:
+    if test['n'] == name:
+      child = test
+      break
+  if child == None:

[bulach, 2014/04/16 16:32:16]

nit: 
if not child:

[Andrew Hayden (chromium.org), 2014/04/16 17:53:21]

Done.

+    child = {'n': name, 'children': []}
+    node['children'].append(child)
+  return child
 

[bulach, 2014/04/16 16:32:16]

nit: another \n

[Andrew Hayden (chromium.org), 2014/04/16 17:53:21]

Done throughout

+def MakeCompactTree(symbols):
+  result = {'n': '/', 'children': [], 'k': 'p', 'maxDepth': 0}
+  for symbol_name, symbol_type, symbol_size, file_path in symbols:
+
+    if 'vtable for ' in symbol_name:
+      symbol_type = '@' # hack to categorize these separately
+    # Take path like '/foo/bar/baz', convert to ['foo', 'bar', 'baz']
+    if file_path:
+      file_path = os.path.normpath(file_path)
+    else:
+      file_path = '(No Path)'
+
+    if file_path.startswith('/'):
+      file_path = file_path[1:]
+    path_parts = file_path.split('/')
+
+    # Find pre-existing node in tree, or update if it already exists
+    node = result
+    depth = 0
+    while len(path_parts) > 0:
+      path_part = path_parts.pop(0)
+      if len(path_part) == 0:
+        continue
+      depth += 1
+      node = _mk_child(node, path_part);
+      node['k'] = 'p' # p for path
+
+    # 'node' is now the file node. Find the symbol-type bucket.
+    node['lastPathElement'] = True
+    node = _mk_child(node, symbol_type)
+    node['t'] = symbol_type
+    node['k'] = 'b' # b for bucket
+    depth += 1
+
+    # 'node' is now the symbol-type bucket. Make the child entry.
+    node = _mk_child(node, symbol_name)
+    if 'children' in node: # Only possible if we're adding duplicate entries!!!
+      del node['children']
+    node['value'] = symbol_size
+    node['t'] = symbol_type
+    node['k'] = 's' # s for symbol
+    depth += 1
+    result['maxDepth'] = max(result['maxDepth'], depth);
+
+  return result
+
+
+# TODO(andrewhayden): Only used for legacy reports. Delete.
 def TreeifySymbols(symbols):
   """Convert symbols into a path-based tree, calculating size information
   along the way.
 
   The result is a dictionary that contains two kinds of nodes:
   1. Leaf nodes, representing source code locations (e.g., c++ files)
      These nodes have the following dictionary entries:
        sizes: a dictionary whose keys are categories (such as code, data,
               vtable, etceteras) and whose values are the size, in bytes, of
               those categories;
@@ skipping 75 matching lines @@
         subkey = '__FUNCTION__'
       elif sym.startswith('CSWTCH.'):
         subkey = 'CSWTCH'
       elif '::' in sym:
         subkey = sym[0:sym.find('::') + 2]
       tree['sizes'][subkey] = tree['sizes'].get(subkey, 0) + size
       tree['size'] += size
   return dirs
 
 
+# TODO(andrewhayden): Only used for legacy reports. Delete.
 def JsonifyTree(tree, name):
   """Convert TreeifySymbols output to a JSON treemap.
 
   The format is very similar, with the notable exceptions being
   lists of children instead of maps and some different attribute names."""
   children = []
   css_class_map = {
                   '[vtable]': 'vtable',
                   '[rodata]': 'read-only_data',
                   '[data]': 'data',
@@ skipping 16 matching lines @@
   # Sort children by size, largest to smallest.
   children.sort(key=lambda child: -child['data']['$area'])
 
   # For leaf nodes, the 'size' attribute is the size of the leaf;
   # Non-leaf nodes don't really have a size, but their 'size' attribute is
   # the sum of the sizes of all their children.
   return {'name': name + ' (' + FormatBytes(tree['size']) + ')',
           'data': { '$area': tree['size'] },
           'children': children }
 
+def DumpCompactTree(symbols, outfile):
+  out = open(outfile, 'w')
+  try:
+    out.write('var tree_data = ' + json.dumps(MakeCompactTree(symbols)))
+  finally:
+    out.flush()
+    out.close()
 
+# TODO(andrewhayden): Only used for legacy reports. Delete.
 def DumpTreemap(symbols, outfile):
   dirs = TreeifySymbols(symbols)
   out = open(outfile, 'w')
   try:
     out.write('var kTree = ' + json.dumps(JsonifyTree(dirs, '/')))
   finally:
     out.flush()
     out.close()
 
 
+# TODO(andrewhayden): Only used for legacy reports. Delete.
 def DumpLargestSymbols(symbols, outfile, n):
   # a list of (sym, type, size, path); sort by size.
   symbols = sorted(symbols, key=lambda x: -x[2])
   dumped = 0
   out = open(outfile, 'w')
   try:
     out.write('var largestSymbols = [\n')
     for sym, type, size, path in symbols:
       if type in ('b', 'w'):
         continue  # skip bss and weak symbols
@@ skipping 23 matching lines @@
     else:
       key = '[no path]'
     if key not in sources:
       sources[key] = {'path': path, 'symbol_count': 0, 'size': 0}
     record = sources[key]
     record['size'] += size
     record['symbol_count'] += 1
   return sources
 
 
+# TODO(andrewhayden): Only used for legacy reports. Delete.
 def DumpLargestSources(symbols, outfile, n):
   map = MakeSourceMap(symbols)
   sources = sorted(map.values(), key=lambda x: -x['size'])
   dumped = 0
   out = open(outfile, 'w')
   try:
     out.write('var largestSources = [\n')
     for record in sources:
       entry = {'size': FormatBytes(record['size']),
                'symbol_count': str(record['symbol_count']),
                'location': record['path']}
       out.write(json.dumps(entry))
       out.write(',\n')
       dumped += 1
       if dumped >= n:
         return
   finally:
     out.write('];\n')
     out.flush()
     out.close()
 
 
+# TODO(andrewhayden): Only used for legacy reports. Delete.
 def DumpLargestVTables(symbols, outfile, n):
   vtables = []
   for symbol, type, size, path in symbols:
     if 'vtable for ' in symbol:
       vtables.append({'symbol': symbol, 'path': path, 'size': size})
   vtables = sorted(vtables, key=lambda x: -x['size'])
   dumped = 0
   out = open(outfile, 'w')
   try:
     out.write('var largestVTables = [\n')
     for record in vtables:
       entry = {'size': FormatBytes(record['size']),
                'symbol': record['symbol'],
                'location': record['path']}
       out.write(json.dumps(entry))
       out.write(',\n')
       dumped += 1
       if dumped >= n:
         return
   finally:
     out.write('];\n')
     out.flush()
     out.close()
 
-
+# TODO(andrewhayden): Switch to Primiano's python-based version.
 def RunParallelAddress2Line(outfile, library, arch, jobs, verbose):
   """Run a parallel addr2line processing engine to dump and resolve symbols."""
   out_dir = os.getenv('CHROMIUM_OUT_DIR', 'out')
   build_type = os.getenv('BUILDTYPE', 'Release')
   classpath = os.path.join(out_dir, build_type, 'lib.java',
                            'binary_size_java.jar')
   cmd = ['java',
          '-classpath', classpath,
          'org.chromium.tools.binary_size.ParallelAddress2Line',
          '--disambiguate',
          '--outfile', outfile,
          '--library', library,
          '--threads', jobs]
   if verbose is True:
     cmd.append('--verbose')
   prefix = os.path.join('third_party', 'android_tools', 'ndk', 'toolchains')
   if arch == 'android-arm':
-    prefix = os.path.join(prefix, 'arm-linux-androideabi-4.7', 'prebuilt',
+    prefix = os.path.join(prefix, 'arm-linux-androideabi-4.8', 'prebuilt',
                           'linux-x86_64', 'bin', 'arm-linux-androideabi-')
     cmd.extend(['--nm', prefix + 'nm', '--addr2line', prefix + 'addr2line'])
   elif arch == 'android-mips':
-    prefix = os.path.join(prefix, 'mipsel-linux-android-4.7', 'prebuilt',
+    prefix = os.path.join(prefix, 'mipsel-linux-android-4.8', 'prebuilt',
                           'linux-x86_64', 'bin', 'mipsel-linux-android-')
     cmd.extend(['--nm', prefix + 'nm', '--addr2line', prefix + 'addr2line'])
   elif arch == 'android-x86':
-    prefix = os.path.join(prefix, 'x86-4.7', 'prebuilt',
+    prefix = os.path.join(prefix, 'x86-4.8', 'prebuilt',
                           'linux-x86_64', 'bin', 'i686-linux-android-')
     cmd.extend(['--nm', prefix + 'nm', '--addr2line', prefix + 'addr2line'])
   # else, use whatever is in PATH (don't pass --nm or --addr2line)
 
   if verbose:
     print cmd
 
   return_code = subprocess.call(cmd)
   if return_code:
     raise RuntimeError('Failed to run ParallelAddress2Line: returned ' +
@@ skipping 70 matching lines @@
                     'This argument is only valid when using --library.')
   parser.add_option('-v', dest='verbose', action='store_true',
                     help='be verbose, printing lots of status information.')
   parser.add_option('--nm-out', metavar='PATH',
                     help='keep the nm output file, and store it at the '
                     'specified path. This is useful if you want to see the '
                     'fully processed nm output after the symbols have been '
                     'mapped to source locations. By default, a tempfile is '
                     'used and is deleted when the program terminates.'
                     'This argument is only valid when using --library.')
+  parser.add_option('--legacy', action='store_true',
+                    help='emit legacy binary size report instead of modern')
   opts, args = parser.parse_args()
 
   if ((not opts.library) and (not opts.nm_in)) or (opts.library and opts.nm_in):
     parser.error('exactly one of --library or --nm-in is required')
   if (opts.nm_in):
     if opts.jobs:
       print >> sys.stderr, ('WARNING: --jobs has no effect '
                             'when used with --nm-in')
     if opts.arch:
       print >> sys.stderr, ('WARNING: --arch has no effect '
                             'when used with --nm-in')
   if not opts.destdir:
     parser.error('--destdir is required argument')
   if not opts.jobs:
     opts.jobs = '1'
   if not opts.arch:
     opts.arch = 'host-native'
 
   symbols = GetNmSymbols(opts.nm_in, opts.nm_out, opts.library, opts.arch,
                            opts.jobs, opts.verbose is True)
   if not os.path.exists(opts.destdir):
     os.makedirs(opts.destdir, 0755)
 
-  DumpTreemap(symbols, os.path.join(opts.destdir, 'treemap-dump.js'))
-  DumpLargestSymbols(symbols,
-                       os.path.join(opts.destdir, 'largest-symbols.js'), 100)
-  DumpLargestSources(symbols,
-                       os.path.join(opts.destdir, 'largest-sources.js'), 100)
-  DumpLargestVTables(symbols,
-                       os.path.join(opts.destdir, 'largest-vtables.js'), 100)
 
-  # TODO(andrewhayden): Switch to D3 for greater flexibility
-  treemap_out = os.path.join(opts.destdir, 'webtreemap')
-  if not os.path.exists(treemap_out):
-    os.makedirs(treemap_out, 0755)
-  treemap_src = os.path.join('third_party', 'webtreemap', 'src')
-  shutil.copy(os.path.join(treemap_src, 'COPYING'), treemap_out)
-  shutil.copy(os.path.join(treemap_src, 'webtreemap.js'), treemap_out)
-  shutil.copy(os.path.join(treemap_src, 'webtreemap.css'), treemap_out)
-  shutil.copy(os.path.join('tools', 'binary_size', 'template', 'index.html'),
-              opts.destdir)
+  if opts.legacy: # legacy report
+    DumpTreemap(symbols, os.path.join(opts.destdir, 'treemap-dump.js'))
+    DumpLargestSymbols(symbols,
+                         os.path.join(opts.destdir, 'largest-symbols.js'), 100)
+    DumpLargestSources(symbols,
+                         os.path.join(opts.destdir, 'largest-sources.js'), 100)
+    DumpLargestVTables(symbols,
+                         os.path.join(opts.destdir, 'largest-vtables.js'), 100)
+    treemap_out = os.path.join(opts.destdir, 'webtreemap')
+    if not os.path.exists(treemap_out):
+      os.makedirs(treemap_out, 0755)
+    treemap_src = os.path.join('third_party', 'webtreemap', 'src')
+    shutil.copy(os.path.join(treemap_src, 'COPYING'), treemap_out)
+    shutil.copy(os.path.join(treemap_src, 'webtreemap.js'), treemap_out)
+    shutil.copy(os.path.join(treemap_src, 'webtreemap.css'), treemap_out)
+    shutil.copy(os.path.join('tools', 'binary_size', 'legacy_template',
+                             'index.html'), opts.destdir)
+  else: # modern report
+    DumpCompactTree(symbols, os.path.join(opts.destdir, 'data.js'))
+    d3_out = os.path.join(opts.destdir, 'd3')
+    if not os.path.exists(d3_out):
+      os.makedirs(d3_out, 0755)
+    d3_src = os.path.join('third_party', 'd3', 'src')
+    template_src = os.path.join('tools', 'binary_size',
+                                'template')
+    shutil.copy(os.path.join(d3_src, 'LICENSE'), d3_out)
+    shutil.copy(os.path.join(d3_src, 'd3.js'), d3_out)
+    shutil.copy(os.path.join(template_src, 'index.html'), opts.destdir)
+    shutil.copy(os.path.join(template_src, 'D3SymbolTreeMap.js'), opts.destdir)
+
   if opts.verbose:
     print 'Report saved to ' + opts.destdir + '/index.html'
 
 
 if __name__ == '__main__':
-  sys.exit(main())
+  sys.exit(main())