Reland of V2 of //tools/binary_size rewrite (diffs).

tools/binary_size/map2size.py

Index: tools/binary_size/map2size.py
diff --git a/tools/binary_size/analyze.py b/tools/binary_size/map2size.py
similarity index 50%
rename from tools/binary_size/analyze.py
rename to tools/binary_size/map2size.py
index 8a4eefc44df4e2107bc8c2a60c72cbe92e1b4c77..3e3b78319be4098cd6471935e4d90b104e5ba7d0 100755
--- a/tools/binary_size/analyze.py
+++ b/tools/binary_size/map2size.py
@@ -6,35 +6,18 @@
 """Main Python API for analyzing binary size."""
 
 import argparse
-import ast
 import distutils.spawn
-import gzip
 import logging
 import os
-import re
 import subprocess
+import sys
 
+import describe
+import file_format
 import function_signature
 import helpers
-import mapfileparser
-import symbols
-
-
-# File format version for .size files.
-_SERIALIZATION_VERSION = 1
-
-
-def _OpenMaybeGz(path, mode=None):
-  """Calls `gzip.open()` if |path| ends in ".gz", otherwise calls `open()`."""
-  if path.endswith('.gz'):
-    if mode and 'w' in mode:
-      return gzip.GzipFile(path, mode, 1)
-    return gzip.open(path, mode)
-  return open(path, mode or 'r')
-
-
-def _EndsWithMaybeGz(path, suffix):
-  return path.endswith(suffix) or path.endswith(suffix + '.gz')
+import linker_map_parser
+import models
 
 
 def _IterLines(s):
@@ -49,7 +32,7 @@ def _IterLines(s):
 
 def _UnmangleRemainingSymbols(symbol_group, tool_prefix):
   """Uses c++filt to unmangle any symbols that need it."""
-  to_process = [s for s in symbol_group if s.name and s.name.startswith('_Z')]
+  to_process = [s for s in symbol_group if s.name.startswith('_Z')]
   if not to_process:
     return
 
@@ -74,7 +57,7 @@ def _NormalizeNames(symbol_group):
   """
   found_prefixes = set()
   for symbol in symbol_group:
-    if not symbol.name or symbol.name.startswith('*'):
+    if symbol.name.startswith('*'):
       # See comment in _RemoveDuplicatesAndCalculatePadding() about when this
       # can happen.
       continue
@@ -105,18 +88,17 @@ def _NormalizeNames(symbol_group):
 def _NormalizeObjectPaths(symbol_group):
   """Ensures that all paths are formatted in a useful way."""
   for symbol in symbol_group:
-    if symbol.path:
-      if symbol.path.startswith('obj/'):
-        # Convert obj/third_party/... -> third_party/...
-        symbol.path = symbol.path[4:]
-      elif symbol.path.startswith('../../'):
-        # Convert ../../third_party/... -> third_party/...
-        symbol.path = symbol.path[6:]
-      if symbol.path.endswith(')'):
-        # Convert foo/bar.a(baz.o) -> foo/bar.a/baz.o
-        start_idx = symbol.path.index('(')
-        paren_path = symbol.path[start_idx + 1:-1]
-        symbol.path = symbol.path[:start_idx] + os.path.sep + paren_path
+    if symbol.path.startswith('obj/'):
+      # Convert obj/third_party/... -> third_party/...
+      symbol.path = symbol.path[4:]
+    elif symbol.path.startswith('../../'):
+      # Convert ../../third_party/... -> third_party/...
+      symbol.path = symbol.path[6:]
+    if symbol.path.endswith(')'):
+      # Convert foo/bar.a(baz.o) -> foo/bar.a/baz.o
+      start_idx = symbol.path.index('(')
+      paren_path = symbol.path[start_idx + 1:-1]
+      symbol.path = symbol.path[:start_idx] + os.path.sep + paren_path
 
 
 def _RemoveDuplicatesAndCalculatePadding(symbol_group):
@@ -130,7 +112,7 @@ def _RemoveDuplicatesAndCalculatePadding(symbol_group):
   for i in xrange(len(all_symbols)):
     prev_symbol = all_symbols[i - 1]
     symbol = all_symbols[i]
-    if prev_symbol.section_name is not symbol.section_name:
+    if prev_symbol.section_name != symbol.section_name:
       continue
     if symbol.address > 0 and prev_symbol.address > 0:
       # Fold symbols that are at the same address (happens in nm output).
@@ -166,103 +148,7 @@ def _RemoveDuplicatesAndCalculatePadding(symbol_group):
         [s for i, s in enumerate(all_symbols) if i not in to_remove])
 
 
-def _PrintStats(result, write_func):
-  """Prints out how accurate |result| is."""
-  for section in symbols.SECTION_TO_SECTION_NAME:
-    if section == 'd':
-      expected_size = sum(v for k, v in result.section_sizes.iteritems()
-                          if k.startswith('.data'))
-    else:
-      expected_size = result.section_sizes[
-          symbols.SECTION_TO_SECTION_NAME[section]]
-
-    def one_stat(group):
-      template = ('Section %s has %.1f%% of %d bytes accounted for from '
-                  '%d symbols. %d bytes are unaccounted for. Padding '
-                  'accounts for %d bytes\n')
-      actual_size = group.size
-      count = len(group)
-      padding = group.padding
-      size_percent = 100.0 * actual_size / expected_size
-      return (template % (section, size_percent, actual_size, count,
-                          expected_size - actual_size, padding))
-
-    in_section = result.symbol_group.WhereInSection(section)
-    write_func(one_stat(in_section))
-
-    star_syms = in_section.WhereNameMatches(r'^\*')
-    attributed_syms = star_syms.Inverted().WhereHasAnyAttribution()
-    anonymous_syms = attributed_syms.Inverted()
-    if star_syms or anonymous_syms:
-      missing_size = star_syms.size + anonymous_syms.size
-      write_func(('+ Without %d merge sections and %d anonymous entries ('
-                  'accounting for %d bytes):\n') % (
-          len(star_syms),  len(anonymous_syms), missing_size))
-      write_func('+ ' + one_stat(attributed_syms))
-
-
-def _SaveResult(result, file_obj):
-  """Saves the result to the given file object."""
-  # Store one bucket per line.
-  file_obj.write('%d\n' % _SERIALIZATION_VERSION)
-  file_obj.write('%r\n' % result.section_sizes)
-  file_obj.write('%d\n' % len(result.symbol_group))
-  prev_section_name = None
-  # Store symbol fields as tab-separated.
-  # Store only non-derived fields.
-  for symbol in result.symbol_group:
-    if symbol.section_name != prev_section_name:
-      file_obj.write('%s\n' % symbol.section_name)
-      prev_section_name = symbol.section_name
-    # Don't write padding nor name since these are derived values.
-    file_obj.write('%x\t%x\t%s\t%s\n' % (
-        symbol.address, symbol.size_without_padding,
-        symbol.function_signature or symbol.name or '',
-        symbol.path or ''))
-
-
-def _LoadResults(file_obj):
-  """Loads a result from the given file."""
-  lines = iter(file_obj)
-  actual_version = int(next(lines))
-  assert actual_version == _SERIALIZATION_VERSION, (
-      'Version mismatch. Need to write some upgrade code.')
-
-  section_sizes = ast.literal_eval(next(lines))
-  num_syms = int(next(lines))
-  symbol_list = [None] * num_syms
-  section_name = None
-  for i in xrange(num_syms):
-    line = next(lines)[:-1]
-    if '\t' not in line:
-      section_name = intern(line)
-      line = next(lines)[:-1]
-    new_sym = symbols.Symbol.__new__(symbols.Symbol)
-    parts = line.split('\t')
-    new_sym.section_name = section_name
-    new_sym.address = int(parts[0], 16)
-    new_sym.size = int(parts[1], 16)
-    new_sym.name = parts[2] or None
-    new_sym.path = parts[3] or None
-    new_sym.padding = 0  # Derived
-    new_sym.function_signature = None  # Derived
-    symbol_list[i] = new_sym
-
-  # Recompute derived values (padding and function names).
-  result = mapfileparser.ParseResult(symbol_list, section_sizes)
-  logging.info('Calculating padding')
-  _RemoveDuplicatesAndCalculatePadding(result.symbol_group)
-  logging.info('Deriving signatures')
-  # Re-parse out function parameters.
-  _NormalizeNames(result.symbol_group.WhereInSection('t'))
-  return result
-
-
 def AddOptions(parser):
-  parser.add_argument('input_file',
-                      help='Path to input file. Can be a linker .map file, an '
-                           'unstripped binary, or a saved result from '
-                           'analyze.py')
   parser.add_argument('--tool-prefix', default='',
                       help='Path prefix for c++filt.')
   parser.add_argument('--output-directory',
@@ -286,7 +172,8 @@ def _DetectToolPrefix(tool_prefix, input_file, output_directory=None):
       with open(build_vars_path) as f:
         build_vars = dict(l.rstrip().split('=', 1) for l in f if '=' in l)
       logging.debug('Found --tool-prefix from build_vars.txt')
-      tool_prefix = build_vars['android_tool_prefix']
+      tool_prefix = os.path.join(output_directory,
+                                 build_vars['android_tool_prefix'])
 
   if os.path.sep not in tool_prefix:
     full_path = distutils.spawn.find_executable(tool_prefix + 'c++filt')
@@ -299,58 +186,63 @@ def _DetectToolPrefix(tool_prefix, input_file, output_directory=None):
   return tool_prefix
 
 
-def AnalyzeWithArgs(args):
-  return Analyze(args.input_file, args.output_directory, args.tool_prefix)
+def AnalyzeWithArgs(args, input_path):
+  return Analyze(input_path, args.output_directory, args.tool_prefix)
 
 
 def Analyze(path, output_directory=None, tool_prefix=''):
-  if _EndsWithMaybeGz(path, '.size'):
-    logging.info('Loading cached results.')
-    with _OpenMaybeGz(path) as f:
-      result = _LoadResults(f)
-  elif not _EndsWithMaybeGz(path, '.map'):
+  if file_format.EndsWithMaybeGz(path, '.size'):
+    logging.debug('Loading results from: %s', path)
+    size_info = file_format.LoadSizeInfo(path)
+    # Recompute derived values (padding and function names).
+    logging.info('Calculating padding')
+    _RemoveDuplicatesAndCalculatePadding(size_info.symbols)
+    logging.info('Deriving signatures')
+    # Re-parse out function parameters.
+    _NormalizeNames(size_info.symbols.WhereInSection('t'))
+    return size_info
+  elif not file_format.EndsWithMaybeGz(path, '.map'):
     raise Exception('Expected input to be a .map or a .size')
   else:
     # Verify tool_prefix early.
     tool_prefix = _DetectToolPrefix(tool_prefix, path, output_directory)
 
-    with _OpenMaybeGz(path) as map_file:
-      result = mapfileparser.MapFileParser().Parse(map_file)
+    with file_format.OpenMaybeGz(path) as map_file:
+      size_info = linker_map_parser.MapFileParser().Parse(map_file)
 
     # Map file for some reason doesn't unmangle all names.
     logging.info('Calculating padding')
-    _RemoveDuplicatesAndCalculatePadding(result.symbol_group)
+    _RemoveDuplicatesAndCalculatePadding(size_info.symbols)
     # Unmangle prints its own log statement.
-    _UnmangleRemainingSymbols(result.symbol_group, tool_prefix)
+    _UnmangleRemainingSymbols(size_info.symbols, tool_prefix)
     # Resolve paths prints its own log statement.
     logging.info('Normalizing names')
-    _NormalizeNames(result.symbol_group)
+    _NormalizeNames(size_info.symbols)
     logging.info('Normalizing paths')
-    _NormalizeObjectPaths(result.symbol_group)
+    _NormalizeObjectPaths(size_info.symbols)
 
   if logging.getLogger().isEnabledFor(logging.INFO):
-    _PrintStats(result, lambda l: logging.info(l.rstrip()))
-  logging.info('Finished analyzing %d symbols', len(result.symbol_group))
-  return result
+    for line in describe.DescribeSizeInfoCoverage(size_info):
+      logging.info(line)
+  logging.info('Finished analyzing %d symbols', len(size_info.symbols))
+  return size_info
 
 
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--output', required=True,
-                      help='Path to store results. Must end in .size or '
-                           '.size.gz')
+def main(argv):
+  parser = argparse.ArgumentParser(argv)
+  parser.add_argument('input_file', help='Path to input .map file.')
+  parser.add_argument('output_file', help='Path to output .size(.gz) file.')
   AddOptions(parser)
-  args = helpers.AddCommonOptionsAndParseArgs(parser)
-  if not _EndsWithMaybeGz(args.output, '.size'):
-    raise Exception('--output must end with .size or .size.gz')
+  args = helpers.AddCommonOptionsAndParseArgs(parser, argv)
+  if not file_format.EndsWithMaybeGz(args.output_file, '.size'):
+    parser.error('output_file must end with .size or .size.gz')
 
-  result = AnalyzeWithArgs(args)
-  logging.info('Saving result to %s', args.output)
-  with _OpenMaybeGz(args.output, 'wb') as f:
-    _SaveResult(result, f)
+  size_info = AnalyzeWithArgs(args, args.input_file)
+  logging.info('Saving result to %s', args.output_file)
+  file_format.SaveSizeInfo(size_info, args.output_file)
 
   logging.info('Done')
 
 
 if __name__ == '__main__':
-  main()
+  sys.exit(main(sys.argv))