supersize: Don't cluster by default. Make Diff() accept only SizeInfo.

tools/binary_size/libsupersize/archive.py

 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Main Python API for analyzing binary size."""
 
 import argparse
 import calendar
 import collections
 import datetime
@@ skipping 20 matching lines @@
 
 def _OpenMaybeGz(path, mode=None):
   """Calls `gzip.open()` if |path| ends in ".gz", otherwise calls `open()`."""
   if path.endswith('.gz'):
     if mode and 'w' in mode:
       return gzip.GzipFile(path, mode, 1)
     return gzip.open(path, mode)
   return open(path, mode or 'r')
 
 
-def _StripLinkerAddedSymbolPrefixes(symbols):
+def _StripLinkerAddedSymbolPrefixes(raw_symbols):
   """Removes prefixes sometimes added to symbol names during link
 
   Removing prefixes make symbol names match up with those found in .o files.
   """
-  for symbol in symbols:
+  for symbol in raw_symbols:
     name = symbol.name
     if name.startswith('startup.'):
       symbol.flags |= models.FLAG_STARTUP
       symbol.name = name[8:]
     elif name.startswith('unlikely.'):
       symbol.flags |= models.FLAG_UNLIKELY
       symbol.name = name[9:]
     elif name.startswith('rel.local.'):
       symbol.flags |= models.FLAG_REL_LOCAL
       symbol.name = name[10:]
     elif name.startswith('rel.'):
       symbol.flags |= models.FLAG_REL
       symbol.name = name[4:]
 
 
-def _UnmangleRemainingSymbols(symbols, tool_prefix):
+def _UnmangleRemainingSymbols(raw_symbols, tool_prefix):
   """Uses c++filt to unmangle any symbols that need it."""
-  to_process = [s for s in symbols if s.name.startswith('_Z')]
+  to_process = [s for s in raw_symbols if s.name.startswith('_Z')]
   if not to_process:
     return
 
   logging.info('Unmangling %d names', len(to_process))
   proc = subprocess.Popen([tool_prefix + 'c++filt'], stdin=subprocess.PIPE,
                           stdout=subprocess.PIPE)
   stdout = proc.communicate('\n'.join(s.name for s in to_process))[0]
   assert proc.returncode == 0
 
   for i, line in enumerate(stdout.splitlines()):
@@ skipping 79 matching lines @@
 
 def _SourcePathForObjectPath(object_path, source_mapper):
   # We don't have source info for prebuilt .a files.
   if not os.path.isabs(object_path) and not object_path.startswith('..'):
     source_path = source_mapper.FindSourceForPath(object_path)
     if source_path:
       return _NormalizeSourcePath(source_path)
   return ''
 
 
-def _ExtractSourcePaths(symbols, source_mapper):
+def _ExtractSourcePaths(raw_symbols, source_mapper):
   """Fills in the |source_path| attribute."""
   logging.debug('Parsed %d .ninja files.', source_mapper.parsed_file_count)
-  for symbol in symbols:
+  for symbol in raw_symbols:
     object_path = symbol.object_path
     if object_path and not symbol.source_path:
       symbol.source_path = _SourcePathForObjectPath(object_path, source_mapper)
 
 
 def _ComputeAnscestorPath(path_list):
   """Returns the common anscestor of the given paths."""
   # Ignore missing paths.
   path_list = [p for p in path_list if p]
   prefix = os.path.commonprefix(path_list)
   # Put the path count as a subdirectory to allow for better grouping when
   # path-based breakdowns.
   if not prefix:
     if len(path_list) < 2:
       return ''
     return os.path.join('{shared}', str(len(path_list)))
   if prefix == path_list[0]:
     return prefix
   assert len(path_list) > 1, 'path_list: ' + repr(path_list)
   return os.path.join(os.path.dirname(prefix), '{shared}', str(len(path_list)))
 
 
 # This must normalize object paths at the same time because normalization
 # needs to occur before finding common ancestor.
 def _ComputeAnscestorPathsAndNormalizeObjectPaths(
-    symbols, object_paths_by_name, source_mapper):
+    raw_symbols, object_paths_by_name, source_mapper):
   num_found_paths = 0
   num_unknown_names = 0
   num_path_mismatches = 0
   num_unmatched_aliases = 0
-  for symbol in symbols:
+  for symbol in raw_symbols:
     name = symbol.name
     if (symbol.IsBss() or
         not name or
         name[0] in '*.' or  # e.g. ** merge symbols, .Lswitch.table
         name == 'startup'):
       symbol.object_path = _NormalizeObjectPath(symbol.object_path)
       continue
 
     object_paths = object_paths_by_name.get(name)
     if object_paths:
@@ skipping 28 matching lines @@
 
     object_paths = [_NormalizeObjectPath(p) for p in object_paths]
     symbol.object_path = _ComputeAnscestorPath(object_paths)
 
   logging.debug('Cross-referenced %d symbols with nm output. '
                 'num_unknown_names=%d num_path_mismatches=%d '
                 'num_unused_aliases=%d', num_found_paths, num_unknown_names,
                 num_path_mismatches, num_unmatched_aliases)
 
 
-def _DiscoverMissedObjectPaths(symbols, elf_object_paths):
+def _DiscoverMissedObjectPaths(raw_symbols, elf_object_paths):
   # Missing object paths are caused by .a files added by -l flags, which are not
   # listed as explicit inputs within .ninja rules.
   parsed_inputs = set(elf_object_paths)
   missed_inputs = set()
-  for symbol in symbols:
+  for symbol in raw_symbols:
     path = symbol.object_path
     if path.endswith(')'):
       # Convert foo/bar.a(baz.o) -> foo/bar.a
       path = path[:path.index('(')]
     if path and path not in parsed_inputs:
       missed_inputs.add(path)
   return missed_inputs
 
 
 def _CalculatePadding(symbols):
@@ skipping 32 matching lines @@
       # Should not happen.
       logging.warning('Large padding of %d between:\n  A) %r\n  B) %r' % (
                       padding, prev_symbol, symbol))
     symbol.padding = padding
     symbol.size += padding
     assert symbol.size >= 0, (
         'Symbol has negative size (likely not sorted propertly): '
         '%r\nprev symbol: %r' % (symbol, prev_symbol))
 
 
-def _ClusterSymbols(symbols):
-  """Returns a new list of symbols with some symbols moved into groups.
-
-  Groups include:
-   * Symbols that have [clone] in their name (created by compiler optimization).
-   * Star symbols (such as "** merge strings", and "** symbol gap")
-
-  To view created groups:
-    Print(size_info.symbols.Filter(lambda s: s.IsGroup()), recursive=True)
-  """
-  # http://unix.stackexchange.com/questions/223013/function-symbol-gets-part-suffix-after-compilation
-  # Example name suffixes:
-  #     [clone .part.322]  # GCC
-  #     [clone .isra.322]  # GCC
-  #     [clone .constprop.1064]  # GCC
-  #     [clone .11064]  # clang
-
-  # Step 1: Create name map, find clones, collect star syms into replacements.
-  logging.debug('Creating name -> symbol map')
-  clone_indices = []
-  indices_by_full_name = {}
-  # (name, full_name) -> [(index, sym),...]
-  replacements_by_name = collections.defaultdict(list)
-  for i, symbol in enumerate(symbols):
-    if symbol.name.startswith('**'):
-      # "symbol gap 3" -> "symbol gaps"
-      name = re.sub(r'\s+\d+$', 's', symbol.name)
-      replacements_by_name[(name, None)].append((i, symbol))
-    elif symbol.full_name:
-      if symbol.full_name.endswith(']') and ' [clone ' in symbol.full_name:
-        clone_indices.append(i)
-      else:
-        indices_by_full_name[symbol.full_name] = i
-
-  # Step 2: Collect same-named clone symbols.
-  logging.debug('Grouping all clones')
-  group_names_by_index = {}
-  for i in clone_indices:
-    symbol = symbols[i]
-    # Multiple attributes could exist, so search from left-to-right.
-    stripped_name = symbol.name[:symbol.name.index(' [clone ')]
-    stripped_full_name = symbol.full_name[:symbol.full_name.index(' [clone ')]
-    name_tup = (stripped_name, stripped_full_name)
-    replacement_list = replacements_by_name[name_tup]
-
-    if not replacement_list:
-      # First occurance, check for non-clone symbol.
-      non_clone_idx = indices_by_full_name.get(stripped_name)
-      if non_clone_idx is not None:
-        non_clone_symbol = symbols[non_clone_idx]
-        replacement_list.append((non_clone_idx, non_clone_symbol))
-        group_names_by_index[non_clone_idx] = stripped_name
-
-    replacement_list.append((i, symbol))
-    group_names_by_index[i] = stripped_name
-
-  # Step 3: Undo clustering when length=1.
-  # Removing these groups means Diff() logic must know about [clone] suffix.
-  to_clear = []
-  for name_tup, replacement_list in replacements_by_name.iteritems():
-    if len(replacement_list) == 1:
-      to_clear.append(name_tup)
-  for name_tup in to_clear:
-    del replacements_by_name[name_tup]
-
-  # Step 4: Replace first symbol from each cluster with a SymbolGroup.
-  before_symbol_count = sum(len(x) for x in replacements_by_name.itervalues())
-  logging.debug('Creating %d symbol groups from %d symbols. %d clones had only '
-                'one symbol.', len(replacements_by_name), before_symbol_count,
-                len(to_clear))
-
-  len_delta = len(replacements_by_name) - before_symbol_count
-  grouped_symbols = [None] * (len(symbols) + len_delta)
-  dest_index = 0
-  src_index = 0
-  seen_names = set()
-  replacement_names_by_index = {}
-  for name_tup, replacement_list in replacements_by_name.iteritems():
-    for tup in replacement_list:
-      replacement_names_by_index[tup[0]] = name_tup
-
-  sorted_items = replacement_names_by_index.items()
-  sorted_items.sort(key=lambda tup: tup[0])
-  for index, name_tup in sorted_items:
-    count = index - src_index
-    grouped_symbols[dest_index:dest_index + count] = (
-        symbols[src_index:src_index + count])
-    src_index = index + 1
-    dest_index += count
-    if name_tup not in seen_names:
-      seen_names.add(name_tup)
-      group_symbols = [tup[1] for tup in replacements_by_name[name_tup]]
-      grouped_symbols[dest_index] = models.SymbolGroup(
-          group_symbols, name=name_tup[0], full_name=name_tup[1],
-          section_name=group_symbols[0].section_name)
-      dest_index += 1
-
-  assert len(grouped_symbols[dest_index:None]) == len(symbols[src_index:None])
-  grouped_symbols[dest_index:None] = symbols[src_index:None]
-  logging.debug('Finished making groups.')
-  return grouped_symbols
-
-
-def _AddSymbolAliases(symbols, aliases_by_address):
+def _AddSymbolAliases(raw_symbols, aliases_by_address):
   # Step 1: Create list of (index_of_symbol, name_list).
   logging.debug('Creating alias list')
   replacements = []
   num_new_symbols = 0
-  for i, s in enumerate(symbols):
+  for i, s in enumerate(raw_symbols):
     # Don't alias padding-only symbols (e.g. ** symbol gap)
     if s.size_without_padding == 0:
       continue
     name_list = aliases_by_address.get(s.address)
     if name_list:
       if s.name not in name_list:
         logging.warning('Name missing from aliases: %s %s', s.name, name_list)
         continue
       replacements.append((i, name_list))
       num_new_symbols += len(name_list) - 1
 
   # Step 2: Create new symbols as siblings to each existing one.
   logging.debug('Creating %d aliases', num_new_symbols)
-  src_cursor_end = len(symbols)
-  symbols += [None] * num_new_symbols
-  dst_cursor_end = len(symbols)
+  src_cursor_end = len(raw_symbols)
+  raw_symbols += [None] * num_new_symbols
+  dst_cursor_end = len(raw_symbols)
   for src_index, name_list in reversed(replacements):
     # Copy over symbols that come after the current one.
     chunk_size = src_cursor_end - src_index - 1
     dst_cursor_end -= chunk_size
     src_cursor_end -= chunk_size
-    symbols[dst_cursor_end:dst_cursor_end + chunk_size] = (
-        symbols[src_cursor_end:src_cursor_end + chunk_size])
-    sym = symbols[src_index]
+    raw_symbols[dst_cursor_end:dst_cursor_end + chunk_size] = (
+        raw_symbols[src_cursor_end:src_cursor_end + chunk_size])
+    sym = raw_symbols[src_index]
     src_cursor_end -= 1
 
     # Create aliases (does not bother reusing the existing symbol).
     aliases = [None] * len(name_list)
     for i, name in enumerate(name_list):
       aliases[i] = models.Symbol(
           sym.section_name, sym.size, address=sym.address, name=name,
           aliases=aliases)
 
     dst_cursor_end -= len(aliases)
-    symbols[dst_cursor_end:dst_cursor_end + len(aliases)] = aliases
+    raw_symbols[dst_cursor_end:dst_cursor_end + len(aliases)] = aliases
 
   assert dst_cursor_end == src_cursor_end
 
 
 def LoadAndPostProcessSizeInfo(path):
   """Returns a SizeInfo for the given |path|."""
   logging.debug('Loading results from: %s', path)
   size_info = file_format.LoadSizeInfo(path)
   _PostProcessSizeInfo(size_info)
   return size_info
 
 
 def _PostProcessSizeInfo(size_info):
   logging.info('Normalizing symbol names')
-  _NormalizeNames(size_info.raw_symbols)
+  _NormalizeNames(size_info.symbols)
   logging.info('Calculating padding')
-  _CalculatePadding(size_info.raw_symbols)
-  logging.info('Grouping decomposed functions')
-  size_info.symbols = models.SymbolGroup(
-      _ClusterSymbols(size_info.raw_symbols))
-  logging.info('Processed %d symbols', len(size_info.raw_symbols))
+  _CalculatePadding(size_info.symbols)
+  logging.info('Processed %d symbols', len(size_info.symbols))
 
 
 def CreateMetadata(map_path, elf_path, apk_path, tool_prefix, output_directory):
   metadata = None
   if elf_path:
     logging.debug('Constructing metadata')
     git_rev = _DetectGitRevision(os.path.dirname(elf_path))
     architecture = _ArchFromElf(elf_path, tool_prefix)
     build_id = BuildIdFromElf(elf_path, tool_prefix)
     timestamp_obj = datetime.datetime.utcfromtimestamp(os.path.getmtime(
@@ skipping 112 matching lines @@
                     len(object_paths_by_name),
                     len(elf_object_paths) + len(missed_object_paths))
       _ComputeAnscestorPathsAndNormalizeObjectPaths(
           raw_symbols, object_paths_by_name, source_mapper)
 
   if not elf_path or not output_directory:
     logging.info('Normalizing object paths.')
     for symbol in raw_symbols:
       symbol.object_path = _NormalizeObjectPath(symbol.object_path)
 
-  size_info = models.SizeInfo(section_sizes, raw_symbols)
+  size_info = models.SizeInfo(section_sizes, models.SymbolGroup(raw_symbols))
 
   # Name normalization not strictly required, but makes for smaller files.
   if raw_only:
     logging.info('Normalizing symbol names')
-    _NormalizeNames(size_info.raw_symbols)
+    _NormalizeNames(size_info.symbols)
   else:
     _PostProcessSizeInfo(size_info)
 
   if logging.getLogger().isEnabledFor(logging.DEBUG):
     # Padding is reported in size coverage logs.
     if raw_only:
-      _CalculatePadding(size_info.raw_symbols)
+      _CalculatePadding(size_info.symbols)
     for line in describe.DescribeSizeInfoCoverage(size_info):
       logging.info(line)
-  logging.info('Recorded info for %d symbols', len(size_info.raw_symbols))
+  logging.info('Recorded info for %d symbols', len(size_info.symbols))
   return size_info
 
 
 def _DetectGitRevision(directory):
   try:
     git_rev = subprocess.check_output(
         ['git', '-C', directory, 'rev-parse', 'HEAD'])
     return git_rev.rstrip()
   except Exception:
     logging.warning('Failed to detect git revision for file metadata.')
@@ skipping 148 matching lines @@
           logging.warning('Packed section not present: %s', packed_section_name)
         else:
           size_info.section_sizes['%s (unpacked)' % packed_section_name] = (
               unstripped_section_sizes.get(packed_section_name))
 
   logging.info('Recording metadata: \n  %s',
                '\n  '.join(describe.DescribeMetadata(size_info.metadata)))
   logging.info('Saving result to %s', args.size_file)
   file_format.SaveSizeInfo(size_info, args.size_file)
   logging.info('Done')