//tools/binary_size: Record packed .rel.dyn section size

tools/binary_size/libsupersize/archive.py

 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Main Python API for analyzing binary size."""
 
 import argparse
 import calendar
 import collections
 import datetime
 import gzip
 import logging
 import os
+import posixpath
 import re
 import subprocess
 import sys
+import tempfile
+import zipfile
 
 import describe
 import file_format
 import function_signature
 import helpers
 import linker_map_parser
 import models
 import ninja_parser
 import paths
 
@@ skipping 171 matching lines @@
         'Symbol has negative size (likely not sorted propertly): '
         '%r\nprev symbol: %r' % (symbol, prev_symbol))
 
 
 def _ClusterSymbols(symbols):
   """Returns a new list of symbols with some symbols moved into groups.
 
   Groups include:
    * Symbols that have [clone] in their name (created by compiler optimization).
    * Star symbols (such as "** merge strings", and "** symbol gap")
+
+  To view created groups:
+    Print(size_info.symbols.Filter(lambda s: s.IsGroup()), recursive=True)
   """
   # http://unix.stackexchange.com/questions/223013/function-symbol-gets-part-suffix-after-compilation
   # Example name suffixes:
-  #     [clone .part.322]
-  #     [clone .isra.322]
-  #     [clone .constprop.1064]
+  #     [clone .part.322]  # GCC
+  #     [clone .isra.322]  # GCC
+  #     [clone .constprop.1064]  # GCC
+  #     [clone .11064]  # clang
 
   # Step 1: Create name map, find clones, collect star syms into replacements.
   logging.debug('Creating name -> symbol map')
   clone_indices = []
   indices_by_full_name = {}
   # (name, full_name) -> [(index, sym),...]
   replacements_by_name = collections.defaultdict(list)
   for i, symbol in enumerate(symbols):
     if symbol.name.startswith('**'):
       # "symbol gap 3" -> "symbol gaps"
@@ skipping 120 matching lines @@
   size_info = models.SizeInfo(section_sizes, raw_symbols)
 
   # Name normalization not strictly required, but makes for smaller files.
   if raw_only:
     logging.info('Normalizing symbol names')
     _NormalizeNames(size_info.raw_symbols)
   else:
     _PostProcessSizeInfo(size_info)
 
   if logging.getLogger().isEnabledFor(logging.DEBUG):
+    # Padding is reported in size coverage logs.
+    if raw_only:
+      _CalculatePadding(size_info.raw_symbols)
     for line in describe.DescribeSizeInfoCoverage(size_info):
       logging.info(line)
   logging.info('Recorded info for %d symbols', len(size_info.raw_symbols))
   return size_info
 
 
 def _DetectGitRevision(directory):
   try:
     git_rev = subprocess.check_output(
         ['git', '-C', directory, 'rev-parse', 'HEAD'])
@@ skipping 28 matching lines @@
   with open(args_path) as f:
     for l in f:
       # Strips #s even if within string literal. Not a problem in practice.
       parts = l.split('#')[0].split('=')
       if len(parts) != 2:
         continue
       args[parts[0].strip()] = parts[1].strip()
   return ["%s=%s" % x for x in sorted(args.iteritems())]
 
 
+def _SectionSizesFromApk(apk_file, elf_file, build_id, lazy_paths):
+  with zipfile.ZipFile(apk_file) as apk, \
+       tempfile.NamedTemporaryFile() as f:
+    target = os.path.basename(elf_file)
+    target_info = next((f for f in apk.infolist()
+                        if posixpath.basename(f.filename) == target), None)
+    assert target_info, (
+        'Could not find apk entry for %s in %s' % (target, apk_file))
+    f.write(apk.read(target_info))
+    f.flush()
+    apk_build_id = BuildIdFromElf(f.name, lazy_paths.tool_prefix)
+    assert apk_build_id == build_id, (
+        'BuildID for %s within %s did not match the one at %s' %
+        (target_info.filename, apk_file, elf_file))
+    return _SectionSizesFromElf(f.name, lazy_paths.tool_prefix)
+
+
 def AddArguments(parser):
   parser.add_argument('size_file', help='Path to output .size file.')
-  parser.add_argument('--elf-file', required=True,
+  parser.add_argument('--apk-file',
+                      help='.apk file to measure. When set, --elf-file will be '
+                            'derived (if unset). Providing the .apk allows '
+                            'for the size of packed relocations to be recorded')
+  parser.add_argument('--elf-file',
                       help='Path to input ELF file. Currently used for '
-                           'capturing metadata. Pass "" to skip '
-                           'metadata collection.')
+                           'capturing metadata.')
   parser.add_argument('--map-file',
                       help='Path to input .map(.gz) file. Defaults to '
-                           '{{elf_file}}.map(.gz)?')
+                           '{{elf_file}}.map(.gz)?. If given without '
+                           '--elf-file, no size metadata will be recorded.')
   parser.add_argument('--no-source-paths', action='store_true',
                       help='Do not use .ninja files to map '
                            'object_path -> source_path')
   parser.add_argument('--tool-prefix', default='',
                       help='Path prefix for c++filt.')
   parser.add_argument('--output-directory',
                       help='Path to the root build directory.')
 
 
 def Run(args, parser):
   if not args.size_file.endswith('.size'):
     parser.error('size_file must end with .size')
 
+  any_input = args.apk_file or args.elf_file or args.map_file
+  if not any_input:
+    parser.error('Most pass at least one of --apk-file, --elf-file, --map-file')
+  lazy_paths = paths.LazyPaths(args=args, input_file=any_input)
+
+  elf_file = args.elf_file
+  if args.apk_file and not elf_file:
+    with zipfile.ZipFile(args.apk_file) as z:
+      lib_infos = [f for f in z.infolist()
+                   if f.filename.endswith('.so') and f.file_size > 0]
+    assert lib_infos, 'APK has not .so files to measure.'

[estevenson, 2017/04/12 16:54:42]

nit: awkward wording.

[agrieve, 2017/04/12 19:37:46]

Done.

+    # TODO(agrieve): Add support for multiple .so files, and take into account
+    #     secondary architectures.

[estevenson, 2017/04/12 16:54:42]

_SectionSizesFromApk() would fail for 64 bit monochrome right now, correct?
Might be worth it to add this note to the README, though I'd assume it's pretty
uncommon for people to regularly build 64 bit chrome..

I wasn't able to verify this properly, I ran:

tools/binary_size/supersize archive output.size --elf-file
out/Release-64/lib.unstripped/libmonochrome.so --apk-file
out/Release-64/apks/MonochromePublic.apk

and hit the "One or more source file paths could not be found" assert.

Most errors were v8 like this:
Could not find source path for obj/v8/v8_base/assembler-arm64.o

And some of these:
W   4201 no obj basename for obj/third_party/libvpx/libvpx.a(loopfilter_neon.o)
W   4201 Could not find source path for
obj/third_party/libvpx/libvpx.a(loopfilter_neon.o)

[agrieve, 2017/04/12 19:37:45]

Didn't hit any of those other errors, but good catch on not testing with arm64
(nor with crazy.).

I've not tested on both ChromeModernPublic, and with 64-bit monochrome_public.

+    apk_so_path = max(lib_infos, key=lambda x:x.file_size).filename
+    elf_file = os.path.join(lazy_paths.output_directory, 'lib.unstripped',
+                            os.path.basename(apk_so_path))
+    logging.debug('Detected --elf-file=%s', elf_file)
+
   if args.map_file:
     if (not args.map_file.endswith('.map')
         and not args.map_file.endswith('.map.gz')):
       parser.error('Expected --map-file to end with .map or .map.gz')
     map_file_path = args.map_file
   else:
-    map_file_path = args.elf_file + '.map'
+    map_file_path = elf_file + '.map'
     if not os.path.exists(map_file_path):
       map_file_path += '.gz'
     if not os.path.exists(map_file_path):
       parser.error('Could not find .map(.gz)? file. Use --map-file.')
 
-  lazy_paths = paths.LazyPaths(args=args, input_file=args.elf_file)
   metadata = None
-  if args.elf_file:
+  if elf_file:
     logging.debug('Constructing metadata')
-    git_rev = _DetectGitRevision(os.path.dirname(args.elf_file))
-    build_id = BuildIdFromElf(args.elf_file, lazy_paths.tool_prefix)
+    git_rev = _DetectGitRevision(os.path.dirname(elf_file))
+    build_id = BuildIdFromElf(elf_file, lazy_paths.tool_prefix)
     timestamp_obj = datetime.datetime.utcfromtimestamp(os.path.getmtime(
-        args.elf_file))
+        elf_file))
     timestamp = calendar.timegm(timestamp_obj.timetuple())
     gn_args = _ParseGnArgs(os.path.join(lazy_paths.output_directory, 'args.gn'))
 
     def relative_to_out(path):
       return os.path.relpath(path, lazy_paths.VerifyOutputDirectory())
 
     metadata = {
         models.METADATA_GIT_REVISION: git_rev,
         models.METADATA_MAP_FILENAME: relative_to_out(map_file_path),
-        models.METADATA_ELF_FILENAME: relative_to_out(args.elf_file),
+        models.METADATA_ELF_FILENAME: relative_to_out(elf_file),
         models.METADATA_ELF_MTIME: timestamp,
         models.METADATA_ELF_BUILD_ID: build_id,
         models.METADATA_GN_ARGS: gn_args,
     }
 
   size_info = CreateSizeInfo(map_file_path, lazy_paths,
                              no_source_paths=args.no_source_paths,
                              raw_only=True)
 
   if metadata:
     size_info.metadata = metadata
     logging.debug('Validating section sizes')
-    elf_section_sizes = _SectionSizesFromElf(args.elf_file,
-                                             lazy_paths.tool_prefix)
+    elf_section_sizes = _SectionSizesFromElf(elf_file, lazy_paths.tool_prefix)
     for k, v in elf_section_sizes.iteritems():
       assert v == size_info.section_sizes.get(k), (
           'ELF file and .map file do not match.')
 
+    if args.apk_file:
+      logging.debug('Extracting section sizes from .so within .apk')
+      unpacked_rel_dyn = size_info.section_sizes.get('.rel.dyn', 0)
+      if not unpacked_rel_dyn:
+        logging.warning('Section .rel.dyn did not exist')
+      # TODO(agrieve): Extracting the .so is slow. Do it on a background thread.
+      size_info.section_sizes = _SectionSizesFromApk(
+          args.apk_file, elf_file, build_id, lazy_paths)
+      size_info.section_sizes['.rel.dyn (unpacked)'] = unpacked_rel_dyn
+
   logging.info('Recording metadata: \n  %s',
                '\n  '.join(describe.DescribeMetadata(size_info.metadata)))
   logging.info('Saving result to %s', args.size_file)
   file_format.SaveSizeInfo(size_info, args.size_file)
   logging.info('Done')