//tools/binary_size: Record packed .rel.dyn section size

tools/binary_size/libsupersize/archive.py

Index: tools/binary_size/libsupersize/archive.py
diff --git a/tools/binary_size/libsupersize/archive.py b/tools/binary_size/libsupersize/archive.py
index 0991f0b8c6be2af46dd509063ffe273bc5130acb..a049b60297ed404c0182612251df556496c36f37 100644
--- a/tools/binary_size/libsupersize/archive.py
+++ b/tools/binary_size/libsupersize/archive.py
@@ -11,9 +11,12 @@ import datetime
 import gzip
 import logging
 import os
+import posixpath
 import re
 import subprocess
 import sys
+import tempfile
+import zipfile
 
 import describe
 import file_format
@@ -205,12 +208,16 @@ def _ClusterSymbols(symbols):
   Groups include:
    * Symbols that have [clone] in their name (created by compiler optimization).
    * Star symbols (such as "** merge strings", and "** symbol gap")
+
+  To view created groups:
+    Print(size_info.symbols.Filter(lambda s: s.IsGroup()), recursive=True)
   """
   # http://unix.stackexchange.com/questions/223013/function-symbol-gets-part-suffix-after-compilation
   # Example name suffixes:
-  #     [clone .part.322]
-  #     [clone .isra.322]
-  #     [clone .constprop.1064]
+  #     [clone .part.322]  # GCC
+  #     [clone .isra.322]  # GCC
+  #     [clone .constprop.1064]  # GCC
+  #     [clone .11064]  # clang
 
   # Step 1: Create name map, find clones, collect star syms into replacements.
   logging.debug('Creating name -> symbol map')
@@ -351,6 +358,9 @@ def CreateSizeInfo(map_path, lazy_paths=None, no_source_paths=False,
     _PostProcessSizeInfo(size_info)
 
   if logging.getLogger().isEnabledFor(logging.DEBUG):
+    # Padding is reported in size coverage logs.
+    if raw_only:
+      _CalculatePadding(size_info.raw_symbols)
     for line in describe.DescribeSizeInfoCoverage(size_info):
       logging.info(line)
   logging.info('Recorded info for %d symbols', len(size_info.raw_symbols))
@@ -399,15 +409,36 @@ def _ParseGnArgs(args_path):
   return ["%s=%s" % x for x in sorted(args.iteritems())]
 
 
+def _SectionSizesFromApk(apk_file, elf_file, build_id, lazy_paths):
+  with zipfile.ZipFile(apk_file) as apk, \
+       tempfile.NamedTemporaryFile() as f:
+    target = os.path.basename(elf_file)
+    target_info = next((f for f in apk.infolist()
+                        if posixpath.basename(f.filename) == target), None)
+    assert target_info, (
+        'Could not find apk entry for %s in %s' % (target, apk_file))
+    f.write(apk.read(target_info))
+    f.flush()
+    apk_build_id = BuildIdFromElf(f.name, lazy_paths.tool_prefix)
+    assert apk_build_id == build_id, (
+        'BuildID for %s within %s did not match the one at %s' %
+        (target_info.filename, apk_file, elf_file))
+    return _SectionSizesFromElf(f.name, lazy_paths.tool_prefix)
+
+
 def AddArguments(parser):
   parser.add_argument('size_file', help='Path to output .size file.')
-  parser.add_argument('--elf-file', required=True,
+  parser.add_argument('--apk-file',
+                      help='.apk file to measure. When set, --elf-file will be '
+                            'derived (if unset). Providing the .apk allows '
+                            'for the size of packed relocations to be recorded')
+  parser.add_argument('--elf-file',
                       help='Path to input ELF file. Currently used for '
-                           'capturing metadata. Pass "" to skip '
-                           'metadata collection.')
+                           'capturing metadata.')
   parser.add_argument('--map-file',
                       help='Path to input .map(.gz) file. Defaults to '
-                           '{{elf_file}}.map(.gz)?')
+                           '{{elf_file}}.map(.gz)?. If given without '
+                           '--elf-file, no size metadata will be recorded.')
   parser.add_argument('--no-source-paths', action='store_true',
                       help='Do not use .ninja files to map '
                            'object_path -> source_path')
@@ -421,26 +452,43 @@ def Run(args, parser):
   if not args.size_file.endswith('.size'):
     parser.error('size_file must end with .size')
 
+  any_input = args.apk_file or args.elf_file or args.map_file
+  if not any_input:
+    parser.error('Most pass at least one of --apk-file, --elf-file, --map-file')
+  lazy_paths = paths.LazyPaths(args=args, input_file=any_input)
+
+  elf_file = args.elf_file
+  if args.apk_file and not elf_file:
+    with zipfile.ZipFile(args.apk_file) as z:
+      lib_infos = [f for f in z.infolist()
+                   if f.filename.endswith('.so') and f.file_size > 0]
+    assert lib_infos, 'APK has not .so files to measure.'

[estevenson, 2017/04/12 16:54:42]

nit: awkward wording.

[agrieve, 2017/04/12 19:37:46]

Done.

+    # TODO(agrieve): Add support for multiple .so files, and take into account
+    #     secondary architectures.

[estevenson, 2017/04/12 16:54:42]

_SectionSizesFromApk() would fail for 64 bit monochrome right now, correct?
Might be worth it to add this note to the README, though I'd assume it's pretty
uncommon for people to regularly build 64 bit chrome..

I wasn't able to verify this properly, I ran:

tools/binary_size/supersize archive output.size --elf-file
out/Release-64/lib.unstripped/libmonochrome.so --apk-file
out/Release-64/apks/MonochromePublic.apk

and hit the "One or more source file paths could not be found" assert.

Most errors were v8 like this:
Could not find source path for obj/v8/v8_base/assembler-arm64.o

And some of these:
W   4201 no obj basename for obj/third_party/libvpx/libvpx.a(loopfilter_neon.o)
W   4201 Could not find source path for
obj/third_party/libvpx/libvpx.a(loopfilter_neon.o)

[agrieve, 2017/04/12 19:37:45]

Didn't hit any of those other errors, but good catch on not testing with arm64
(nor with crazy.).

I've not tested on both ChromeModernPublic, and with 64-bit monochrome_public.

+    apk_so_path = max(lib_infos, key=lambda x:x.file_size).filename
+    elf_file = os.path.join(lazy_paths.output_directory, 'lib.unstripped',
+                            os.path.basename(apk_so_path))
+    logging.debug('Detected --elf-file=%s', elf_file)
+
   if args.map_file:
     if (not args.map_file.endswith('.map')
         and not args.map_file.endswith('.map.gz')):
       parser.error('Expected --map-file to end with .map or .map.gz')
     map_file_path = args.map_file
   else:
-    map_file_path = args.elf_file + '.map'
+    map_file_path = elf_file + '.map'
     if not os.path.exists(map_file_path):
       map_file_path += '.gz'
     if not os.path.exists(map_file_path):
       parser.error('Could not find .map(.gz)? file. Use --map-file.')
 
-  lazy_paths = paths.LazyPaths(args=args, input_file=args.elf_file)
   metadata = None
-  if args.elf_file:
+  if elf_file:
     logging.debug('Constructing metadata')
-    git_rev = _DetectGitRevision(os.path.dirname(args.elf_file))
-    build_id = BuildIdFromElf(args.elf_file, lazy_paths.tool_prefix)
+    git_rev = _DetectGitRevision(os.path.dirname(elf_file))
+    build_id = BuildIdFromElf(elf_file, lazy_paths.tool_prefix)
     timestamp_obj = datetime.datetime.utcfromtimestamp(os.path.getmtime(
-        args.elf_file))
+        elf_file))
     timestamp = calendar.timegm(timestamp_obj.timetuple())
     gn_args = _ParseGnArgs(os.path.join(lazy_paths.output_directory, 'args.gn'))
 
@@ -450,7 +498,7 @@ def Run(args, parser):
     metadata = {
         models.METADATA_GIT_REVISION: git_rev,
         models.METADATA_MAP_FILENAME: relative_to_out(map_file_path),
-        models.METADATA_ELF_FILENAME: relative_to_out(args.elf_file),
+        models.METADATA_ELF_FILENAME: relative_to_out(elf_file),
         models.METADATA_ELF_MTIME: timestamp,
         models.METADATA_ELF_BUILD_ID: build_id,
         models.METADATA_GN_ARGS: gn_args,
@@ -463,12 +511,21 @@ def Run(args, parser):
   if metadata:
     size_info.metadata = metadata
     logging.debug('Validating section sizes')
-    elf_section_sizes = _SectionSizesFromElf(args.elf_file,
-                                             lazy_paths.tool_prefix)
+    elf_section_sizes = _SectionSizesFromElf(elf_file, lazy_paths.tool_prefix)
     for k, v in elf_section_sizes.iteritems():
       assert v == size_info.section_sizes.get(k), (
           'ELF file and .map file do not match.')
 
+    if args.apk_file:
+      logging.debug('Extracting section sizes from .so within .apk')
+      unpacked_rel_dyn = size_info.section_sizes.get('.rel.dyn', 0)
+      if not unpacked_rel_dyn:
+        logging.warning('Section .rel.dyn did not exist')
+      # TODO(agrieve): Extracting the .so is slow. Do it on a background thread.
+      size_info.section_sizes = _SectionSizesFromApk(
+          args.apk_file, elf_file, build_id, lazy_paths)
+      size_info.section_sizes['.rel.dyn (unpacked)'] = unpacked_rel_dyn
+
   logging.info('Recording metadata: \n  %s',
                '\n  '.join(describe.DescribeMetadata(size_info.metadata)))
   logging.info('Saving result to %s', args.size_file)