Report more apk size metrics in resource_sizes.py

build/android/resource_sizes.py

 #!/usr/bin/python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Prints the size of each given file and optionally computes the size of
    libchrome.so without the dependencies added for building with android NDK.
    Also breaks down the contents of the APK to determine the installed size
    and assign size contributions to different classes of file.
 """
@@ skipping 150 matching lines @@
   """Prints the sizes of each given file.
 
      Args:
        files: List of files to print sizes for.
   """
   for f in files:
     ReportPerfResult(chartjson, 'ResourceSizes', os.path.basename(f) + ' size',
                      os.path.getsize(f), 'bytes')
 
 
+class _FileGroup(object):
+  """Represents a category that apk files can fall into."""
+
+  def __init__(self, name):
+    self.name = name
+    self._zip_infos = []
+    self._extracted = []
+
+  def AddZipInfo(self, zip_info, extracted=False):
+    self._zip_infos.append(zip_info)
+    self._extracted.append(extracted)
+
+  def GetNumEntries(self):
+    return len(self._zip_infos)
+
+  def FindByPattern(self, pattern):
+    return next(i for i in self._zip_infos if re.match(pattern, i.filename))
+
+  def FindLargest(self):
+    return max(self._zip_infos, key=lambda i: i.file_size)
+
+  def ComputeZippedSize(self):
+    return sum(i.compress_size for i in self._zip_infos)
+
+  def ComputeUncompressedSize(self):
+    return sum(i.file_size for i in self._zip_infos)
+
+  def ComputeExtractedSize(self):
+    ret = 0
+    for zi, extracted in zip(self._zip_infos, self._extracted):
+      if extracted:
+        ret += zi.file_size
+    return ret
+
+  def ComputeInstallSize(self):
+    return self.ComputeExtractedSize() + self.ComputeZippedSize()
+
+
 def PrintApkAnalysis(apk_filename, chartjson=None):
   """Analyse APK to determine size contributions of different file classes."""
-  # Define a named tuple type for file grouping.
-  # name: Human readable name for this file group
-  # regex: Regular expression to match filename
-  # extracted: Function that takes a file name and returns whether the file is
-  #            extracted from the apk at install/runtime.
-  FileGroup = collections.namedtuple('FileGroup',
-                                     ['name', 'regex', 'extracted'])
+  file_groups = []
 
-  # File groups are checked in sequence, so more specific regexes should be
-  # earlier in the list.
-  YES = lambda _: True
-  NO = lambda _: False
-  FILE_GROUPS = (
-      FileGroup('Native code', r'\.so$', lambda f: 'crazy' not in f),
-      FileGroup('Java code', r'\.dex$', YES),
-      FileGroup('Native resources (no l10n)',
-                r'^assets/.*(resources|percent)\.pak$', NO),
-      # For locale paks, assume only english paks are extracted.
-      # Handles locale paks as bother resources or assets (.lpak or .pak).
-      FileGroup('Native resources (l10n)',
-                r'\.lpak$|^assets/.*(?!resources|percent)\.pak$',
-                lambda f: 'en_' in f or 'en-' in f),
-      FileGroup('ICU (i18n library) data', r'^assets/icudtl\.dat$', NO),
-      FileGroup('V8 Snapshots', r'^assets/.*\.bin$', NO),
-      FileGroup('PNG drawables', r'\.png$', NO),
-      FileGroup('Non-compiled Android resources', r'^res/', NO),
-      FileGroup('Compiled Android resources', r'\.arsc$', NO),
-      FileGroup('Package metadata', r'^(META-INF/|AndroidManifest\.xml$)', NO),
-      FileGroup('Unknown files', r'.', NO),
-      )
+  def make_group(name):
+    group = _FileGroup(name)
+    file_groups.append(group)
+    return group
+
+  native_code = make_group('Native code')
+  java_code = make_group('Java code')
+  native_resources_no_translations = make_group('Native resources (no l10n)')
+  translations = make_group('Native resources (l10n)')
+  icu_data = make_group('ICU (i18n library) data')
+  v8_snapshots = make_group('V8 Snapshots')
+  png_drawables = make_group('PNG drawables')
+  res_directory = make_group('Non-compiled Android resources')
+  arsc = make_group('Compiled Android resources')
+  metadata = make_group('Package metadata')
+  unknown = make_group('Unknown files')
 
   apk = zipfile.ZipFile(apk_filename, 'r')
   try:
     apk_contents = apk.infolist()
   finally:
     apk.close()
 
   total_apk_size = os.path.getsize(apk_filename)
   apk_basename = os.path.basename(apk_filename)
 
-  found_files = {}
-  for group in FILE_GROUPS:
-    found_files[group] = []
+  for member in apk_contents:
+    filename = member.filename
+    if filename.endswith('/'):
+      continue
 
-  for member in apk_contents:
-    for group in FILE_GROUPS:
-      if re.search(group.regex, member.filename):
-        found_files[group].append(member)
-        break
+    if filename.endswith('.so'):
+      native_code.AddZipInfo(member, 'crazy' not in filename)
+    elif filename.endswith('.dex'):
+      java_code.AddZipInfo(member, True)
+    elif re.search(r'^assets/.*(resources|percent)\.pak$', filename):
+      native_resources_no_translations.AddZipInfo(member)
+    elif re.search(r'\.lpak$|^assets/.*(?!resources|percent)\.pak$', filename):
+      translations.AddZipInfo(member, 'en_' in filename or 'en-' in filename)
+    elif filename == 'assets/icudtl.dat':
+      icu_data.AddZipInfo(member)
+    elif filename.endswith('.bin'):
+      v8_snapshots.AddZipInfo(member)
+    elif filename.endswith('.png') or filename.endswith('.webp'):
+      png_drawables.AddZipInfo(member)
+    elif filename.startswith('res/'):
+      res_directory.AddZipInfo(member)
+    elif filename.endswith('.arsc'):
+      arsc.AddZipInfo(member)
+    elif filename.startswith('META-INF') or filename == 'AndroidManifest.xml':
+      metadata.AddZipInfo(member)
     else:
-      raise KeyError('No group found for file "%s"' % member.filename)
+      unknown.AddZipInfo(member)
 
   total_install_size = total_apk_size
 
-  for group in FILE_GROUPS:
-    uncompressed_size = 0
-    packed_size = 0
-    extracted_size = 0
-    for member in found_files[group]:
-      uncompressed_size += member.file_size
-      packed_size += member.compress_size
-      # Assume that if a file is not compressed, then it is not extracted.
-      is_compressed = member.compress_type != zipfile.ZIP_STORED
-      if is_compressed and group.extracted(member.filename):
-        extracted_size += member.file_size
-    install_size = packed_size + extracted_size
-    total_install_size += extracted_size
+  for group in file_groups:
+    install_size = group.ComputeInstallSize()
+    total_install_size += group.ComputeExtractedSize()
 
     ReportPerfResult(chartjson, apk_basename + '_Breakdown',
-                     group.name + ' size', packed_size, 'bytes')
+                     group.name + ' size', group.ComputeZippedSize(), 'bytes')
     ReportPerfResult(chartjson, apk_basename + '_InstallBreakdown',
                      group.name + ' size', install_size, 'bytes')
     ReportPerfResult(chartjson, apk_basename + '_Uncompressed',
-                     group.name + ' size', uncompressed_size, 'bytes')
+                     group.name + ' size', group.ComputeUncompressedSize(),
+                     'bytes')
 
-  transfer_size = _CalculateCompressedSize(apk_filename)
+  ReportPerfResult(chartjson, apk_basename + '_InstallSize', 'APK size',
+                   total_apk_size, 'bytes')
   ReportPerfResult(chartjson, apk_basename + '_InstallSize',
                    'Estimated installed size', total_install_size, 'bytes')
-  ReportPerfResult(chartjson, apk_basename + '_InstallSize', 'APK size',
-                   total_apk_size, 'bytes')
+  transfer_size = _CalculateCompressedSize(apk_filename)
   ReportPerfResult(chartjson, apk_basename + '_TransferSize',
                    'Transfer size (deflate)', transfer_size, 'bytes')
 
+  # Size of main dex vs remaining.
+  main_dex_info = java_code.FindByPattern('classes.dex')
+  if main_dex_info:
+    main_dex_size = main_dex_info.file_size
+    ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                     'main dex size', main_dex_size, 'bytes')
+    secondary_size = java_code.ComputeUncompressedSize() - main_dex_size
+    ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                     'secondary dex size', secondary_size, 'bytes')
+
+  # Size of main .so vs remaining.
+  main_lib_info = native_code.FindLargest()
+  if main_lib_info:
+    main_lib_size = main_lib_info.file_size
+    ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                     'main lib size', main_lib_size, 'bytes')
+    secondary_size = native_code.ComputeUncompressedSize() - main_lib_size
+    ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                     'other lib size', secondary_size, 'bytes')
+
+  # Main metric that we want to monitor for jumps.
+  normalized_apk_size = total_apk_size
+  # Always look at uncompressed .dex & .so.
+  normalized_apk_size -= java_code.ComputeZippedSize()
+  normalized_apk_size += java_code.ComputeUncompressedSize()
+  normalized_apk_size -= native_code.ComputeZippedSize()
+  normalized_apk_size += native_code.ComputeUncompressedSize()
+  # Avoid noise caused when strings change and translations haven't yet been
+  # updated.
+  english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak')
+  if english_pak:
+    normalized_apk_size -= translations.ComputeZippedSize()
+    # 1.17 found by looking at Chrome.apk and seeing how much smaller en-US.pak
+    # is relative to the average locale .pak.
+    normalized_apk_size += int(
+        english_pak.compress_size * translations.GetNumEntries() * 1.17)
+
+  ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                   'normalized apk size', normalized_apk_size, 'bytes')
+
+  ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                   'file count', len(apk_contents), 'zip entries')
+
 
 def IsPakFileName(file_name):
   """Returns whether the given file name ends with .pak or .lpak."""
   return file_name.endswith('.pak') or file_name.endswith('.lpak')
 
 
 def PrintPakAnalysis(apk_filename, min_pak_resource_size):
   """Print sizes of all resources in all pak files in |apk_filename|."""
   print
   print 'Analyzing pak files in %s...' % apk_filename
@@ skipping 193 matching lines @@
   option_parser.add_option('--so-with-symbols-path',
                            help='Mostly obsolete. Use .so within .apk instead.')
   option_parser.add_option('--min-pak-resource-size', type='int',
                            default=20*1024,
                            help='Minimum byte size of displayed pak resources.')
   option_parser.add_option('--build_type', dest='build_type', default='Debug',
                            help='Obsoleted by --chromium-output-directory.')
   option_parser.add_option('--chromium-output-directory',
                            help='Location of the build artifacts. '
                                 'Takes precidence over --build_type.')
-  option_parser.add_option('--chartjson', action="store_true",
+  option_parser.add_option('--chartjson', action='store_true',
                            help='Sets output mode to chartjson.')
   option_parser.add_option('--output-dir', default='.',
                            help='Directory to save chartjson to.')
+  option_parser.add_option('--no-output-dir', action='store_true',
+                           help='Skip all measurements that rely on having '
+                                'output-dir')
   option_parser.add_option('-d', '--device',
                            help='Dummy option for perf runner.')
   options, args = option_parser.parse_args(argv)
   files = args[1:]
   chartjson = _BASE_CHART.copy() if options.chartjson else None
 
   constants.SetBuildType(options.build_type)
   if options.chromium_output_directory:
     constants.SetOutputDirectory(options.chromium_output_directory)
-  constants.CheckOutputDirectory()
+  if not options.no_output_dir:
+    constants.CheckOutputDirectory()
+    devil_chromium.Initialize()
 
   # For backward compatibilty with buildbot scripts, treat --so-path as just
   # another file to print the size of. We don't need it for anything special any
   # more.
   if options.so_path:
     files.append(options.so_path)
 
   if not files:
     option_parser.error('Must specify a file')
 
-  devil_chromium.Initialize()
-
   if options.so_with_symbols_path:
     si_count = _PrintStaticInitializersCount(options.so_with_symbols_path)
     ReportPerfResult(chartjson, 'StaticInitializersCount', 'count', si_count,
                      'count')
 
   PrintResourceSizes(files, chartjson=chartjson)
 
   for f in files:
     if f.endswith('.apk'):
       PrintApkAnalysis(f, chartjson=chartjson)
-      PrintPakAnalysis(f, options.min_pak_resource_size)
       _PrintDexAnalysis(f, chartjson=chartjson)
-      if not options.so_with_symbols_path:
-        _PrintStaticInitializersCountFromApk(f, chartjson=chartjson)
+      if not options.no_output_dir:
+        PrintPakAnalysis(f, options.min_pak_resource_size)
+        if not options.so_with_symbols_path:
+          _PrintStaticInitializersCountFromApk(f, chartjson=chartjson)
 
   if chartjson:
     results_path = os.path.join(options.output_dir, 'results-chart.json')
     logging.critical('Dumping json to %s', results_path)
     with open(results_path, 'w') as json_file:
       json.dump(chartjson, json_file)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))