Report more apk size metrics in resource_sizes.py

build/android/resource_sizes.py

Index: build/android/resource_sizes.py
diff --git a/build/android/resource_sizes.py b/build/android/resource_sizes.py
index 6c77ec3e61dfafeeb3506e4050015b021f7d60f7..56e5a4969f6043df9f2f1f5b05e21c91dbc6471b 100755
--- a/build/android/resource_sizes.py
+++ b/build/android/resource_sizes.py
@@ -168,38 +168,64 @@ def PrintResourceSizes(files, chartjson=None):
                      os.path.getsize(f), 'bytes')
 
 
+class _FileGroup(object):
+  """Represents a category that apk files can fall into."""
+
+  def __init__(self, name):
+    self.name = name
+    self._zip_infos = []
+    self._extracted = []
+
+  def AddZipInfo(self, zip_info, extracted=False):
+    self._zip_infos.append(zip_info)
+    self._extracted.append(extracted)
+
+  def GetNumEntries(self):
+    return len(self._zip_infos)
+
+  def FindByPattern(self, pattern):
+    return next(i for i in self._zip_infos if re.match(pattern, i.filename))
+
+  def FindLargest(self):
+    return max(self._zip_infos, key=lambda i: i.file_size)
+
+  def ComputeZippedSize(self):
+    return sum(i.compress_size for i in self._zip_infos)
+
+  def ComputeUncompressedSize(self):
+    return sum(i.file_size for i in self._zip_infos)
+
+  def ComputeExtractedSize(self):
+    ret = 0
+    for zi, extracted in zip(self._zip_infos, self._extracted):
+      if extracted:
+        ret += zi.file_size
+    return ret
+
+  def ComputeInstallSize(self):
+    return self.ComputeExtractedSize() + self.ComputeZippedSize()
+
+
 def PrintApkAnalysis(apk_filename, chartjson=None):
   """Analyse APK to determine size contributions of different file classes."""
-  # Define a named tuple type for file grouping.
-  # name: Human readable name for this file group
-  # regex: Regular expression to match filename
-  # extracted: Function that takes a file name and returns whether the file is
-  #            extracted from the apk at install/runtime.
-  FileGroup = collections.namedtuple('FileGroup',
-                                     ['name', 'regex', 'extracted'])
-
-  # File groups are checked in sequence, so more specific regexes should be
-  # earlier in the list.
-  YES = lambda _: True
-  NO = lambda _: False
-  FILE_GROUPS = (
-      FileGroup('Native code', r'\.so$', lambda f: 'crazy' not in f),
-      FileGroup('Java code', r'\.dex$', YES),
-      FileGroup('Native resources (no l10n)',
-                r'^assets/.*(resources|percent)\.pak$', NO),
-      # For locale paks, assume only english paks are extracted.
-      # Handles locale paks as bother resources or assets (.lpak or .pak).
-      FileGroup('Native resources (l10n)',
-                r'\.lpak$|^assets/.*(?!resources|percent)\.pak$',
-                lambda f: 'en_' in f or 'en-' in f),
-      FileGroup('ICU (i18n library) data', r'^assets/icudtl\.dat$', NO),
-      FileGroup('V8 Snapshots', r'^assets/.*\.bin$', NO),
-      FileGroup('PNG drawables', r'\.png$', NO),
-      FileGroup('Non-compiled Android resources', r'^res/', NO),
-      FileGroup('Compiled Android resources', r'\.arsc$', NO),
-      FileGroup('Package metadata', r'^(META-INF/|AndroidManifest\.xml$)', NO),
-      FileGroup('Unknown files', r'.', NO),
-      )
+  file_groups = []
+
+  def make_group(name):
+    group = _FileGroup(name)
+    file_groups.append(group)
+    return group
+
+  native_code = make_group('Native code')
+  java_code = make_group('Java code')
+  native_resources_no_translations = make_group('Native resources (no l10n)')
+  translations = make_group('Native resources (l10n)')
+  icu_data = make_group('ICU (i18n library) data')
+  v8_snapshots = make_group('V8 Snapshots')
+  png_drawables = make_group('PNG drawables')
+  res_directory = make_group('Non-compiled Android resources')
+  arsc = make_group('Compiled Android resources')
+  metadata = make_group('Package metadata')
+  unknown = make_group('Unknown files')
 
   apk = zipfile.ZipFile(apk_filename, 'r')
   try:
@@ -210,49 +236,99 @@ def PrintApkAnalysis(apk_filename, chartjson=None):
   total_apk_size = os.path.getsize(apk_filename)
   apk_basename = os.path.basename(apk_filename)
 
-  found_files = {}
-  for group in FILE_GROUPS:
-    found_files[group] = []
-
   for member in apk_contents:
-    for group in FILE_GROUPS:
-      if re.search(group.regex, member.filename):
-        found_files[group].append(member)
-        break
+    filename = member.filename
+    if filename.endswith('/'):
+      continue
+
+    if filename.endswith('.so'):
+      native_code.AddZipInfo(member, 'crazy' not in filename)
+    elif filename.endswith('.dex'):
+      java_code.AddZipInfo(member, True)
+    elif re.search(r'^assets/.*(resources|percent)\.pak$', filename):
+      native_resources_no_translations.AddZipInfo(member)
+    elif re.search(r'\.lpak$|^assets/.*(?!resources|percent)\.pak$', filename):
+      translations.AddZipInfo(member, 'en_' in filename or 'en-' in filename)
+    elif filename == 'assets/icudtl.dat':
+      icu_data.AddZipInfo(member)
+    elif filename.endswith('.bin'):
+      v8_snapshots.AddZipInfo(member)
+    elif filename.endswith('.png') or filename.endswith('.webp'):
+      png_drawables.AddZipInfo(member)
+    elif filename.startswith('res/'):
+      res_directory.AddZipInfo(member)
+    elif filename.endswith('.arsc'):
+      arsc.AddZipInfo(member)
+    elif filename.startswith('META-INF') or filename == 'AndroidManifest.xml':
+      metadata.AddZipInfo(member)
     else:
-      raise KeyError('No group found for file "%s"' % member.filename)
+      unknown.AddZipInfo(member)
 
   total_install_size = total_apk_size
 
-  for group in FILE_GROUPS:
-    uncompressed_size = 0
-    packed_size = 0
-    extracted_size = 0
-    for member in found_files[group]:
-      uncompressed_size += member.file_size
-      packed_size += member.compress_size
-      # Assume that if a file is not compressed, then it is not extracted.
-      is_compressed = member.compress_type != zipfile.ZIP_STORED
-      if is_compressed and group.extracted(member.filename):
-        extracted_size += member.file_size
-    install_size = packed_size + extracted_size
-    total_install_size += extracted_size
+  for group in file_groups:
+    install_size = group.ComputeInstallSize()
+    total_install_size += group.ComputeExtractedSize()
 
     ReportPerfResult(chartjson, apk_basename + '_Breakdown',
-                     group.name + ' size', packed_size, 'bytes')
+                     group.name + ' size', group.ComputeZippedSize(), 'bytes')
     ReportPerfResult(chartjson, apk_basename + '_InstallBreakdown',
                      group.name + ' size', install_size, 'bytes')
     ReportPerfResult(chartjson, apk_basename + '_Uncompressed',
-                     group.name + ' size', uncompressed_size, 'bytes')
+                     group.name + ' size', group.ComputeUncompressedSize(),
+                     'bytes')
 
-  transfer_size = _CalculateCompressedSize(apk_filename)
-  ReportPerfResult(chartjson, apk_basename + '_InstallSize',
-                   'Estimated installed size', total_install_size, 'bytes')
   ReportPerfResult(chartjson, apk_basename + '_InstallSize', 'APK size',
                    total_apk_size, 'bytes')
+  ReportPerfResult(chartjson, apk_basename + '_InstallSize',
+                   'Estimated installed size', total_install_size, 'bytes')
+  transfer_size = _CalculateCompressedSize(apk_filename)
   ReportPerfResult(chartjson, apk_basename + '_TransferSize',
                    'Transfer size (deflate)', transfer_size, 'bytes')
 
+  # Size of main dex vs remaining.
+  main_dex_info = java_code.FindByPattern('classes.dex')
+  if main_dex_info:
+    main_dex_size = main_dex_info.file_size
+    ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                     'main dex size', main_dex_size, 'bytes')
+    secondary_size = java_code.ComputeUncompressedSize() - main_dex_size
+    ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                     'secondary dex size', secondary_size, 'bytes')
+
+  # Size of main .so vs remaining.
+  main_lib_info = native_code.FindLargest()
+  if main_lib_info:
+    main_lib_size = main_lib_info.file_size
+    ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                     'main lib size', main_lib_size, 'bytes')
+    secondary_size = native_code.ComputeUncompressedSize() - main_lib_size
+    ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                     'other lib size', secondary_size, 'bytes')
+
+  # Main metric that we want to monitor for jumps.
+  normalized_apk_size = total_apk_size
+  # Always look at uncompressed .dex & .so.
+  normalized_apk_size -= java_code.ComputeZippedSize()
+  normalized_apk_size += java_code.ComputeUncompressedSize()
+  normalized_apk_size -= native_code.ComputeZippedSize()
+  normalized_apk_size += native_code.ComputeUncompressedSize()
+  # Avoid noise caused when strings change and translations haven't yet been
+  # updated.
+  english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak')
+  if english_pak:
+    normalized_apk_size -= translations.ComputeZippedSize()
+    # 1.17 found by looking at Chrome.apk and seeing how much smaller en-US.pak
+    # is relative to the average locale .pak.
+    normalized_apk_size += int(
+        english_pak.compress_size * translations.GetNumEntries() * 1.17)
+
+  ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                   'normalized apk size', normalized_apk_size, 'bytes')
+
+  ReportPerfResult(chartjson, apk_basename + '_Specifics',
+                   'file count', len(apk_contents), 'zip entries')
+
 
 def IsPakFileName(file_name):
   """Returns whether the given file name ends with .pak or .lpak."""
@@ -466,10 +542,13 @@ Pass any number of files to graph their sizes. Any files with the extension
   option_parser.add_option('--chromium-output-directory',
                            help='Location of the build artifacts. '
                                 'Takes precidence over --build_type.')
-  option_parser.add_option('--chartjson', action="store_true",
+  option_parser.add_option('--chartjson', action='store_true',
                            help='Sets output mode to chartjson.')
   option_parser.add_option('--output-dir', default='.',
                            help='Directory to save chartjson to.')
+  option_parser.add_option('--no-output-dir', action='store_true',
+                           help='Skip all measurements that rely on having '
+                                'output-dir')
   option_parser.add_option('-d', '--device',
                            help='Dummy option for perf runner.')
   options, args = option_parser.parse_args(argv)
@@ -479,7 +558,9 @@ Pass any number of files to graph their sizes. Any files with the extension
   constants.SetBuildType(options.build_type)
   if options.chromium_output_directory:
     constants.SetOutputDirectory(options.chromium_output_directory)
-  constants.CheckOutputDirectory()
+  if not options.no_output_dir:
+    constants.CheckOutputDirectory()
+    devil_chromium.Initialize()
 
   # For backward compatibilty with buildbot scripts, treat --so-path as just
   # another file to print the size of. We don't need it for anything special any
@@ -490,8 +571,6 @@ Pass any number of files to graph their sizes. Any files with the extension
   if not files:
     option_parser.error('Must specify a file')
 
-  devil_chromium.Initialize()
-
   if options.so_with_symbols_path:
     si_count = _PrintStaticInitializersCount(options.so_with_symbols_path)
     ReportPerfResult(chartjson, 'StaticInitializersCount', 'count', si_count,
@@ -502,10 +581,11 @@ Pass any number of files to graph their sizes. Any files with the extension
   for f in files:
     if f.endswith('.apk'):
       PrintApkAnalysis(f, chartjson=chartjson)
-      PrintPakAnalysis(f, options.min_pak_resource_size)
       _PrintDexAnalysis(f, chartjson=chartjson)
-      if not options.so_with_symbols_path:
-        _PrintStaticInitializersCountFromApk(f, chartjson=chartjson)
+      if not options.no_output_dir:
+        PrintPakAnalysis(f, options.min_pak_resource_size)
+        if not options.so_with_symbols_path:
+          _PrintStaticInitializersCountFromApk(f, chartjson=chartjson)
 
   if chartjson:
     results_path = os.path.join(options.output_dir, 'results-chart.json')