resource_sizes.py: Better pakfile and dex normalization.

build/android/resource_sizes.py

 #!/usr/bin/python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Prints the size of each given file and optionally computes the size of
    libchrome.so without the dependencies added for building with android NDK.
    Also breaks down the contents of the APK to determine the installed size
    and assign size contributions to different classes of file.
 """
@@ skipping 93 matching lines @@
     'benchmark_name': 'resource_sizes',
     'benchmark_description': 'APK resource size information.',
     'trace_rerun_options': [],
     'charts': {}
 }
 _DUMP_STATIC_INITIALIZERS_PATH = os.path.join(
     host_paths.DIR_SOURCE_ROOT, 'tools', 'linux', 'dump-static-initializers.py')
 # Pragma exists when enable_resource_whitelist_generation=true.
 _RC_HEADER_RE = re.compile(
     r'^#define (?P<name>\w+) (?:_Pragma\(.*?\) )?(?P<id>\d+)$')
+_RE_NON_LANGUAGE_PAK = re.compile(r'^assets/.*(resources|percent)\.pak$')
+_RE_LANGUAGE_PAK = re.compile(

[agrieve, 2017/06/14 14:11:12]

nit: RE_LANGUAGE_PAK -> RE_COMPRESSED_LANGUAGE_PAK

[estevenson, 2017/06/15 02:56:44]

Done.

+    r'\.lpak$|^assets/(?!stored-locales/).*(?!resources|percent)\.pak$')
+_RE_STORED_LANGUAGE_PAK = re.compile(
+    r'\.lpak$|^assets/stored-locales/.*(?!resources|percent)\.pak$')

[agrieve, 2017/06/14 14:11:13]

nit: can remove .lpak from this list. .lpak is historical, but was only ever
used to allow compression.

[estevenson, 2017/06/15 02:56:43]

Done.

 _READELF_SIZES_METRICS = {
   'text': ['.text'],
   'data': ['.data', '.rodata', '.data.rel.ro', '.data.rel.ro.local'],
   'relocations': ['.rel.dyn', '.rel.plt', '.rela.dyn', '.rela.plt'],
   'unwind': ['.ARM.extab', '.ARM.exidx', '.eh_frame', '.eh_frame_hdr',],
   'symbols': ['.dynsym', '.dynstr', '.dynamic', '.shstrtab', '.got', '.plt',
               '.got.plt', '.hash'],
   'bss': ['.bss'],
   'other': ['.init_array', '.fini_array', '.comment', '.note.gnu.gold-version',
             '.ARM.attributes', '.note.gnu.build-id', '.gnu.version',
@@ skipping 34 matching lines @@
   return section_sizes
 
 
 def _ParseLibBuildId(so_path, tools_prefix):
   """Returns the Build ID of the given native library."""
   stdout = _RunReadelf(so_path, ['-n'], tools_prefix)
   match = re.search(r'Build ID: (\w+)', stdout)
   return match.group(1) if match else None
 
 
+def _ParseManifestAttributes(apk_path):
+  # Check if the manifest specifies whether or not to extract native libs.
+  skip_extract_lib = False
+  output = cmd_helper.GetCmdOutput([
+      'aapt', 'd', 'xmltree', apk_path, 'AndroidManifest.xml'])

[agrieve, 2017/06/14 14:11:12]

I don't think we can assume aapt will be in the PATH.  I think to get this, the
best way is to read build_vars.txt.

[estevenson, 2017/06/15 02:56:43]

Forgot that we already have AAPT_PATH via devil.android.sdk.build_tools. Done.

+  m = re.search(r'extractNativeLibs\(.*\)=\(.*\)(\w)', output)
+  if m:
+    skip_extract_lib = not bool(int(m.group(1)))
+
+  # Dex decompression overhead varies by Android version.
+  output = cmd_helper.GetCmdOutput(['aapt', 'd', 'badging', apk_path])

[agrieve, 2017/06/14 14:11:12]

nit: no need to run both badging and xmltree. The minSdkVersion is in the output
of xmltree already:

    E: uses-sdk (line=19)
      A: android:minSdkVersion(0x0101020c)=(type 0x10)0x18
      A: android:targetSdkVersion(0x01010270)=(type 0x10)0x1a

[estevenson, 2017/06/15 02:56:43]

Ahh missed that. Done!

+  sdk_version = int(re.search(r'sdkVersion:\'(\d+)\'', output).group(1))
+  if sdk_version < 21:
+    dex_multiplier = 1
+  elif sdk_version < 24:
+    dex_multiplier = 3
+  elif 'monochrome' in apk_path.lower():  # Extracted for WebView and Chrome.

[agrieve, 2017/06/14 14:11:13]

Worth a comment saying why this is the case.

[estevenson, 2017/06/15 02:56:44]

Done. Can you confirm that it should be a multiplier of 2 here not 3?

+    dex_multiplier = 3

[estevenson, 2017/06/12 16:56:28]

Should this be 2?

+  else:
+    dex_multiplier = 1
+
+  return dex_multiplier, skip_extract_lib
+
+
 def CountStaticInitializers(so_path, tools_prefix):
   # Static initializers expected in official builds. Note that this list is
   # built using 'nm' on libchrome.so which results from a GCC official build
   # (i.e. Clang is not supported currently).
   def get_elf_section_size(readelf_stdout, section_name):
     # Matches: .ctors PROGBITS 000000000516add0 5169dd0 000010 00 WA 0 0 8
     match = re.search(r'\.%s.*$' % re.escape(section_name),
                       readelf_stdout, re.MULTILINE)
     if not match:
       return (False, -1)
@@ skipping 22 matching lines @@
   return si_count
 
 
 def GetStaticInitializers(so_path, tools_prefix):
   output = cmd_helper.GetCmdOutput([_DUMP_STATIC_INITIALIZERS_PATH, '-d',
                                     so_path, '-t', tools_prefix])
   summary = re.search(r'Found \d+ static initializers in (\d+) files.', output)
   return output.splitlines()[:-1], int(summary.group(1))
 
 
+def _NormalizeLanguagePaks(translations, normalized_apk_size, factor):
+  english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak')
+  num_translations = translations.GetNumEntries()
+  if english_pak:
+    normalized_apk_size -= translations.ComputeZippedSize()
+    normalized_apk_size += int(
+        english_pak.compress_size * num_translations * factor)
+  return normalized_apk_size
+
+
 def _NormalizeResourcesArsc(apk_path):
   """Estimates the expected overhead of untranslated strings in resources.arsc.
 
   See http://crbug.com/677966 for why this is necessary.
   """
   aapt_output = _RunAaptDumpResources(apk_path)
 
   # en-rUS is in the default config and may be cluttered with non-translatable
   # strings, so en-rGB is a better baseline for finding missing translations.
   en_strings = _CreateResourceIdValueMap(aapt_output, 'en-rGB')
@@ skipping 55 matching lines @@
     perf_tests_results_helper.PrintPerfResult(
         graph_title, trace_title, [value], units)
 
 
 class _FileGroup(object):
   """Represents a category that apk files can fall into."""
 
   def __init__(self, name):
     self.name = name
     self._zip_infos = []
-    self._extracted = []
+    self._extracted_multipliers = []
 
-  def AddZipInfo(self, zip_info, extracted=False):
+  def AddZipInfo(self, zip_info, extracted_multiplier=None):

[agrieve, 2017/06/14 14:11:13]

nit: maybe make the default 0, so that you don't have to add the "if multiplier"
below?

[estevenson, 2017/06/15 02:56:44]

Done.

     self._zip_infos.append(zip_info)
-    self._extracted.append(extracted)
+    self._extracted_multipliers.append(extracted_multiplier)
 
   def AllEntries(self):
     return iter(self._zip_infos)
 
   def GetNumEntries(self):
     return len(self._zip_infos)
 
   def FindByPattern(self, pattern):
     return next((i for i in self._zip_infos if re.match(pattern, i.filename)),
                 None)
 
   def FindLargest(self):
     if not self._zip_infos:
       return None
     return max(self._zip_infos, key=lambda i: i.file_size)
 
   def ComputeZippedSize(self):
     return sum(i.compress_size for i in self._zip_infos)
 
   def ComputeUncompressedSize(self):
     return sum(i.file_size for i in self._zip_infos)
 
   def ComputeExtractedSize(self):
     ret = 0
-    for zi, extracted in zip(self._zip_infos, self._extracted):
-      if extracted:
-        ret += zi.file_size
+    for zi, multiplier in zip(self._zip_infos, self._extracted_multipliers):
+      if multiplier:
+        ret += zi.file_size * int(multiplier)
     return ret
 
   def ComputeInstallSize(self):
     return self.ComputeExtractedSize() + self.ComputeZippedSize()
 
 
 def PrintApkAnalysis(apk_filename, tools_prefix, chartjson=None):
   """Analyse APK to determine size contributions of different file classes."""
   file_groups = []
 
   def make_group(name):
     group = _FileGroup(name)
     file_groups.append(group)
     return group
 
   native_code = make_group('Native code')
   java_code = make_group('Java code')
   native_resources_no_translations = make_group('Native resources (no l10n)')
   translations = make_group('Native resources (l10n)')
+  stored_translations = make_group('Native resources stored (l10n)')
   icu_data = make_group('ICU (i18n library) data')
   v8_snapshots = make_group('V8 Snapshots')
   png_drawables = make_group('PNG drawables')
   res_directory = make_group('Non-compiled Android resources')
   arsc = make_group('Compiled Android resources')
   metadata = make_group('Package metadata')
   unknown = make_group('Unknown files')
   notices = make_group('licenses.notice file')
 
   apk = zipfile.ZipFile(apk_filename, 'r')
   try:
     apk_contents = apk.infolist()
   finally:
     apk.close()
 
+  dex_multiplier, skip_extract_lib = _ParseManifestAttributes(apk_filename)
   total_apk_size = os.path.getsize(apk_filename)
   apk_basename = os.path.basename(apk_filename)
-
   for member in apk_contents:
     filename = member.filename
     if filename.endswith('/'):
       continue
-
     if filename.endswith('.so'):
-      native_code.AddZipInfo(member, 'crazy' not in filename)
+      should_extract_lib = not (skip_extract_lib or 'crazy' in filename)
+      native_code.AddZipInfo(member, should_extract_lib)

[agrieve, 2017/06/14 14:11:12]

nit: change to int(should_extract_lib)

[estevenson, 2017/06/15 02:56:43]

Done.

     elif filename.endswith('.dex'):
-      java_code.AddZipInfo(member, True)
-    elif re.search(r'^assets/.*(resources|percent)\.pak$', filename):
+      java_code.AddZipInfo(member, dex_multiplier)
+    elif re.search(_RE_NON_LANGUAGE_PAK, filename):
       native_resources_no_translations.AddZipInfo(member)
-    elif re.search(r'\.lpak$|^assets/.*(?!resources|percent)\.pak$', filename):
+    elif re.search(_RE_LANGUAGE_PAK, filename):
       translations.AddZipInfo(member, 'en_' in filename or 'en-' in filename)

[agrieve, 2017/06/14 14:11:13]

nit: add int() to 2nd param

[estevenson, 2017/06/15 02:56:44]

Done.

+    elif re.search(_RE_STORED_LANGUAGE_PAK, filename):
+      stored_translations.AddZipInfo(
+          member, 'en_' in filename or 'en-' in filename)

[agrieve, 2017/06/14 14:11:13]

I think 2nd param here should always be 0 (never extracted)

[estevenson, 2017/06/15 02:56:43]

Definitely. Done.

     elif filename == 'assets/icudtl.dat':
       icu_data.AddZipInfo(member)
     elif filename.endswith('.bin'):
       v8_snapshots.AddZipInfo(member)
     elif filename.endswith('.png') or filename.endswith('.webp'):
       png_drawables.AddZipInfo(member)
     elif filename.startswith('res/'):
       res_directory.AddZipInfo(member)
     elif filename.endswith('.arsc'):
       arsc.AddZipInfo(member)
@@ skipping 54 matching lines @@
 
   # Main metric that we want to monitor for jumps.
   normalized_apk_size = total_apk_size
   # Always look at uncompressed .dex & .so.
   normalized_apk_size -= java_code.ComputeZippedSize()
   normalized_apk_size += java_code.ComputeUncompressedSize()
   normalized_apk_size -= native_code.ComputeZippedSize()
   normalized_apk_size += native_code.ComputeUncompressedSize()
   # Avoid noise caused when strings change and translations haven't yet been
   # updated.
-  english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak')
   num_translations = translations.GetNumEntries()
-  if english_pak and num_translations > 1:
-    normalized_apk_size -= translations.ComputeZippedSize()
-    # 1.17 found by looking at Chrome.apk and seeing how much smaller en-US.pak

[agrieve, 2017/06/14 14:11:13]

can we keep this comment?

[estevenson, 2017/06/15 02:56:44]

Done.

-    # is relative to the average locale .pak.
-    normalized_apk_size += int(
-        english_pak.compress_size * num_translations * 1.17)
+  if num_translations > 1:
+    normalized_apk_size = _NormalizeLanguagePaks(
+        translations, normalized_apk_size, 1.17)
+    normalized_apk_size = _NormalizeLanguagePaks(
+        stored_translations, normalized_apk_size, 1.43)
     normalized_apk_size += int(_NormalizeResourcesArsc(apk_filename))
 
   ReportPerfResult(chartjson, apk_basename + '_Specifics',
                    'normalized apk size', normalized_apk_size, 'bytes')
 
   ReportPerfResult(chartjson, apk_basename + '_Specifics',
                    'file count', len(apk_contents), 'zip entries')
 
   for info in unknown.AllEntries():
     print 'Unknown entry:', info.filename, info.compress_size
@@ skipping 327 matching lines @@
         chartjson=chartjson)
   if chartjson:
     results_path = os.path.join(args.output_dir, 'results-chart.json')
     logging.critical('Dumping json to %s', results_path)
     with open(results_path, 'w') as json_file:
       json.dump(chartjson, json_file)
 
 
 if __name__ == '__main__':
   sys.exit(main())