resource_sizes.py: Better pakfile and dex normalization.

build/android/resource_sizes.py

 #!/usr/bin/python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Prints the size of each given file and optionally computes the size of
    libchrome.so without the dependencies added for building with android NDK.
    Also breaks down the contents of the APK to determine the installed size
    and assign size contributions to different classes of file.
 """
@@ skipping 93 matching lines @@
     'benchmark_name': 'resource_sizes',
     'benchmark_description': 'APK resource size information.',
     'trace_rerun_options': [],
     'charts': {}
 }
 _DUMP_STATIC_INITIALIZERS_PATH = os.path.join(
     host_paths.DIR_SOURCE_ROOT, 'tools', 'linux', 'dump-static-initializers.py')
 # Pragma exists when enable_resource_whitelist_generation=true.
 _RC_HEADER_RE = re.compile(
     r'^#define (?P<name>\w+) (?:_Pragma\(.*?\) )?(?P<id>\d+)$')
+_RE_NON_LANGUAGE_PAK = re.compile(r'^assets/.*(resources|percent)\.pak$')
+_RE_COMPRESSED_LANGUAGE_PAK = re.compile(
+    r'\.lpak$|^assets/(?!stored-locales/).*(?!resources|percent)\.pak$')
+_RE_STORED_LANGUAGE_PAK = re.compile(r'^assets/stored-locales/.*\.pak$')
 _READELF_SIZES_METRICS = {
   'text': ['.text'],
   'data': ['.data', '.rodata', '.data.rel.ro', '.data.rel.ro.local'],
   'relocations': ['.rel.dyn', '.rel.plt', '.rela.dyn', '.rela.plt'],
   'unwind': ['.ARM.extab', '.ARM.exidx', '.eh_frame', '.eh_frame_hdr',],
   'symbols': ['.dynsym', '.dynstr', '.dynamic', '.shstrtab', '.got', '.plt',
               '.got.plt', '.hash'],
   'bss': ['.bss'],
   'other': ['.init_array', '.fini_array', '.comment', '.note.gnu.gold-version',
             '.ARM.attributes', '.note.gnu.build-id', '.gnu.version',
@@ skipping 34 matching lines @@
   return section_sizes
 
 
 def _ParseLibBuildId(so_path, tools_prefix):
   """Returns the Build ID of the given native library."""
   stdout = _RunReadelf(so_path, ['-n'], tools_prefix)
   match = re.search(r'Build ID: (\w+)', stdout)
   return match.group(1) if match else None
 
 
+def _ParseManifestAttributes(apk_path):
+  # Check if the manifest specifies whether or not to extract native libs.
+  skip_extract_lib = False
+  output = cmd_helper.GetCmdOutput([
+      _AAPT_PATH.read(), 'd', 'xmltree', apk_path, 'AndroidManifest.xml'])
+  m = re.search(r'extractNativeLibs\(.*\)=\(.*\)(\w)', output)
+  if m:
+    skip_extract_lib = not bool(int(m.group(1)))
+
+  m = re.search(r'android:minSdkVersion\(\w+\)=\(type \w+\)(\w+)\n', output)
+  sdk_version = int(m.group(1), 16)
+  # Dex decompression overhead varies by Android version.
+  if sdk_version < 21:
+    dex_multiplier = 1
+  elif sdk_version < 24:
+    dex_multiplier = 3
+  elif 'monochrome' in apk_path.lower():
+    # Monochrome consists of WebView and Chrome packaged into a single APK, so

[agrieve, 2017/06/16 01:37:16]

It's not that both apps extract the dex, it's that on Android N+, there is
profile-guided dexopt, which ends up being 1x multiplied, but as another app
loads your dex, it switches to non-profile-guided dexopt (3x multiplier). 

Because webview loads the dex, and webview lives in random other apps,
Monochrome falls into the bucket of having full dexopt, making it the same as
pre-N (so, 3x).

Would be great if you added this explanation as a comment :P (perhaps cleaned up
a bit)

[estevenson, 2017/06/16 14:56:19]

Ahhh yes that's what it is, couldn't remember. Fixed up these comments a bit.

+    # we need to account for both apps extracting the dex file.
+    dex_multiplier = 2
+  else:
+    dex_multiplier = 1
+
+  return dex_multiplier, skip_extract_lib
+
+
 def CountStaticInitializers(so_path, tools_prefix):
   # Static initializers expected in official builds. Note that this list is
   # built using 'nm' on libchrome.so which results from a GCC official build
   # (i.e. Clang is not supported currently).
   def get_elf_section_size(readelf_stdout, section_name):
     # Matches: .ctors PROGBITS 000000000516add0 5169dd0 000010 00 WA 0 0 8
     match = re.search(r'\.%s.*$' % re.escape(section_name),
                       readelf_stdout, re.MULTILINE)
     if not match:
       return (False, -1)
@@ skipping 22 matching lines @@
   return si_count
 
 
 def GetStaticInitializers(so_path, tools_prefix):
   output = cmd_helper.GetCmdOutput([_DUMP_STATIC_INITIALIZERS_PATH, '-d',
                                     so_path, '-t', tools_prefix])
   summary = re.search(r'Found \d+ static initializers in (\d+) files.', output)
   return output.splitlines()[:-1], int(summary.group(1))
 
 
+def _NormalizeLanguagePaks(translations, normalized_apk_size, factor):
+  english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak')
+  num_translations = translations.GetNumEntries()
+  if english_pak:
+    normalized_apk_size -= translations.ComputeZippedSize()
+    normalized_apk_size += int(
+        english_pak.compress_size * num_translations * factor)
+  return normalized_apk_size
+
+
 def _NormalizeResourcesArsc(apk_path):
   """Estimates the expected overhead of untranslated strings in resources.arsc.
 
   See http://crbug.com/677966 for why this is necessary.
   """
   aapt_output = _RunAaptDumpResources(apk_path)
 
   # en-rUS is in the default config and may be cluttered with non-translatable
   # strings, so en-rGB is a better baseline for finding missing translations.
   en_strings = _CreateResourceIdValueMap(aapt_output, 'en-rGB')
@@ skipping 55 matching lines @@
     perf_tests_results_helper.PrintPerfResult(
         graph_title, trace_title, [value], units)
 
 
 class _FileGroup(object):
   """Represents a category that apk files can fall into."""
 
   def __init__(self, name):
     self.name = name
     self._zip_infos = []
-    self._extracted = []
+    self._extracted_multipliers = []
 
-  def AddZipInfo(self, zip_info, extracted=False):
+  def AddZipInfo(self, zip_info, extracted_multiplier=0):
     self._zip_infos.append(zip_info)
-    self._extracted.append(extracted)
+    self._extracted_multipliers.append(extracted_multiplier)
 
   def AllEntries(self):
     return iter(self._zip_infos)
 
   def GetNumEntries(self):
     return len(self._zip_infos)
 
   def FindByPattern(self, pattern):
     return next((i for i in self._zip_infos if re.match(pattern, i.filename)),
                 None)
 
   def FindLargest(self):
     if not self._zip_infos:
       return None
     return max(self._zip_infos, key=lambda i: i.file_size)
 
   def ComputeZippedSize(self):
     return sum(i.compress_size for i in self._zip_infos)
 
   def ComputeUncompressedSize(self):
     return sum(i.file_size for i in self._zip_infos)
 
   def ComputeExtractedSize(self):
     ret = 0
-    for zi, extracted in zip(self._zip_infos, self._extracted):
-      if extracted:
-        ret += zi.file_size
+    for zi, multiplier in zip(self._zip_infos, self._extracted_multipliers):
+      ret += zi.file_size * multiplier
     return ret
 
   def ComputeInstallSize(self):
     return self.ComputeExtractedSize() + self.ComputeZippedSize()
 
 
 def PrintApkAnalysis(apk_filename, tools_prefix, chartjson=None):
   """Analyse APK to determine size contributions of different file classes."""
   file_groups = []
 
   def make_group(name):
     group = _FileGroup(name)
     file_groups.append(group)
     return group
 
   native_code = make_group('Native code')
   java_code = make_group('Java code')
   native_resources_no_translations = make_group('Native resources (no l10n)')
   translations = make_group('Native resources (l10n)')
+  stored_translations = make_group('Native resources stored (l10n)')
   icu_data = make_group('ICU (i18n library) data')
   v8_snapshots = make_group('V8 Snapshots')
   png_drawables = make_group('PNG drawables')
   res_directory = make_group('Non-compiled Android resources')
   arsc = make_group('Compiled Android resources')
   metadata = make_group('Package metadata')
   unknown = make_group('Unknown files')
   notices = make_group('licenses.notice file')
 
   apk = zipfile.ZipFile(apk_filename, 'r')
   try:
     apk_contents = apk.infolist()
   finally:
     apk.close()
 
+  dex_multiplier, skip_extract_lib = _ParseManifestAttributes(apk_filename)
   total_apk_size = os.path.getsize(apk_filename)
   apk_basename = os.path.basename(apk_filename)
-
   for member in apk_contents:
     filename = member.filename
     if filename.endswith('/'):
       continue
-
     if filename.endswith('.so'):
-      native_code.AddZipInfo(member, 'crazy' not in filename)
+      should_extract_lib = not (skip_extract_lib or 'crazy' in filename)
+      native_code.AddZipInfo(member, int(should_extract_lib))
     elif filename.endswith('.dex'):
-      java_code.AddZipInfo(member, True)
-    elif re.search(r'^assets/.*(resources|percent)\.pak$', filename):
+      java_code.AddZipInfo(member, dex_multiplier)
+    elif re.search(_RE_NON_LANGUAGE_PAK, filename):
       native_resources_no_translations.AddZipInfo(member)
-    elif re.search(r'\.lpak$|^assets/.*(?!resources|percent)\.pak$', filename):
-      translations.AddZipInfo(member, 'en_' in filename or 'en-' in filename)
+    elif re.search(_RE_COMPRESSED_LANGUAGE_PAK, filename):
+      translations.AddZipInfo(member)
+    elif re.search(_RE_STORED_LANGUAGE_PAK, filename):
+      stored_translations.AddZipInfo(
+          member, 'en_' in filename or 'en-' in filename)

[agrieve, 2017/06/16 01:37:16]

add int()

[estevenson, 2017/06/16 14:56:19]

Oops, must have mixed these up. Added int() to the compressed pak above, left
out the extracted_multiplier for stored_translations.

Also updated calls to AddZipInfo() to use "extracted_multiplier=".

     elif filename == 'assets/icudtl.dat':
       icu_data.AddZipInfo(member)
     elif filename.endswith('.bin'):
       v8_snapshots.AddZipInfo(member)
     elif filename.endswith('.png') or filename.endswith('.webp'):
       png_drawables.AddZipInfo(member)
     elif filename.startswith('res/'):
       res_directory.AddZipInfo(member)
     elif filename.endswith('.arsc'):
       arsc.AddZipInfo(member)
@@ skipping 54 matching lines @@
 
   # Main metric that we want to monitor for jumps.
   normalized_apk_size = total_apk_size
   # Always look at uncompressed .dex & .so.
   normalized_apk_size -= java_code.ComputeZippedSize()
   normalized_apk_size += java_code.ComputeUncompressedSize()
   normalized_apk_size -= native_code.ComputeZippedSize()
   normalized_apk_size += native_code.ComputeUncompressedSize()
   # Avoid noise caused when strings change and translations haven't yet been
   # updated.
-  english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak')
   num_translations = translations.GetNumEntries()
-  if english_pak and num_translations > 1:
-    normalized_apk_size -= translations.ComputeZippedSize()
-    # 1.17 found by looking at Chrome.apk and seeing how much smaller en-US.pak
-    # is relative to the average locale .pak.
-    normalized_apk_size += int(
-        english_pak.compress_size * num_translations * 1.17)
+  if num_translations > 1:
+    # Multipliers found by looking at MonochromePublic.apk and seeing how much
+    # smaller en-US.pak is relative to the average locale.pak.
+    normalized_apk_size = _NormalizeLanguagePaks(
+        translations, normalized_apk_size, 1.17)
+    normalized_apk_size = _NormalizeLanguagePaks(
+        stored_translations, normalized_apk_size, 1.43)
     normalized_apk_size += int(_NormalizeResourcesArsc(apk_filename))
 
   ReportPerfResult(chartjson, apk_basename + '_Specifics',
                    'normalized apk size', normalized_apk_size, 'bytes')
 
   ReportPerfResult(chartjson, apk_basename + '_Specifics',
                    'file count', len(apk_contents), 'zip entries')
 
   for info in unknown.AllEntries():
     print 'Unknown entry:', info.filename, info.compress_size
@@ skipping 296 matching lines @@
                          default=apk_downloader.DEFAULT_BUILDER,
                          help='Builder name to use for reference APK for patch '
                          'size estimates.')
   argparser.add_argument('--reference-apk-bucket',
                          default=apk_downloader.DEFAULT_BUCKET,
                          help='Storage bucket holding reference APKs.')
   argparser.add_argument('apk', help='APK file path.')
   args = argparser.parse_args()
 
   chartjson = _BASE_CHART.copy() if args.chartjson else None
-
   if args.chromium_output_directory:
     constants.SetOutputDirectory(args.chromium_output_directory)
   if not args.no_output_dir:
     constants.CheckOutputDirectory()
     devil_chromium.Initialize()
     build_vars = _ReadBuildVars(constants.GetOutDirectory())
     tools_prefix = os.path.join(constants.GetOutDirectory(),
                                 build_vars['android_tool_prefix'])
   else:
     tools_prefix = ''
@@ skipping 10 matching lines @@
         chartjson=chartjson)
   if chartjson:
     results_path = os.path.join(args.output_dir, 'results-chart.json')
     logging.critical('Dumping json to %s', results_path)
     with open(results_path, 'w') as json_file:
       json.dump(chartjson, json_file)
 
 
 if __name__ == '__main__':
   sys.exit(main())