Speed up incremental_install by caching device checksums between runs.

build/android/incremental_install/installer.py

 #!/usr/bin/env python
 #
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Install *_incremental.apk targets as well as their dependent files."""
 
 import argparse
 import glob
@@ skipping 16 matching lines @@
 sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'gyp'))
 from util import build_utils
 
 
 def _TransformDexPaths(paths):
   """Given paths like ["/a/b/c", "/a/c/d"], returns ["b.c", "c.d"]."""
   prefix_len = len(os.path.commonprefix(paths))
   return [p[prefix_len:].replace(os.sep, '.') for p in paths]
 
 
+def _Execute(concurrently, funcs):
+  """Calls all functions in |funcs| concurrently or in sequence."""
+  timer = time_profile.TimeProfile()
+  if concurrently:
+    reraiser_thread.RunAsync(funcs)
+  else:
+    for f in funcs:
+      f()
+  timer.Stop(log=False)
+  return timer
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('apk_path',
                       help='The path to the APK to install.')
   parser.add_argument('--split',
                       action='append',
                       dest='splits',
                       help='A glob matching the apk splits. '
                            'Can be specified multiple times.')
   parser.add_argument('--lib-dir',
                       help='Path to native libraries directory.')
   parser.add_argument('--dex-files',
                       help='List of dex files to push.',
                       action='append',
                       default=[])
   parser.add_argument('-d', '--device', dest='device',
                       help='Target device for apk to install on.')
   parser.add_argument('--uninstall',
                       action='store_true',
                       default=False,
                       help='Remove the app and all side-loaded files.')
   parser.add_argument('--output-directory',
                       help='Path to the root build directory.')
   parser.add_argument('--no-threading',
                       action='store_true',

[jbudorick, 2015/10/01 17:15:26]

nit: using dest='threading', action='store_false' here would help clear up the
double-negative below.

[agrieve, 2015/10/01 18:30:16]

Done.

                       default=False,
                       help='Do not install and push concurrently')
   parser.add_argument('-v',
                       '--verbose',
                       dest='verbose_count',
                       default=0,
                       action='count',
                       help='Verbose level (multiple times for more)')
 
   args = parser.parse_args()
 
   run_tests_helper.SetLogLevel(args.verbose_count)
   constants.SetBuildType('Debug')
   if args.output_directory:
     constants.SetOutputDirectory(args.output_directory)
 
   main_timer = time_profile.TimeProfile()
   install_timer = time_profile.TimeProfile()
   push_native_timer = time_profile.TimeProfile()
   push_dex_timer = time_profile.TimeProfile()
 
   if args.device:
     # Retries are annoying when commands fail for legitimate reasons. Might want
     # to enable them if this is ever used on bots though.
-    device = device_utils.DeviceUtils(args.device, default_retries=0)
+    device = device_utils.DeviceUtils(
+        args.device, default_retries=0, enable_device_files_cache=True)
   else:
-    devices = device_utils.DeviceUtils.HealthyDevices(default_retries=0)
+    devices = device_utils.DeviceUtils.HealthyDevices(
+        default_retries=0, enable_device_files_cache=True)
     if not devices:
       raise device_errors.NoDevicesError()
     elif len(devices) == 1:
       device = devices[0]
     else:
       all_devices = device_utils.DeviceUtils.parallel(devices)
       msg = ('More than one device available.\n'
              'Use --device=SERIAL to select a device.\n'
              'Available devices:\n')
       descriptions = all_devices.pMap(lambda d: d.build_description).pGet(None)
       for d, desc in zip(devices, descriptions):
         msg += '  %s (%s)\n' % (d, desc)
       raise Exception(msg)
 
   apk_help = apk_helper.ApkHelper(args.apk_path)
   apk_package = apk_help.GetPackageName()
   device_incremental_dir = '/data/local/tmp/incremental-app-%s' % apk_package
 
   if args.uninstall:
     device.Uninstall(apk_package)
     device.RunShellCommand(['rm', '-rf', device_incremental_dir],
                            check_return=True)
     logging.info('Uninstall took %s seconds.', main_timer.GetDelta())
     return
 
-  if device.build_version_sdk >= version_codes.MARSHMALLOW:
-    if apk_help.HasIsolatedProcesses():
-      raise Exception('Cannot use perform incremental installs on Android M+ '
-                      'without first disabling isolated processes. Use GN arg: '
-                      'disable_incremental_isolated_processes=true to do so.')
-
   # Install .apk(s) if any of them have changed.
   def do_install():
     install_timer.Start()
     if args.splits:
       splits = []
       for split_glob in args.splits:
         splits.extend((f for f in glob.glob(split_glob)))
       device.InstallSplitApk(args.apk_path, splits, reinstall=True,
                              allow_cached_props=True, permissions=())
     else:
@@ skipping 16 matching lines @@
       with build_utils.TempDir() as temp_dir:
         device_dex_dir = posixpath.join(device_incremental_dir, 'dex')
         # Ensure no two files have the same name.
         transformed_names = _TransformDexPaths(args.dex_files)
         for src_path, dest_name in zip(args.dex_files, transformed_names):
           shutil.copyfile(src_path, os.path.join(temp_dir, dest_name))
         device.PushChangedFiles([(temp_dir, device_dex_dir)],
                                 delete_device_stale=True)
       push_dex_timer.Stop(log=False)
 
+  def check_sdk_version():
+    if device.build_version_sdk >= version_codes.MARSHMALLOW:
+      if apk_help.HasIsolatedProcesses():
+        raise Exception('Cannot use perform incremental installs on Android M+ '
+                        'without first disabling isolated processes.\n'
+                        'To do so, use GN arg:\n'
+                        '    disable_incremental_isolated_processes=true')
+
+  cache_path = '%s/files-cache.json' % device_incremental_dir
+  def restore_cache():
+    # Delete the cached file so that any exceptions cause the next attempt
+    # to re-compute md5s.
+    cmd = 'P=%s;cat $P 2>/dev/null && rm $P' % cache_path
+    lines = device.RunShellCommand(cmd, large_output=True)

[jbudorick, 2015/10/01 17:15:27]

This appears to be using the failure mode as a valid path. In that case, I'd
recommend either explicitly specifying check_return=False or specifying
check_return=True and catching an exception for the case where the cache doesn't
exist.

(I'm planning to eventually at least flip the default for check_return if not
switch it to always-on.)

[agrieve, 2015/10/01 18:30:16]

Done.

+    if lines:
+      device.LoadCacheData(lines[0])
+    else:
+      logging.info('Device cache not found: %s', cache_path)
+
+  def save_cache():
+    cache_data = device.DumpCacheData()
+    device.WriteFile(cache_path, cache_data)
+
   # Create 2 lock files:
   # * install.lock tells the app to pause on start-up (until we release it).
   # * firstrun.lock is used by the app to pause all secondary processes until
   #   the primary process finishes loading the .dex / .so files.
   def create_lock_files():
     # Creates or zeros out lock files.
     cmd = ('D="%s";'
            'mkdir -p $D &&'
            'echo -n >$D/install.lock 2>$D/firstrun.lock')
     device.RunShellCommand(cmd % device_incremental_dir, check_return=True)
 
   # The firstrun.lock is released by the app itself.
   def release_installer_lock():
     device.RunShellCommand('echo > %s/install.lock' % device_incremental_dir,
                            check_return=True)
 
-  create_lock_files()
   # Concurrency here speeds things up quite a bit, but DeviceUtils hasn't
   # been designed for multi-threading. Enabling only because this is a
   # developer-only tool.
-  if args.no_threading:
-    do_install()
-    do_push_files()
-  else:
-    reraiser_thread.RunAsync((do_install, do_push_files))
-  release_installer_lock()
-  logging.info('Took %s seconds (install=%s, libs=%s, dex=%s)',
-               main_timer.GetDelta(), install_timer.GetDelta(),
-               push_native_timer.GetDelta(), push_dex_timer.GetDelta())
+  concurrently = not args.no_threading

[jbudorick, 2015/10/01 17:15:27]

by using dest='threading' above, you could eliminate concurrently and just use
args.threading directly.

[agrieve, 2015/10/01 18:30:16]

Done. Although sadly the 80 character limit adds an extra level of complexity to
trying to write succinct code :(.

+  setup_timer = _Execute(
+      concurrently, (create_lock_files, restore_cache, check_sdk_version))
+
+  _Execute(concurrently, (do_install, do_push_files))
+
+  finalize_timer = _Execute(concurrently, (release_installer_lock, save_cache))
+
+  logging.info(
+      'Took %s seconds (setup=%s, install=%s, libs=%s, dex=%s, finalize=%s)',
+      main_timer.GetDelta(), setup_timer.GetDelta(), install_timer.GetDelta(),
+      push_native_timer.GetDelta(), push_dex_timer.GetDelta(),
+      finalize_timer.GetDelta())
 
 
 if __name__ == '__main__':
   sys.exit(main())