Speed up incremental_install by caching device checksums between runs.

build/android/incremental_install/installer.py

Index: build/android/incremental_install/installer.py
diff --git a/build/android/incremental_install/installer.py b/build/android/incremental_install/installer.py
index bc0954b524aababe8dba37bd48b9691840d84cb0..b5696624ca5517c46558570bef51e870a316620b 100755
--- a/build/android/incremental_install/installer.py
+++ b/build/android/incremental_install/installer.py
@@ -34,6 +34,18 @@ def _TransformDexPaths(paths):
   return [p[prefix_len:].replace(os.sep, '.') for p in paths]
 
 
+def _Execute(concurrently, funcs):
+  """Calls all functions in |funcs| concurrently or in sequence."""
+  timer = time_profile.TimeProfile()
+  if concurrently:
+    reraiser_thread.RunAsync(funcs)
+  else:
+    for f in funcs:
+      f()
+  timer.Stop(log=False)
+  return timer
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('apk_path',
@@ -83,9 +95,11 @@ def main():
   if args.device:
     # Retries are annoying when commands fail for legitimate reasons. Might want
     # to enable them if this is ever used on bots though.
-    device = device_utils.DeviceUtils(args.device, default_retries=0)
+    device = device_utils.DeviceUtils(
+        args.device, default_retries=0, enable_device_files_cache=True)
   else:
-    devices = device_utils.DeviceUtils.HealthyDevices(default_retries=0)
+    devices = device_utils.DeviceUtils.HealthyDevices(
+        default_retries=0, enable_device_files_cache=True)
     if not devices:
       raise device_errors.NoDevicesError()
     elif len(devices) == 1:
@@ -111,12 +125,6 @@ def main():
     logging.info('Uninstall took %s seconds.', main_timer.GetDelta())
     return
 
-  if device.build_version_sdk >= version_codes.MARSHMALLOW:
-    if apk_help.HasIsolatedProcesses():
-      raise Exception('Cannot use perform incremental installs on Android M+ '
-                      'without first disabling isolated processes. Use GN arg: '
-                      'disable_incremental_isolated_processes=true to do so.')
-
   # Install .apk(s) if any of them have changed.
   def do_install():
     install_timer.Start()
@@ -153,6 +161,29 @@ def main():
                                 delete_device_stale=True)
       push_dex_timer.Stop(log=False)
 
+  def check_sdk_version():
+    if device.build_version_sdk >= version_codes.MARSHMALLOW:
+      if apk_help.HasIsolatedProcesses():
+        raise Exception('Cannot use perform incremental installs on Android M+ '
+                        'without first disabling isolated processes.\n'
+                        'To do so, use GN arg:\n'
+                        '    disable_incremental_isolated_processes=true')
+
+  cache_path = '%s/files-cache.json' % device_incremental_dir
+  def restore_cache():
+    # Delete the cached file so that any exceptions cause the next attempt
+    # to re-compute md5s.
+    cmd = 'P=%s;cat $P 2>/dev/null && rm $P' % cache_path
+    lines = device.RunShellCommand(cmd, large_output=True)

[jbudorick, 2015/10/01 17:15:27]

This appears to be using the failure mode as a valid path. In that case, I'd
recommend either explicitly specifying check_return=False or specifying
check_return=True and catching an exception for the case where the cache doesn't
exist.

(I'm planning to eventually at least flip the default for check_return if not
switch it to always-on.)

[agrieve, 2015/10/01 18:30:16]

Done.

+    if lines:
+      device.LoadCacheData(lines[0])
+    else:
+      logging.info('Device cache not found: %s', cache_path)
+
+  def save_cache():
+    cache_data = device.DumpCacheData()
+    device.WriteFile(cache_path, cache_data)
+
   # Create 2 lock files:
   # * install.lock tells the app to pause on start-up (until we release it).
   # * firstrun.lock is used by the app to pause all secondary processes until
@@ -169,19 +200,22 @@ def main():
     device.RunShellCommand('echo > %s/install.lock' % device_incremental_dir,
                            check_return=True)
 
-  create_lock_files()
   # Concurrency here speeds things up quite a bit, but DeviceUtils hasn't
   # been designed for multi-threading. Enabling only because this is a
   # developer-only tool.
-  if args.no_threading:
-    do_install()
-    do_push_files()
-  else:
-    reraiser_thread.RunAsync((do_install, do_push_files))
-  release_installer_lock()
-  logging.info('Took %s seconds (install=%s, libs=%s, dex=%s)',
-               main_timer.GetDelta(), install_timer.GetDelta(),
-               push_native_timer.GetDelta(), push_dex_timer.GetDelta())
+  concurrently = not args.no_threading

[jbudorick, 2015/10/01 17:15:27]

by using dest='threading' above, you could eliminate concurrently and just use
args.threading directly.

[agrieve, 2015/10/01 18:30:16]

Done. Although sadly the 80 character limit adds an extra level of complexity to
trying to write succinct code :(.

+  setup_timer = _Execute(
+      concurrently, (create_lock_files, restore_cache, check_sdk_version))
+
+  _Execute(concurrently, (do_install, do_push_files))
+
+  finalize_timer = _Execute(concurrently, (release_installer_lock, save_cache))
+
+  logging.info(
+      'Took %s seconds (setup=%s, install=%s, libs=%s, dex=%s, finalize=%s)',
+      main_timer.GetDelta(), setup_timer.GetDelta(), install_timer.GetDelta(),
+      push_native_timer.GetDelta(), push_dex_timer.GetDelta(),
+      finalize_timer.GetDelta())
 
 
 if __name__ == '__main__':