Speed up incremental_install by caching device checksums between runs.

build/android/devil/android/device_utils.py

Index: build/android/devil/android/device_utils.py
diff --git a/build/android/devil/android/device_utils.py b/build/android/devil/android/device_utils.py
index 432677f91917734e31f48c9392339ce3c30de90d..1d315cd2757d26dccffb7bae5fe2e4b6cac9cca9 100644
--- a/build/android/devil/android/device_utils.py
+++ b/build/android/devil/android/device_utils.py
@@ -10,6 +10,7 @@ Eventually, this will be based on adb_wrapper.
 
 import collections
 import itertools
+import json
 import logging
 import multiprocessing
 import os
@@ -163,7 +164,8 @@ class DeviceUtils(object):
   # Property in /data/local.prop that controls Java assertions.
   JAVA_ASSERT_PROPERTY = 'dalvik.vm.enableassertions'
 
-  def __init__(self, device, default_timeout=_DEFAULT_TIMEOUT,
+  def __init__(self, device, enable_device_files_cache=False,

[jbudorick, 2015/10/01 17:15:26]

Document this parameter.

[agrieve, 2015/10/01 18:30:16]

Done.

+               default_timeout=_DEFAULT_TIMEOUT,
                default_retries=_DEFAULT_RETRIES):
     """DeviceUtils constructor.
 
@@ -187,6 +189,7 @@ class DeviceUtils(object):
     self._commands_installed = None
     self._default_timeout = default_timeout
     self._default_retries = default_retries
+    self._enable_device_files_cache = enable_device_files_cache
     self._cache = {}
     self._client_caches = {}
     assert hasattr(self, decorators.DEFAULT_TIMEOUT_ATTR)
@@ -1102,14 +1105,31 @@ class DeviceUtils(object):
         track_stale == False
     """
     try:
-      host_checksums = md5sum.CalculateHostMd5Sums([host_path])
-      interesting_device_paths = [device_path]
-      if not track_stale and os.path.isdir(host_path):
-        interesting_device_paths = [
-            posixpath.join(device_path, os.path.relpath(p, host_path))
-            for p in host_checksums.keys()]
-      device_checksums = md5sum.CalculateDeviceMd5Sums(
-          interesting_device_paths, self)
+      specific_device_paths = [device_path]
+      ignore_other_files = not track_stale and os.path.isdir(host_path)
+      if ignore_other_files:
+        specific_device_paths = []
+        for root, _, filenames in os.walk(host_path):
+          relative_dir = root[len(host_path) + 1:]
+          specific_device_paths.extend(
+              posixpath.join(device_path, relative_dir, f) for f in filenames)
+
+      def device_sums_helper():
+        if self._enable_device_files_cache:
+          cache_entry = self._cache['device_path_checksums'].get(device_path)
+          if cache_entry and cache_entry[0] == ignore_other_files:
+            return dict(cache_entry[1])
+
+        sums = md5sum.CalculateDeviceMd5Sums(specific_device_paths, self)
+
+        if self._enable_device_files_cache:
+          cache_entry = [ignore_other_files, sums]
+          self._cache['device_path_checksums'][device_path] = cache_entry
+        return dict(sums)
+
+      host_checksums, device_checksums = reraiser_thread.RunAsync((
+          lambda: md5sum.CalculateHostMd5Sums([host_path]),
+          device_sums_helper))
     except EnvironmentError as e:
       logging.warning('Error calculating md5: %s', e)
       return ([(host_path, device_path)], [], [])
@@ -1917,8 +1937,32 @@ class DeviceUtils(object):
         'package_apk_checksums': {},
         # Map of property_name -> value
         'getprop': {},
+        # Map of device_path -> [ignore_other_files, map of path->checksum]

[jbudorick, 2015/10/01 17:15:26]

Isn't this just a map of device_path -> checksum?

[agrieve, 2015/10/01 18:30:16]

The comment is correct. The reason it's not a map of device_path -> checksum:
- Say you push to /sdcard/foo:
  - Entries are now: /sdcard/foo/file1, /sdcard/foo/file2
- Say you now push to /sdcard:
  - You should not use the cached entries because they don't include all the
files.

Likewise, the ignore_other_files is necessary because it causes some files to be
skipped over when set.

+        'device_path_checksums': {},
     }
 
+  def LoadCacheData(self, data):
+    """Initializes the cache from data created using DumpCacheData."""
+    obj = json.loads(data)
+    self._cache['package_apk_paths'] = obj.get('package_apk_paths', {})
+    package_apk_checksums = obj.get('package_apk_checksums', {})
+    for k, v in package_apk_checksums.iteritems():
+      package_apk_checksums[k] = set(v)
+    self._cache['package_apk_checksums'] = package_apk_checksums
+    device_path_checksums = obj.get('device_path_checksums', {})
+    self._cache['device_path_checksums'] = device_path_checksums
+
+  def DumpCacheData(self):
+    """Dumps the current cache state to a string."""
+    obj = {}
+    obj['package_apk_paths'] = self._cache['package_apk_paths']
+    obj['package_apk_checksums'] = self._cache['package_apk_checksums']
+    # JSON can't handle sets.
+    for k, v in obj['package_apk_checksums'].iteritems():
+      obj['package_apk_checksums'][k] = list(v)
+    obj['device_path_checksums'] = self._cache['device_path_checksums']
+    return json.dumps(obj, separators=(',', ':'))
+
   @classmethod
   def parallel(cls, devices, async=False):
     """Creates a Parallelizer to operate over the provided list of devices.