Speed up incremental_install by caching device checksums between runs.

build/android/devil/android/device_utils.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Provides a variety of device interactions based on adb.
 
 Eventually, this will be based on adb_wrapper.
 """
 # pylint: disable=unused-argument
 
 import collections
 import itertools
+import json
 import logging
 import multiprocessing
 import os
 import posixpath
 import re
 import shutil
 import tempfile
 import time
 import zipfile
 
@@ skipping 133 matching lines @@
   _MAX_ADB_OUTPUT_LENGTH = 32768
   _LAUNCHER_FOCUSED_RE = re.compile(
       r'\s*mCurrentFocus.*(Launcher|launcher).*')
   _VALID_SHELL_VARIABLE = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
 
   LOCAL_PROPERTIES_PATH = posixpath.join('/', 'data', 'local.prop')
 
   # Property in /data/local.prop that controls Java assertions.
   JAVA_ASSERT_PROPERTY = 'dalvik.vm.enableassertions'
 
-  def __init__(self, device, default_timeout=_DEFAULT_TIMEOUT,
+  def __init__(self, device, enable_device_files_cache=False,
+               default_timeout=_DEFAULT_TIMEOUT,
                default_retries=_DEFAULT_RETRIES):
     """DeviceUtils constructor.
 
     Args:
       device: Either a device serial, an existing AdbWrapper instance, or an
-              an existing AndroidCommands instance.
+        an existing AndroidCommands instance.
+      enable_device_files_cache: For PushChangedFiles(), cache checksums of
+        pushed files rather than recomputing them on a subsequent call.
       default_timeout: An integer containing the default number of seconds to
-                       wait for an operation to complete if no explicit value
-                       is provided.
+        wait for an operation to complete if no explicit value is provided.
       default_retries: An integer containing the default number or times an
-                       operation should be retried on failure if no explicit
-                       value is provided.
+        operation should be retried on failure if no explicit value is provided.
     """
     self.adb = None
     if isinstance(device, basestring):
       self.adb = adb_wrapper.AdbWrapper(device)
     elif isinstance(device, adb_wrapper.AdbWrapper):
       self.adb = device
     else:
       raise ValueError('Unsupported device value: %r' % device)
     self._commands_installed = None
     self._default_timeout = default_timeout
     self._default_retries = default_retries
+    self._enable_device_files_cache = enable_device_files_cache
     self._cache = {}
     self._client_caches = {}
     assert hasattr(self, decorators.DEFAULT_TIMEOUT_ATTR)
     assert hasattr(self, decorators.DEFAULT_RETRIES_ATTR)
 
     self._ClearCache()
 
   def __eq__(self, other):
     """Checks whether |other| refers to the same device as |self|.
 
@@ skipping 861 matching lines @@
 
     Raises:
       CommandFailedError on failure.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
 
     all_changed_files = []
     all_stale_files = []
     missing_dirs = []
+    cache_commit_funcs = []
     for h, d in host_device_tuples:
-      changed_files, up_to_date_files, stale_files = (
+      changed_files, up_to_date_files, stale_files, cache_commit_func = (
           self._GetChangedAndStaleFiles(h, d, delete_device_stale))
       all_changed_files += changed_files
       all_stale_files += stale_files
+      cache_commit_funcs.append(cache_commit_func)
       if (os.path.isdir(h) and changed_files and not up_to_date_files
           and not stale_files):
         missing_dirs.append(d)
 
     if delete_device_stale and all_stale_files:
       self.RunShellCommand(['rm', '-f'] + all_stale_files,
                              check_return=True)
 
     if all_changed_files:
       if missing_dirs:
         self.RunShellCommand(['mkdir', '-p'] + missing_dirs, check_return=True)
       self._PushFilesImpl(host_device_tuples, all_changed_files)
+    for func in cache_commit_funcs:
+      func()
 
   def _GetChangedAndStaleFiles(self, host_path, device_path, track_stale=False):
     """Get files to push and delete
 
     Args:
       host_path: an absolute path of a file or directory on the host
       device_path: an absolute path of a file or directory on the device
       track_stale: whether to bother looking for stale files (slower)
 
     Returns:
       a three-element tuple
       1st element: a list of (host_files_path, device_files_path) tuples to push
       2nd element: a list of host_files_path that are up-to-date
       3rd element: a list of stale files under device_path, or [] when
         track_stale == False
     """
     try:
+      # Length calculations below assume no trailing /.
+      host_path = host_path.rstrip('/')
+      device_path = device_path.rstrip('/')
+
       specific_device_paths = [device_path]
-      if not track_stale and os.path.isdir(host_path):
+      ignore_other_files = not track_stale and os.path.isdir(host_path)
+      if ignore_other_files:
         specific_device_paths = []
         for root, _, filenames in os.walk(host_path):
           relative_dir = root[len(host_path) + 1:]
           specific_device_paths.extend(
               posixpath.join(device_path, relative_dir, f) for f in filenames)
 
+      def device_sums_helper():
+        if self._enable_device_files_cache:
+          cache_entry = self._cache['device_path_checksums'].get(device_path)
+          if cache_entry and cache_entry[0] == ignore_other_files:
+            return dict(cache_entry[1])
+
+        sums = md5sum.CalculateDeviceMd5Sums(specific_device_paths, self)
+
+        if self._enable_device_files_cache:
+          cache_entry = [ignore_other_files, sums]
+          self._cache['device_path_checksums'][device_path] = cache_entry
+        return dict(sums)
+
       host_checksums, device_checksums = reraiser_thread.RunAsync((
           lambda: md5sum.CalculateHostMd5Sums([host_path]),
-          lambda: md5sum.CalculateDeviceMd5Sums(specific_device_paths, self)))
+          device_sums_helper))
     except EnvironmentError as e:
       logging.warning('Error calculating md5: %s', e)
-      return ([(host_path, device_path)], [], [])
+      return ([(host_path, device_path)], [], [], lambda: 0)
 
     to_push = []
     up_to_date = []
     to_delete = []
     if os.path.isfile(host_path):
       host_checksum = host_checksums.get(host_path)
       device_checksum = device_checksums.get(device_path)
       if host_checksum == device_checksum:
         up_to_date.append(host_path)
       else:
         to_push.append((host_path, device_path))
     else:
       for host_abs_path, host_checksum in host_checksums.iteritems():
         device_abs_path = posixpath.join(
             device_path, os.path.relpath(host_abs_path, host_path))
         device_checksum = device_checksums.pop(device_abs_path, None)
         if device_checksum == host_checksum:
           up_to_date.append(host_abs_path)
         else:
           to_push.append((host_abs_path, device_abs_path))
       to_delete = device_checksums.keys()
-    return (to_push, up_to_date, to_delete)
+
+    def cache_commit_func():
+      if not self._enable_device_files_cache:
+        return
+      new_sums = {posixpath.join(device_path, path[len(host_path) + 1:]): val
+                  for path, val in host_checksums.iteritems()}
+      cache_entry = [ignore_other_files, new_sums]
+      self._cache['device_path_checksums'][device_path] = cache_entry
+
+    return (to_push, up_to_date, to_delete, cache_commit_func)
 
   def _ComputeDeviceChecksumsForApks(self, package_name):
     ret = self._cache['package_apk_checksums'].get(package_name)
     if ret is None:
       device_paths = self._GetApplicationPathsInternal(package_name)
       file_to_checksums = md5sum.CalculateDeviceMd5Sums(device_paths, self)
       ret = set(file_to_checksums.values())
       self._cache['package_apk_checksums'][package_name] = ret
     return ret
 
@@ skipping 762 matching lines @@
     """Clears all caches."""
     for client in self._client_caches:
       self._client_caches[client].clear()
     self._cache = {
         # Map of packageId -> list of on-device .apk paths
         'package_apk_paths': {},
         # Map of packageId -> set of on-device .apk checksums
         'package_apk_checksums': {},
         # Map of property_name -> value
         'getprop': {},
+        # Map of device_path -> [ignore_other_files, map of path->checksum]
+        'device_path_checksums': {},
     }
 
+  def LoadCacheData(self, data):
+    """Initializes the cache from data created using DumpCacheData."""
+    obj = json.loads(data)
+    self._cache['package_apk_paths'] = obj.get('package_apk_paths', {})
+    package_apk_checksums = obj.get('package_apk_checksums', {})
+    for k, v in package_apk_checksums.iteritems():
+      package_apk_checksums[k] = set(v)
+    self._cache['package_apk_checksums'] = package_apk_checksums
+    device_path_checksums = obj.get('device_path_checksums', {})
+    self._cache['device_path_checksums'] = device_path_checksums
+
+  def DumpCacheData(self):
+    """Dumps the current cache state to a string."""
+    obj = {}
+    obj['package_apk_paths'] = self._cache['package_apk_paths']
+    obj['package_apk_checksums'] = self._cache['package_apk_checksums']
+    # JSON can't handle sets.
+    for k, v in obj['package_apk_checksums'].iteritems():
+      obj['package_apk_checksums'][k] = list(v)
+    obj['device_path_checksums'] = self._cache['device_path_checksums']
+    return json.dumps(obj, separators=(',', ':'))
+
   @classmethod
   def parallel(cls, devices, async=False):
     """Creates a Parallelizer to operate over the provided list of devices.
 
     If |devices| is either |None| or an empty list, the Parallelizer will
     operate over all attached devices that have not been blacklisted.
 
     Args:
       devices: A list of either DeviceUtils instances or objects from
                from which DeviceUtils instances can be constructed. If None,
@@ skipping 47 matching lines @@
     if ('android.permission.WRITE_EXTERNAL_STORAGE' in permissions
         and 'android.permission.READ_EXTERNAL_STORAGE' not in permissions):
       permissions.append('android.permission.READ_EXTERNAL_STORAGE')
     cmd = ';'.join('pm grant %s %s' %(package, p) for p in permissions)
     if cmd:
       output = self.RunShellCommand(cmd)
       if output:
         logging.warning('Possible problem when granting permissions. Blacklist '
                         'may need to be updated.')
         logging.warning(output)