Cache device apk checksums in device_utils.py

build/android/pylib/device/device_utils.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Provides a variety of device interactions based on adb.
 
 Eventually, this will be based on adb_wrapper.
 """
 # pylint: disable=unused-argument
 
@@ skipping 158 matching lines @@
     elif isinstance(device, pylib.android_commands.AndroidCommands):
       self.adb = adb_wrapper.AdbWrapper(device.GetDevice())
       self.old_interface = device
     else:
       raise ValueError('Unsupported device value: %r' % device)
     self._commands_installed = None
     self._default_timeout = default_timeout
     self._default_retries = default_retries
     self._cache = {}
     self._client_caches = {}
+    self._package_to_device_apk_checksums_cache = {}
+    self._package_to_device_apk_paths_cache = {}
     assert hasattr(self, decorators.DEFAULT_TIMEOUT_ATTR)
     assert hasattr(self, decorators.DEFAULT_RETRIES_ATTR)
 
+    # We need to flush the cache on uninstall, so wrap self.adb.Uninstall
+    # in order to ensure we detect calls to it.
+    old_adb_uninstall = self.adb.Uninstall
+    def WrappedAdbUninstall(package_name, *args, **kwargs):
+      try:
+        old_adb_uninstall(package_name, *args, **kwargs)
+      except:
+        # Clear cache since we can't be sure of the state.
+        self._package_to_device_apk_paths_cache.pop(package_name, 0)
+        self._package_to_device_apk_checksums_cache.pop(package_name, 0)
+        raise
+      self._package_to_device_apk_paths_cache[package_name] = []

[jbudorick, 2015/08/06 19:28:22]

I would still prefer to use self._cache and do some additional cache management
for this key.

[agrieve, 2015/08/11 14:07:46]

Done.

+      self._package_to_device_apk_checksums_cache[package_name] = set()
+    self.adb.Uninstall = WrappedAdbUninstall
+
   def __eq__(self, other):
     """Checks whether |other| refers to the same device as |self|.
 
     Args:
       other: The object to compare to. This can be a basestring, an instance
         of adb_wrapper.AdbWrapper, or an instance of DeviceUtils.
     Returns:
       Whether |other| refers to the same device as |self|.
     """
     return self.adb.GetDeviceSerial() == str(other)
@@ skipping 153 matching lines @@
   @decorators.WithTimeoutAndRetriesFromInstance()
   def GetApplicationPaths(self, package, timeout=None, retries=None):
     """Get the paths of the installed apks on the device for the given package.
 
     Args:
       package: Name of the package.
 
     Returns:
       List of paths to the apks on the device for the given package.
     """
+    return self._GetApplicationPathsInternal(package)
+
+  def _GetApplicationPathsInternal(self, package, skip_cache=False):
+    cached_result = self._package_to_device_apk_paths_cache.get(package)
+    if cached_result is not None and not skip_cache:
+      return list(cached_result)
     # 'pm path' is liable to incorrectly exit with a nonzero number starting
     # in Lollipop.
     # TODO(jbudorick): Check if this is fixed as new Android versions are
     # released to put an upper bound on this.
     should_check_return = (self.build_version_sdk <
                            constants.ANDROID_SDK_VERSION_CODES.LOLLIPOP)
     output = self.RunShellCommand(
         ['pm', 'path', package], check_return=should_check_return)
     apks = []
     for line in output:
       if not line.startswith('package:'):
         raise device_errors.CommandFailedError(
             'pm path returned: %r' % '\n'.join(output), str(self))
       apks.append(line[len('package:'):])
+    self._package_to_device_apk_paths_cache[package] = list(apks)
     return apks
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def GetApplicationDataDirectory(self, package, timeout=None, retries=None):
     """Get the data directory on the device for the given package.
 
     Args:
       package: Name of the package.
 
     Returns:
@@ skipping 32 matching lines @@
     def sd_card_ready():
       try:
         self.RunShellCommand(['test', '-d', self.GetExternalStoragePath()],
                              check_return=True)
         return True
       except device_errors.AdbCommandFailedError:
         return False
 
     def pm_ready():
       try:
-        return self.GetApplicationPaths('android')
+        return self._GetApplicationPathsInternal('android', skip_cache=True)
       except device_errors.CommandFailedError:
         return False
 
     def boot_completed():
       return self.GetProp('sys.boot_completed') == '1'
 
     def wifi_enabled():
       return 'Wi-Fi is enabled' in self.RunShellCommand(['dumpsys', 'wifi'],
                                                         check_return=False)
 
@@ skipping 49 matching lines @@
       reinstall: A boolean indicating if we should keep any existing app data.
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
       CommandFailedError if the installation fails.
       CommandTimeoutError if the installation times out.
       DeviceUnreachableError on missing device.
     """
     package_name = apk_helper.GetPackageName(apk_path)
-    device_paths = self.GetApplicationPaths(package_name)
+    device_paths = self._GetApplicationPathsInternal(package_name)
     if device_paths:
       if len(device_paths) > 1:
         logging.warning(
             'Installing single APK (%s) when split APKs (%s) are currently '
             'installed.', apk_path, ' '.join(device_paths))
-      (files_to_push, _) = self._GetChangedAndStaleFiles(
-          apk_path, device_paths[0])
-      should_install = bool(files_to_push)
+      apks_to_install, host_checksums = (
+          self._ComputeStaleApks(package_name, [apk_path]))
+      should_install = bool(apks_to_install)
       if should_install and not reinstall:
         self.adb.Uninstall(package_name)
     else:
       should_install = True
+      host_checksums = None
     if should_install:
-      self.adb.Install(apk_path, reinstall=reinstall)
+      # We won't know the resulting device apk names.
+      self._package_to_device_apk_paths_cache.pop(package_name, 0)
+      try:
+        self.adb.Install(apk_path, reinstall=reinstall)
+        self._package_to_device_apk_checksums_cache[package_name] = (
+            host_checksums)
+      except:
+        self._package_to_device_apk_paths_cache.pop(package_name, 0)
+        raise
 
   @decorators.WithTimeoutAndRetriesDefaults(
       INSTALL_DEFAULT_TIMEOUT,
       INSTALL_DEFAULT_RETRIES)
   def InstallSplitApk(self, base_apk, split_apks, reinstall=False,
                       timeout=None, retries=None):
     """Install a split APK.
 
     Noop if all of the APK splits are already installed.
 
     Args:
       base_apk: A string of the path to the base APK.
       split_apks: A list of strings of paths of all of the APK splits.
       reinstall: A boolean indicating if we should keep any existing app data.
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
       CommandFailedError if the installation fails.
       CommandTimeoutError if the installation times out.
       DeviceUnreachableError on missing device.
       DeviceVersionError if device SDK is less than Android L.
     """
     self._CheckSdkLevel(constants.ANDROID_SDK_VERSION_CODES.LOLLIPOP)
 
     all_apks = [base_apk] + split_select.SelectSplits(
         self, base_apk, split_apks)
     package_name = apk_helper.GetPackageName(base_apk)
-    device_apk_paths = self.GetApplicationPaths(package_name)
+    device_apk_paths = self._GetApplicationPathsInternal(package_name)
 
     if device_apk_paths:
       partial_install_package = package_name
-      device_checksums = md5sum.CalculateDeviceMd5Sums(device_apk_paths, self)
-      host_checksums = md5sum.CalculateHostMd5Sums(all_apks)
-      apks_to_install = [k for (k, v) in host_checksums.iteritems()
-                         if v not in device_checksums.values()]
+      apks_to_install, host_checksums = (
+          self._ComputeStaleApks(package_name, all_apks))
       if apks_to_install and not reinstall:
         self.adb.Uninstall(package_name)
         partial_install_package = None
         apks_to_install = all_apks
     else:
       partial_install_package = None
       apks_to_install = all_apks
+      host_checksums = None
     if apks_to_install:
-      self.adb.InstallMultiple(
-          apks_to_install, partial=partial_install_package, reinstall=reinstall)
+      # We won't know the resulting device apk names.
+      self._package_to_device_apk_paths_cache.pop(package_name, 0)
+      try:
+        self.adb.InstallMultiple(
+            apks_to_install, partial=partial_install_package,
+            reinstall=reinstall)
+        self._package_to_device_apk_checksums_cache[package_name] = (
+            host_checksums)
+      except:
+        self._package_to_device_apk_paths_cache.pop(package_name, 0)

[jbudorick, 2015/08/06 19:28:22]

Do you need to do this? If we hit an exception in InstallMultiple, we won't have
put the package_name entry back in yet.

[agrieve, 2015/08/11 14:07:46]

Nice catch. Gone.

+        raise
 
   def _CheckSdkLevel(self, required_sdk_level):
     """Raises an exception if the device does not have the required SDK level.
     """
     if self.build_version_sdk < required_sdk_level:
       raise device_errors.DeviceVersionError(
           ('Requires SDK level %s, device is SDK level %s' %
            (required_sdk_level, self.build_version_sdk)),
            device_serial=self.adb.GetDeviceSerial())
 
@@ skipping 324 matching lines @@
 
     Raises:
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     # Check that the package exists before clearing it for android builds below
     # JB MR2. Necessary because calling pm clear on a package that doesn't exist
     # may never return.
     if ((self.build_version_sdk >=
          constants.ANDROID_SDK_VERSION_CODES.JELLY_BEAN_MR2)
-        or self.GetApplicationPaths(package)):
+        or self._GetApplicationPathsInternal(package)):
       self.RunShellCommand(['pm', 'clear', package], check_return=True)
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def SendKeyEvent(self, keycode, timeout=None, retries=None):
     """Sends a keycode to the device.
 
     See the pylib.constants.keyevent module for suitable keycode values.
 
     Args:
       keycode: A integer keycode to send to the device.
@@ skipping 103 matching lines @@
       to_push = []
       for host_abs_path, host_checksum in host_checksums.iteritems():
         device_abs_path = '%s/%s' % (
             real_device_path, os.path.relpath(host_abs_path, real_host_path))
         device_checksum = device_checksums.pop(device_abs_path, None)
         if device_checksum != host_checksum:
           to_push.append((host_abs_path, device_abs_path))
       to_delete = device_checksums.keys()
       return (to_push, to_delete)
 
+  def _ComputeDeviceChecksumsForApks(self, package_name):
+    ret = self._package_to_device_apk_checksums_cache.get(package_name)
+    if ret is None:
+      device_paths = self._GetApplicationPathsInternal(package_name)
+      file_to_checksums = md5sum.CalculateDeviceMd5Sums(device_paths, self)
+      ret = set(file_to_checksums.values())
+      self._package_to_device_apk_checksums_cache[package_name] = ret
+    return ret
+
+  def _ComputeStaleApks(self, package_name, host_apk_paths):

[jbudorick, 2015/08/06 19:28:22]

I'm curious about the component time savings of caching the application path &
of caching the checksum. I don't particularly like replicating the
GetChangedFiles logic, and if this isn't saving very much time I'd rather not
add it.

Alternatively, maybe we should be doing some amount of caching like this more
broadly.

[agrieve, 2015/08/11 14:07:46]

When I first started down this path, I tried to add caching to
_GetChangedAndStaleFiles but hit a wall. 

The fundamental difference is that there's no way to (quickly) know the
on-device file name of an apk given the host apk. That's why in InstallSplitApk,
it checks whether _any_ of the on-device .apk checksums match a host checksum to
know whether or not the file is stale.

This approach was already used by InstallSplitApk, but now is a bit more
formalized in these functions (with caching)

+    host_checksums = md5sum.CalculateHostMd5Sums(host_apk_paths)
+    device_checksums = self._ComputeDeviceChecksumsForApks(package_name)
+    stale_apks = [k for (k, v) in host_checksums.iteritems()
+                  if v not in device_checksums]
+    return stale_apks, set(host_checksums.values())
+
   def _PushFilesImpl(self, host_device_tuples, files):
     size = sum(host_utils.GetRecursiveDiskUsage(h) for h, _ in files)
     file_count = len(files)
     dir_size = sum(host_utils.GetRecursiveDiskUsage(h)
                    for h, _ in host_device_tuples)
     dir_file_count = 0
     for h, _ in host_device_tuples:
       if os.path.isdir(h):
         dir_file_count += sum(len(f) for _r, _d, f in os.walk(h))
       else:
@@ skipping 675 matching lines @@
     """Returns client cache."""
     if client_name not in self._client_caches:
       self._client_caches[client_name] = {}
     return self._client_caches[client_name]
 
   def _ClearCache(self):
     """Clears all caches."""
     for client in self._client_caches:
       self._client_caches[client].clear()
     self._cache.clear()
+    self._package_to_device_apk_paths_cache.clear()
+    self._package_to_device_apk_checksums_cache.clear()
 
   @classmethod
   def parallel(cls, devices=None, async=False):
     """Creates a Parallelizer to operate over the provided list of devices.
 
     If |devices| is either |None| or an empty list, the Parallelizer will
     operate over all attached devices that have not been blacklisted.
 
     Args:
       devices: A list of either DeviceUtils instances or objects from
@@ skipping 20 matching lines @@
   def HealthyDevices(cls):
     blacklist = device_blacklist.ReadBlacklist()
     def blacklisted(adb):
       if adb.GetDeviceSerial() in blacklist:
         logging.warning('Device %s is blacklisted.', adb.GetDeviceSerial())
         return True
       return False
 
     return [cls(adb) for adb in adb_wrapper.AdbWrapper.Devices()
             if not blacklisted(adb)]