remove stale test data on the device

build/android/pylib/device/device_utils.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Provides a variety of device interactions based on adb.
 
 Eventually, this will be based on adb_wrapper.
 """
 # pylint: disable=unused-argument
 
@@ skipping 467 matching lines @@
       retries: number of retries
 
     Raises:
       CommandFailedError if the installation fails.
       CommandTimeoutError if the installation times out.
       DeviceUnreachableError on missing device.
     """
     package_name = apk_helper.GetPackageName(apk_path)
     device_path = self.GetApplicationPath(package_name)
     if device_path is not None:
-      should_install = bool(self._GetChangedFilesImpl(apk_path, device_path))
+      (files_to_push, _) = self._GetChangedAndStaleFiles(
+          apk_path, device_path)
+      should_install = bool(files_to_push)
       if should_install and not reinstall:
         self.adb.Uninstall(package_name)
     else:
       should_install = True
     if should_install:
       self.adb.Install(apk_path, reinstall=reinstall)
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def RunShellCommand(self, cmd, check_return=False, cwd=None, env=None,
                       as_root=False, single_line=False, large_output=False,
@@ skipping 326 matching lines @@
     self.RunShellCommand(['input', 'keyevent', format(keycode, 'd')],
                          check_return=True)
 
   PUSH_CHANGED_FILES_DEFAULT_TIMEOUT = 10 * _DEFAULT_TIMEOUT
   PUSH_CHANGED_FILES_DEFAULT_RETRIES = _DEFAULT_RETRIES
 
   @decorators.WithTimeoutAndRetriesDefaults(
       PUSH_CHANGED_FILES_DEFAULT_TIMEOUT,
       PUSH_CHANGED_FILES_DEFAULT_RETRIES)
   def PushChangedFiles(self, host_device_tuples, timeout=None,
-                       retries=None):
+                       retries=None, delete_device_stale=False):
     """Push files to the device, skipping files that don't need updating.
 
+    When a directory is pushed, it is traversed recursively on the host and
+    all files in it are pushed to the device as needed.
+    Additionally, if delete_device_stale option is True,
+    files that exist on the device but don't exist on the host are deleted.
+
     Args:
       host_device_tuples: A list of (host_path, device_path) tuples, where
         |host_path| is an absolute path of a file or directory on the host
         that should be minimially pushed to the device, and |device_path| is
         an absolute path of the destination on the device.
       timeout: timeout in seconds
       retries: number of retries
+      delete_device_stale: option to delete stale files on device
 
     Raises:
       CommandFailedError on failure.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
 
-    files = []
+    all_changed_files = []
+    all_stale_files = []
     for h, d in host_device_tuples:
       if os.path.isdir(h):
         self.RunShellCommand(['mkdir', '-p', d], check_return=True)
-      files += self._GetChangedFilesImpl(h, d)
+      (changed_files, stale_files) = self._GetChangedAndStaleFiles(h, d)
+      all_changed_files += changed_files
+      all_stale_files += stale_files
 
-    if not files:
+    if delete_device_stale:
+      for stale_file_path in all_stale_files:
+        self.RunShellCommand(['rm', '-f', stale_file_path],

[jbudorick, 2015/06/12 19:54:44]

Does this work as

  self.RunShellCommand(['rm', '-f'] + all_stale_files, ...)

?

[Menglin, 2015/06/12 22:31:46]

Yes it does, though in the description of the function writes
This (the cmd being a sequence)
allows to easily pass arguments containing spaces or special characters
without having to worry about getting quoting right. Whenever possible, it
is recomended to pass |cmd| as a sequence.

[perezju, 2015/06/15 09:03:52]

I guess what John is saying is whether we could remove all files with a single
shell call, instead of removing one by one (and yes, always passing as a list of
args, since that is preferred).

I thought about that too, but wasn't sure how nicely would adb play with that if
there are many files and the command becomes really big. I know we have other
checks in place for that, but adb always makes me nervous :P. Anyway, as John
suggested, let's go bold and try removing all_stale_files in one go. That would
certainly be more efficient in most cases.

As a side (not for this CL), maybe we want to provide a RemoveFiles method?
Seems to be a common operation and small details (e.g. the need for '-f' and
check_return) are easy to slip.

+                             check_return=True)
+
+    if not all_changed_files:
       return
 
+    self._PushFilesImpl(host_device_tuples, all_changed_files)
+
+  def _GetChangedAndStaleFiles(self, host_path, device_path):
+    """Get files to push and delete
+
+    Args:
+      host_path: an absolute path of a file or directory on the host
+      device_path: an absolute path of a file or directory on the device
+
+    Returns:
+      a two-element tuple
+      1st element: a list of (host_files_path, device_files_path) tuples to push
+      2nd element: a list of stale files under device_path
+    """
+    real_host_path = os.path.realpath(host_path)
+    try:
+      real_device_path = self.RunShellCommand(
+          ['realpath', device_path], single_line=True, check_return=True)
+    except device_errors.CommandFailedError:
+      real_device_path = None
+    if not real_device_path:
+      return ([(host_path, device_path)], [])
+
+    try:
+      host_checksums = md5sum.CalculateHostMd5Sums([real_host_path])
+      device_checksums = md5sum.CalculateDeviceMd5Sums(
+        [real_device_path], self)
+    except EnvironmentError as e:
+      logging.warning('Error calculating md5: %s', e)
+      return ([(host_path, device_path)], [])
+
+    if os.path.isfile(host_path):
+      host_checksum = host_checksums.get(real_host_path)
+      device_checksum = device_checksums.get(real_device_path)
+      if host_checksum != device_checksum:
+        return ([(host_path, device_path)], [])
+      else:
+        return ([], [])
+    else:
+      to_push = []
+      to_delete = []

[jbudorick, 2015/06/12 19:54:45]

nit: delete this line

[Menglin, 2015/06/12 22:31:46]

Done.

+      for host_abs_path, host_checksum in host_checksums.iteritems():
+        device_abs_path = '%s/%s' % (
+            real_device_path, os.path.relpath(host_abs_path, real_host_path))
+        device_checksum = device_checksums.pop(device_abs_path, None)
+        if device_checksum != host_checksum:
+          to_push.append((host_abs_path, device_abs_path))
+      to_delete = device_checksums.keys()
+      return (to_push, to_delete)
+
+  def _PushFilesImpl(self, host_device_tuples, files):
     size = sum(host_utils.GetRecursiveDiskUsage(h) for h, _ in files)
     file_count = len(files)
     dir_size = sum(host_utils.GetRecursiveDiskUsage(h)
                    for h, _ in host_device_tuples)
     dir_file_count = 0
     for h, _ in host_device_tuples:
       if os.path.isdir(h):
         dir_file_count += sum(len(f) for _r, _d, f in os.walk(h))
       else:
         dir_file_count += 1
@@ skipping 10 matching lines @@
         dir_push_duration < zip_duration or not self._commands_installed):
       self._PushChangedFilesIndividually(host_device_tuples)
     elif push_duration < zip_duration or not self._commands_installed:
       self._PushChangedFilesIndividually(files)
     else:
       self._PushChangedFilesZipped(files)
       self.RunShellCommand(
           ['chmod', '-R', '777'] + [d for _, d in host_device_tuples],
           as_root=True, check_return=True)
 
-  def _GetChangedFilesImpl(self, host_path, device_path):
-    real_host_path = os.path.realpath(host_path)
-    try:
-      real_device_path = self.RunShellCommand(
-          ['realpath', device_path], single_line=True, check_return=True)
-    except device_errors.CommandFailedError:
-      real_device_path = None
-    if not real_device_path:
-      return [(host_path, device_path)]
-
-    try:
-      host_checksums = md5sum.CalculateHostMd5Sums([real_host_path])
-      device_paths_to_md5 = (
-          real_device_path if os.path.isfile(real_host_path)
-          else ('%s/%s' % (real_device_path, os.path.relpath(p, real_host_path))
-                for p in host_checksums.iterkeys()))
-      device_checksums = md5sum.CalculateDeviceMd5Sums(
-          device_paths_to_md5, self)
-    except EnvironmentError as e:
-      logging.warning('Error calculating md5: %s', e)
-      return [(host_path, device_path)]
-
-    if os.path.isfile(host_path):
-      host_checksum = host_checksums.get(real_host_path)
-      device_checksum = device_checksums.get(real_device_path)
-      if host_checksum != device_checksum:
-        return [(host_path, device_path)]
-      else:
-        return []
-    else:
-      to_push = []
-      for host_abs_path, host_checksum in host_checksums.iteritems():
-        device_abs_path = '%s/%s' % (
-            real_device_path, os.path.relpath(host_abs_path, real_host_path))
-        if (device_checksums.get(device_abs_path) != host_checksum):
-          to_push.append((host_abs_path, device_abs_path))
-      return to_push
-
   def _InstallCommands(self):
     if self._commands_installed is None:
       try:
         if not install_commands.Installed(self):
           install_commands.InstallCommands(self)
         self._commands_installed = True
       except Exception as e:
         logging.warning('unzip not available: %s' % str(e))
         self._commands_installed = False
 
@@ skipping 662 matching lines @@
     blacklist = device_blacklist.ReadBlacklist()
     def blacklisted(adb):
       if adb.GetDeviceSerial() in blacklist:
         logging.warning('Device %s is blacklisted.', adb.GetDeviceSerial())
         return True
       return False
 
     return [cls(adb) for adb in adb_wrapper.AdbWrapper.Devices()
             if not blacklisted(adb)]