[Android] Add zip pushing and refine push mode selection.

build/android/pylib/device/device_utils.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Provides a variety of device interactions based on adb.
 
 Eventually, this will be based on adb_wrapper.
 """
 # pylint: disable=W0613
 
+import multiprocessing
+import os
 import pipes
+import shutil
 import sys
+import tempfile
 import time
+import zipfile
 
 import pylib.android_commands
 from pylib.device import adb_wrapper
 from pylib.device import decorators
 from pylib.device import device_errors
+from pylib.device.commands import install_commands
 from pylib.utils import apk_helper
+from pylib.utils import host_utils
 from pylib.utils import parallelizer
 
 _DEFAULT_TIMEOUT = 30
 _DEFAULT_RETRIES = 3
 
 
 @decorators.WithExplicitTimeoutAndRetries(
     _DEFAULT_TIMEOUT, _DEFAULT_RETRIES)
 def GetAVDs():
   """Returns a list of Android Virtual Devices.
@@ skipping 24 matching lines @@
     Args:
       device: Either a device serial, an existing AdbWrapper instance, an
               an existing AndroidCommands instance, or nothing.
       default_timeout: An integer containing the default number of seconds to
                        wait for an operation to complete if no explicit value
                        is provided.
       default_retries: An integer containing the default number or times an
                        operation should be retried on failure if no explicit
                        value is provided.
     """
+    self.adb = None
     self.old_interface = None
     if isinstance(device, basestring):
+      self.adb = adb_wrapper.AdbWrapper(device)
       self.old_interface = pylib.android_commands.AndroidCommands(device)
     elif isinstance(device, adb_wrapper.AdbWrapper):
+      self.adb = device
       self.old_interface = pylib.android_commands.AndroidCommands(str(device))
     elif isinstance(device, pylib.android_commands.AndroidCommands):
+      self.adb = adb_wrapper.AdbWrapper(device.GetDevice())
       self.old_interface = device
     elif not device:
+      self.adb = adb_wrapper.AdbWrapper('')
       self.old_interface = pylib.android_commands.AndroidCommands()
     else:
       raise ValueError('Unsupported type passed for argument "device"')
+    self._commands_installed = False
     self._default_timeout = default_timeout
     self._default_retries = default_retries
     assert(hasattr(self, decorators.DEFAULT_TIMEOUT_ATTR))
     assert(hasattr(self, decorators.DEFAULT_RETRIES_ATTR))
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def IsOnline(self, timeout=None, retries=None):
     """Checks whether the device is online.
 
     Args:
       timeout: timeout in seconds
       retries: number of retries
 
     Returns:
       True if the device is online, False otherwise.
 
     Raises:
       CommandTimeoutError on timeout.
     """
+    return self._IsOnlineImpl()
+
+  def _IsOnlineImpl(self):
     return self.old_interface.IsOnline()
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def HasRoot(self, timeout=None, retries=None):
     """Checks whether or not adbd has root privileges.
 
     Args:
       timeout: timeout in seconds
       retries: number of retries
 
     Returns:
       True if adbd has root privileges, False otherwise.
 
     Raises:
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     return self._HasRootImpl()
 
   def _HasRootImpl(self):
-    """Implementation of HasRoot.
-
-    This is split from HasRoot to allow other DeviceUtils methods to call
-    HasRoot without spawning a new timeout thread.
-
-    Returns:
-      Same as for |HasRoot|.
-
-    Raises:
-      Same as for |HasRoot|.
-    """
     return self.old_interface.IsRootEnabled()
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def EnableRoot(self, timeout=None, retries=None):
     """Restarts adbd with root privileges.
 
     Args:
       timeout: timeout in seconds
       retries: number of retries
 
@@ skipping 14 matching lines @@
       retries: number of retries
 
     Returns:
       The device's path to its SD card.
 
     Raises:
       CommandFailedError if the external storage path could not be determined.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
+    return self._GetExternalStoragePathImpl()
+
+  def _GetExternalStoragePathImpl(self):
     try:
       return self.old_interface.GetExternalStorage()
     except AssertionError as e:
       raise device_errors.CommandFailedError(
           str(e), device=str(self)), None, sys.exc_info()[2]
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def WaitUntilFullyBooted(self, wifi=False, timeout=None, retries=None):
     """Wait for the device to fully boot.
 
     This means waiting for the device to boot, the package manager to be
     available, and the SD card to be ready. It can optionally mean waiting
     for wifi to come up, too.
 
     Args:
       wifi: A boolean indicating if we should wait for wifi to come up or not.
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
       CommandFailedError on failure.
       CommandTimeoutError if one of the component waits times out.
       DeviceUnreachableError if the device becomes unresponsive.
     """
     self._WaitUntilFullyBootedImpl(wifi=wifi, timeout=timeout)
 
   def _WaitUntilFullyBootedImpl(self, wifi=False, timeout=None):
-    """Implementation of WaitUntilFullyBooted.
-
-    This is split from WaitUntilFullyBooted to allow other DeviceUtils methods
-    to call WaitUntilFullyBooted without spawning a new timeout thread.
-
-    TODO(jbudorick) Remove the timeout parameter once this is no longer
-    implemented via AndroidCommands.
-
-    Args:
-      wifi: Same as for |WaitUntilFullyBooted|.
-      timeout: timeout in seconds
-
-    Raises:
-      Same as for |WaitUntilFullyBooted|.
-    """
     if timeout is None:
       timeout = self._default_timeout
     self.old_interface.WaitForSystemBootCompleted(timeout)
     self.old_interface.WaitForDevicePm()
     self.old_interface.WaitForSdCardReady(timeout)
     if wifi:
       while not 'Wi-Fi is enabled' in (
           self._RunShellCommandImpl('dumpsys wifi')):
         time.sleep(1)
 
@@ skipping 63 matching lines @@
         out = self.old_interface.Install(apk_path, reinstall=reinstall)
         for line in out.splitlines():
           if 'Failure' in line:
             raise device_errors.CommandFailedError(
                 line.strip(), device=str(self))
       except AssertionError as e:
         raise device_errors.CommandFailedError(
             str(e), device=str(self)), None, sys.exc_info()[2]
 
   @decorators.WithTimeoutAndRetriesFromInstance()
-  def RunShellCommand(self, cmd, check_return=False, as_root=False,
-                      timeout=None, retries=None):
+  def RunShellCommand(self, cmd, check_return=False, as_root=False, cwd=None,
+                      env=None, timeout=None, retries=None):
     """Run an ADB shell command.
 
     TODO(jbudorick) Switch the default value of check_return to True after
     AndroidCommands is gone.
 
     Args:
       cmd: A list containing the command to run on the device and any arguments.
       check_return: A boolean indicating whether or not the return code should
                     be checked.
       as_root: A boolean indicating whether the shell command should be run
                with root privileges.
+      cwd: The device directory in which the command should be run.
+      env: The environment variables with which the command should be run.
       timeout: timeout in seconds
       retries: number of retries
 
     Returns:
       The output of the command.
 
     Raises:
       CommandFailedError if check_return is True and the return code is nozero.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
-    return self._RunShellCommandImpl(cmd, check_return=check_return,
-                                     as_root=as_root, timeout=timeout)
+    return self._RunShellCommandImpl(
+        cmd, check_return=check_return, as_root=as_root, cwd=cwd, env=env,
+        timeout=timeout)
 
   def _RunShellCommandImpl(self, cmd, check_return=False, as_root=False,
-                           timeout=None):
-    """Implementation of RunShellCommand.
-
-    This is split from RunShellCommand to allow other DeviceUtils methods to
-    call RunShellCommand without spawning a new timeout thread.
-
-    TODO(jbudorick) Remove the timeout parameter once this is no longer
-    implemented via AndroidCommands.
-
-    Args:
-      cmd: Same as for |RunShellCommand|.
-      check_return: Same as for |RunShellCommand|.
-      as_root: Same as for |RunShellCommand|.
-      timeout: timeout in seconds
-
-    Raises:
-      Same as for |RunShellCommand|.
-
-    Returns:
-      Same as for |RunShellCommand|.
-    """
+                           cwd=None, env=None, timeout=None):
+    # TODO(jbudorick): Remove the timeout parameter once this is no longer
+    # backed by AndroidCommands.
     if isinstance(cmd, list):
       cmd = ' '.join(cmd)
-    if as_root and not self.HasRoot():
+    if as_root and not self._HasRootImpl():
       cmd = 'su -c %s' % cmd
+    if env:
+      cmd = '%s %s' % (
+          ' '.join('%s=%s' % (k, v) for k, v in env.iteritems()), cmd)
+    if cwd:
+      cmd = 'cd %s && %s' % (cwd, cmd)
     if check_return:
       code, output = self.old_interface.GetShellCommandStatusAndOutput(
           cmd, timeout_time=timeout)
       if int(code) != 0:
         raise device_errors.AdbCommandFailedError(
             cmd.split(), 'Nonzero exit code (%d)' % code, device=str(self))
     else:
       output = self.old_interface.RunShellCommand(cmd, timeout_time=timeout)
     return output
 
@@ skipping 147 matching lines @@
       DeviceUnreachableError on missing device.
     """
     self.old_interface.SendKeyEvent(keycode)
 
   PUSH_CHANGED_FILES_DEFAULT_TIMEOUT = 10 * _DEFAULT_TIMEOUT
   PUSH_CHANGED_FILES_DEFAULT_RETRIES = _DEFAULT_RETRIES
 
   @decorators.WithTimeoutAndRetriesDefaults(
       PUSH_CHANGED_FILES_DEFAULT_TIMEOUT,
       PUSH_CHANGED_FILES_DEFAULT_RETRIES)
-  def PushChangedFiles(self, host_path, device_path, timeout=None,
+  def PushChangedFiles(self, host_device_tuples, timeout=None,
                        retries=None):
     """Push files to the device, skipping files that don't need updating.
 
     Args:
-      host_path: A string containing the absolute path to the file or directory
-                 on the host that should be minimally pushed to the device.
-      device_path: A string containing the absolute path of the destination on
-                   the device.
+      host_device_tuples: A list of (host_path, device_path) tuples, where
+        |host_path| is an absolute path of a file or directory on the host
+        that should be minimially pushed to the device, and |device_path| is
+        an absolute path of the destination on the device.
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
       CommandFailedError on failure.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
-    self.old_interface.PushIfNeeded(host_path, device_path)
+
+    files = []
+    for h, d in host_device_tuples:
+      if os.path.isdir(h) and not self._FileExistsImpl(d):
+        self._RunShellCommandImpl(['mkdir', '-p', '"%s"' % d],

[cjhopman, 2014/10/09 17:50:23]

It's may be a little faster to just unconditionally make the directory (since
almost all the cost of file_exists/mkdir is adb overhead).

[jbudorick, 2014/10/09 19:26:40]

dropped the _FileExistsImpl call

+                                  check_return=True)
+      files += self.old_interface.GetFilesChanged(h, d)
+
+    if not files:
+      return
+
+    size = sum(host_utils.GetRecursiveDiskUsage(h) for h, _ in files)
+    file_count = len(files)
+    dir_size = sum(host_utils.GetRecursiveDiskUsage(h)
+                   for h, _ in host_device_tuples)
+    dir_file_count = 0
+    for h, _ in host_device_tuples:
+      if os.path.isdir(h):
+        dir_file_count += sum(len(f) for _r, _d, f in os.walk(h))
+      else:
+        dir_file_count += 1
+
+    push_duration = self._ApproximateDuration(
+        file_count, file_count, size, False)
+    dir_push_duration = self._ApproximateDuration(
+        len(host_device_tuples), dir_file_count, dir_size, False)
+    zip_duration = self._ApproximateDuration(1, 1, size, True)
+
+    if dir_push_duration < push_duration and dir_push_duration < zip_duration:
+      self._PushChangedFilesIndividually(host_device_tuples)
+    elif push_duration < zip_duration:
+      self._PushChangedFilesIndividually(files)
+    else:
+      self._PushChangedFilesZipped(files)
+      self._RunShellCommandImpl(
+          ['chmod', '-R', '777'] + [d for _, d in host_device_tuples],
+          as_root=True)
+
+  @staticmethod
+  def _ApproximateDuration(adb_calls, file_count, byte_count, is_zipping):
+    # We approximate the time to push a set of files to a device as:
+    #   t = c1 * a + c2 * f + c3 + b / c4 + b / (c5 * c6), where
+    #     t: total time (sec)
+    #     c1: adb call time delay (sec)
+    #     a: number of times adb is called (unitless)
+    #     c2: push time delay (sec)
+    #     f: number of files pushed via adb (unitless)
+    #     c3: zip time delay (sec)
+    #     c4: zip rate (bytes/sec)
+    #     b: total number of bytes (bytes)
+    #     c5: transfer rate (bytes/sec)
+    #     c6: compression ratio (unitless)
+
+    # All of these are approximations.
+    ADB_CALL_PENALTY = 0.1 # seconds
+    ADB_PUSH_PENALTY = 0.01 # seconds
+    ZIP_PENALTY = 2.0 # seconds
+    ZIP_RATE = 10000000.0 # bytes / second
+    TRANSFER_RATE = 2000000.0 # bytes / second
+    COMPRESSION_RATIO = 2.0 # unitless
+
+    adb_call_time = ADB_CALL_PENALTY * adb_calls
+    adb_push_setup_time = ADB_PUSH_PENALTY * file_count
+    if is_zipping:
+      zip_time = ZIP_PENALTY + byte_count / ZIP_RATE
+      transfer_time = byte_count / (TRANSFER_RATE * COMPRESSION_RATIO)
+    else:
+      zip_time = 0
+      transfer_time = byte_count / TRANSFER_RATE
+    return (adb_call_time + adb_push_setup_time + zip_time + transfer_time)
+
+  def _PushChangedFilesIndividually(self, files):
+    for h, d in files:
+      self.adb.Push(h, d)
+
+  def _PushChangedFilesZipped(self, files):
+    if not files:
+      return
+
+    self._InstallCommands()
+    zip_dir = tempfile.mkdtemp()

[cjhopman, 2014/10/09 17:50:23]

this can use tempfile.NamedTemporaryFile for the zip file instead of the
hand-rolled temp dir.

[jbudorick, 2014/10/09 19:26:40]

What I actually want is a temporary file _name_, not an open temporary file
object. NamedTemporaryFile does the latter, and the logic it uses for temporary
name creation is (understandably) not public. Making a temporary directory
ensures that the temporary file I eventually create won't conflict with one
created on another thread accidentally.

I suppose something like:

  zip_file = tempfile.NamedTemporaryFile(suffix='.zip', delete=False)
  zip_file.close()
  zip_path = zip_file.name()
  try:
    ...
  finally:
    os.remove(zip_path)

but I think I prefer what I've got to that.

[cjhopman, 2014/10/09 20:27:39]

You can actually do:

with tempfile.NamedTemporaryFile(suffix='.zip'):
  zip_path = zip_file.name
  ...

and just keep the tempfile open, but it's up to you.

[jbudorick, 2014/10/10 09:33:47]

I was thinking python would choke on having the same file open in two processes,
but it works and results in cleaner code. Changed.

+    try:
+      zip_path = os.path.abspath(os.path.join(zip_dir, 'tmp.zip'))
+
+      zip_proc = multiprocessing.Process(
+          target=DeviceUtils._CreateDeviceZip,
+          args=(zip_path, files))
+      zip_proc.start()
+      zip_proc.join()
+
+      zip_on_device = '%s/tmp.zip' % self._GetExternalStoragePathImpl()
+      try:
+        self.adb.Push(zip_path, zip_on_device)
+        self._RunShellCommandImpl(
+            ['unzip', zip_on_device],
+            as_root=True, check_return=True,
+            env={'PATH': '$PATH:%s' % install_commands.BIN_DIR})
+      finally:
+        if zip_proc.is_alive():
+          zip_proc.terminate()
+        if self._IsOnlineImpl():
+          self._RunShellCommandImpl(['rm', zip_on_device])
+    finally:
+      shutil.rmtree(zip_dir)
+
+  def _InstallCommands(self):
+    if not self._commands_installed and not install_commands.Installed(self):
+      install_commands.InstallCommands(self)
+      self._commands_installed = True
+
+  @staticmethod
+  def _CreateDeviceZip(zip_path, host_device_tuples):
+    with zipfile.ZipFile(zip_path, 'w') as zip_file:
+      def AddDirectory(z, h, d):
+        z.write(h, d, zipfile.ZIP_STORED)
+        for i in os.listdir(h):
+          hi = os.path.join(h, i)
+          di = '%s/%s' % (d, i)
+          if os.path.isdir(i):
+            AddDirectory(z, hi, di)
+          else:
+            z.write(hi, di, zipfile.ZIP_DEFLATED)
+
+      for host_path, device_path in host_device_tuples:

[cjhopman, 2014/10/09 17:50:23]

I think this would be simpler with os.walk.

something like:

for host_path, device_path in host_device_tuples:
  for root, _, files in os.walk(host_path):
    hd = os.path.join(host_path, root)
    dd = '%s/%s' % (device_path, root)
    z.write(hd, dd, zipfile.ZIP_STORED)
    for f in files:
      z.write(os.path.join(hd, f), '%s/%s' % (dd, f), zipfile.ZIP_DEFLATED)

      

[jbudorick, 2014/10/09 19:26:40]

Done with some tweaks.

+        if os.path.isdir(host_path):
+          AddDirectory(zip_file, host_path, device_path)
+        else:
+          zip_file.write(host_path, device_path, zipfile.ZIP_DEFLATED)
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def FileExists(self, device_path, timeout=None, retries=None):
     """Checks whether the given file exists on the device.
 
     Args:
       device_path: A string containing the absolute path to the file on the
                    device.
       timeout: timeout in seconds
       retries: number of retries
 
     Returns:
       True if the file exists on the device, False otherwise.
 
     Raises:
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     return self._FileExistsImpl(device_path)
 
   def _FileExistsImpl(self, device_path):
-    """Implementation of FileExists.
-
-    This is split from FileExists to allow other DeviceUtils methods to call
-    FileExists without spawning a new timeout thread.
-
-    Args:
-      device_path: Same as for |FileExists|.
-
-    Returns:
-      True if the file exists on the device, False otherwise.
-
-    Raises:
-      Same as for |FileExists|.
-    """
     return self.old_interface.FileExistsOnDevice(device_path)
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def PullFile(self, device_path, host_path, timeout=None, retries=None):
     """Pull a file from the device.
 
     Args:
       device_path: A string containing the absolute path of the file to pull
                    from the device.
       host_path: A string containing the absolute path of the destination on
@@ skipping 24 matching lines @@
       retries: number of retries
 
     Returns:
       The contents of the file at |device_path| as a list of lines.
 
     Raises:
       CommandFailedError if the file can't be read.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
-    # TODO(jbudorick) Evaluate whether we awant to return a list of lines after
+    # TODO(jbudorick) Evaluate whether we want to return a list of lines after
     # the implementation switch, and if file not found should raise exception.
     if as_root:
       if not self.old_interface.CanAccessProtectedFileContents():
         raise device_errors.CommandFailedError(
           'Cannot read from %s with root privileges.' % device_path)
       return self.old_interface.GetProtectedFileContents(device_path)
     else:
       return self.old_interface.GetFileContents(device_path)
 
   @decorators.WithTimeoutAndRetriesFromInstance()
@@ skipping 136 matching lines @@
       A dict mapping process name to PID for each process that contained the
       provided |process_name|.
 
     Raises:
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     return self._GetPidsImpl(process_name)
 
   def _GetPidsImpl(self, process_name):
-    """Implementation of GetPids.
-
-    This is split from GetPids to allow other DeviceUtils methods to call
-    GetPids without spawning a new timeout thread.
-
-    Args:
-      process_name: A string containing the process name to get the PIDs for.
-
-    Returns:
-      A dict mapping process name to PID for each process that contained the
-      provided |process_name|.
-
-    Raises:
-      DeviceUnreachableError on missing device.
-    """
     procs_pids = {}
     for line in self._RunShellCommandImpl('ps'):
       try:
         ps_data = line.split()
         if process_name in ps_data[-1]:
           procs_pids[ps_data[-1]] = ps_data[1]
       except IndexError:
         pass
     return procs_pids
 
@@ skipping 80 matching lines @@
       A Parallelizer operating over |devices|.
     """
     if not devices or len(devices) == 0:
       devices = pylib.android_commands.GetAttachedDevices()
     parallelizer_type = (parallelizer.Parallelizer if async
                          else parallelizer.SyncParallelizer)
     return parallelizer_type([
         d if isinstance(d, DeviceUtils) else DeviceUtils(d)
         for d in devices])