New run shell implementation for DeviceUtils

build/android/pylib/device/device_utils.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Provides a variety of device interactions based on adb.
 
 Eventually, this will be based on adb_wrapper.
 """
 # pylint: disable=W0613
 
 import logging
 import multiprocessing
 import os
-import pipes
+import re
 import sys
 import tempfile
 import time
 import zipfile
 
 import pylib.android_commands
+from pylib import cmd_helper
 from pylib.device import adb_wrapper
 from pylib.device import decorators
 from pylib.device import device_errors
 from pylib.device.commands import install_commands
 from pylib.utils import apk_helper
 from pylib.utils import host_utils
 from pylib.utils import parallelizer
 
 _DEFAULT_TIMEOUT = 30
 _DEFAULT_RETRIES = 3
+_VALID_SHELL_VARIABLE = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')

[jbudorick, 2014/10/17 09:04:53]

Not sure about having this at this scope; it should probably be a constant in
DeviceUtils.

(The other two are out here because they're used by top level functions. All
other constants in this file are defined within DeviceUtils.)

[perezju, 2014/10/17 11:17:09]

Done.

 
 
 @decorators.WithExplicitTimeoutAndRetries(
     _DEFAULT_TIMEOUT, _DEFAULT_RETRIES)
 def GetAVDs():
   """Returns a list of Android Virtual Devices.
 
   Returns:
     A list containing the configured AVDs.
   """
@@ skipping 39 matching lines @@
       self.adb = adb_wrapper.AdbWrapper(device.GetDevice())
       self.old_interface = device
     elif not device:
       self.adb = adb_wrapper.AdbWrapper('')
       self.old_interface = pylib.android_commands.AndroidCommands()
     else:
       raise ValueError('Unsupported type passed for argument "device"')
     self._commands_installed = None
     self._default_timeout = default_timeout
     self._default_retries = default_retries
+    self._cache = {}
     assert(hasattr(self, decorators.DEFAULT_TIMEOUT_ATTR))
     assert(hasattr(self, decorators.DEFAULT_RETRIES_ATTR))
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def IsOnline(self, timeout=None, retries=None):
     """Checks whether the device is online.
 
     Args:
       timeout: timeout in seconds
       retries: number of retries
@@ skipping 20 matching lines @@
     Returns:
       True if adbd has root privileges, False otherwise.
 
     Raises:
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     return self._HasRootImpl()
 
   def _HasRootImpl(self):
-    return self.old_interface.IsRootEnabled()
+    try:

[jbudorick, 2014/10/17 09:04:53]

Sneaking a conversion in here, I see.

+      self._RunShellCommandImpl('ls /root')
+      return True
+    except device_errors.AdbShellCommandFailedError:
+      return False
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def EnableRoot(self, timeout=None, retries=None):
     """Restarts adbd with root privileges.
 
     Args:
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
@@ skipping 34 matching lines @@
       The device's path to its SD card.
 
     Raises:
       CommandFailedError if the external storage path could not be determined.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     return self._GetExternalStoragePathImpl()
 
   def _GetExternalStoragePathImpl(self):
-    try:
-      return self.old_interface.GetExternalStorage()
-    except AssertionError as e:
-      raise device_errors.CommandFailedError(
-          str(e), device=str(self)), None, sys.exc_info()[2]
+    if 'external_storage' in self._cache:

[jbudorick, 2014/10/17 09:04:53]

Another sneaky conversion.

+      return self._cache['external_storage']
+
+    value = self._RunShellCommandImpl('echo $EXTERNAL_STORAGE').rstrip()
+    if not value:
+      raise device_errors.CommandFailedError('$EXTERNAL_STORAGE is not set',
+                                             str(self))
+    self._cache['external_storage'] = value
+    return value
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def WaitUntilFullyBooted(self, wifi=False, timeout=None, retries=None):
     """Wait for the device to fully boot.
 
     This means waiting for the device to boot, the package manager to be
     available, and the SD card to be ready. It can optionally mean waiting
     for wifi to come up, too.
 
     Args:
       wifi: A boolean indicating if we should wait for wifi to come up or not.
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
       CommandFailedError on failure.
       CommandTimeoutError if one of the component waits times out.
       DeviceUnreachableError if the device becomes unresponsive.
     """
     self._WaitUntilFullyBootedImpl(wifi=wifi, timeout=timeout)
 
   def _WaitUntilFullyBootedImpl(self, wifi=False, timeout=None):
     if timeout is None:
       timeout = self._default_timeout
     self.old_interface.WaitForSystemBootCompleted(timeout)
     self.old_interface.WaitForDevicePm()
     self.old_interface.WaitForSdCardReady(timeout)
     if wifi:
       while not 'Wi-Fi is enabled' in (
-          self._RunShellCommandImpl('dumpsys wifi')):
+          self.old_interface.RunShellCommand('dumpsys wifi')):

[jbudorick, 2014/10/17 09:04:53]

What's up with this? Is this because of the unit test?

[perezju, 2014/10/17 11:17:09]

Yes, because all the others are calls to old_interface, I had to also move back
this call or try writing some DeviceUtilsHybridImplTest with mixed calls to
run_command.RunCommand and adb.Shell. :S

[jbudorick, 2014/10/17 15:48:00]

Hmm. I'm not really a fan of either of those options, but I can't think of a
reasonable alternative.

         time.sleep(1)
 
   REBOOT_DEFAULT_TIMEOUT = 10 * _DEFAULT_TIMEOUT
   REBOOT_DEFAULT_RETRIES = _DEFAULT_RETRIES
 
   @decorators.WithTimeoutAndRetriesDefaults(
       REBOOT_DEFAULT_TIMEOUT,
       REBOOT_DEFAULT_RETRIES)
   def Reboot(self, block=True, timeout=None, retries=None):
     """Reboot the device.
@@ skipping 55 matching lines @@
         out = self.old_interface.Install(apk_path, reinstall=reinstall)
         for line in out.splitlines():
           if 'Failure' in line:
             raise device_errors.CommandFailedError(
                 line.strip(), device=str(self))
       except AssertionError as e:
         raise device_errors.CommandFailedError(
             str(e), device=str(self)), None, sys.exc_info()[2]
 
   @decorators.WithTimeoutAndRetriesFromInstance()
-  def RunShellCommand(self, cmd, check_return=False, as_root=False, cwd=None,
-                      env=None, timeout=None, retries=None):
+  def RunShellCommand(self, cmd, check_return=False, want_lines=True,
+                      cwd=None, env=None, as_root=False,
+                      timeout=None, retries=None):
     """Run an ADB shell command.
 
-    TODO(jbudorick) Switch the default value of check_return to True after
-    AndroidCommands is gone.
+    When the command to run |cmd| is given as a string, it will be interpreted
+    and run by the shell on the device.
+
+    Alternatively, one can supply |cmd| as a sequence containing the name of
+    the command to run followed by its arguments. In this case, arguments are
+    passed to the command exactly as given, without any further processing by
+    the shell. This allows to easily pass arguments containing spaces or
+    special characters without having to worry about getting quoting right.
+
+    TODO(perezju) Remove the options |check_return| and |want_lines|,
+      or change their default vaules, when callers have switched to the new
+      behaviour.
 
     Args:
-      cmd: A list containing the command to run on the device and any arguments.
+      cmd: A string with the full command to run on the device, or a sequence
+        containing the command and its arguments.
       check_return: A boolean indicating whether or not the return code should
-                    be checked.
-      as_root: A boolean indicating whether the shell command should be run
-               with root privileges.
+        be checked.
+      want_lines: A boolean indicating whether to return the output of the

[jbudorick, 2014/10/17 09:04:53]

After looking at this and thinking about it, I'm not crazy about this. I'm not
convinced that an entire string _should_ be the new behavior.

Even if we do want to go that way, we shouldn't attempt to do so in this CL. The
switch to AdbWrapper.Shell has sweeping effects on our code as it is.

+        command as a list of lines or a single string.
       cwd: The device directory in which the command should be run.
       env: The environment variables with which the command should be run.
+      as_root: A boolean indicating whether the shell command should be run
+        with root privileges.
       timeout: timeout in seconds
       retries: number of retries
 
     Returns:
-      The output of the command.
+      The output of the command either as list of lines, when want_lines is
+        True, or a single string, otherwise.
 
     Raises:
-      CommandFailedError if check_return is True and the return code is nozero.
+      AdbShellCommandFailedError if check_return is True and the exit code of
+        the command run on the device is non-zero.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
-    return self._RunShellCommandImpl(
-        cmd, check_return=check_return, as_root=as_root, cwd=cwd, env=env,
-        timeout=timeout)
+    try:

[jbudorick, 2014/10/17 09:04:53]

This implementation all should move into the _Impl function. The _only_ thing in
here should be the call to _RunShellCommandImpl. The only point of the _Impl
split is to allow calling functions that are already running in their own
timeout/retry thread to avoid spawning another one.

Splitting the functions beyond that will make it more challenging to revisit the
decorator implementation to do something smarter. At the moment, we can revise
the decorator, remove the _Impl functions, and change all calls to _Impl
functions to whatever smarter thing we come up with.

[perezju, 2014/10/17 11:17:09]

Done.

+      output = self._RunShellCommandImpl(cmd, cwd=cwd, env=env, as_root=as_root)
+    except device_errors.AdbShellCommandFailedError as e:
+      if check_return:
+        raise
+      else:
+        output = e.output
+    if want_lines:
+      output = output.splitlines()
+    return output
 
-  def _RunShellCommandImpl(self, cmd, check_return=False, as_root=False,
-                           cwd=None, env=None, timeout=None):
-    # TODO(jbudorick): Remove the timeout parameter once this is no longer
-    # backed by AndroidCommands.
-    if isinstance(cmd, list):
-      cmd = ' '.join(cmd)
+  def _RunShellCommandImpl(self, cmd, cwd=None, env=None, as_root=False):

[jbudorick, 2014/10/17 09:04:53]

You'll have to restore the options you split between the two functions.

+    '''Implementation of RunShellCommand.
+
+    Note that this function already implements the proposed new behaviour of
+    RunShellCommand (i.e, check_return=True, want_lines=False) and is available
+    to be used by other DeviceUtils' methods.
+    '''
+    def env_quote(key, value):
+      if not _VALID_SHELL_VARIABLE.match(key):
+        raise KeyError('Invalid shell variable name %r' % key)
+      # using cmd_helper.dquote to quote but allow interpolation of shell vars
+      return '%s=%s' % (key, cmd_helper.dquote(value))

[jbudorick, 2014/10/17 09:04:53]

Maybe the dquote functionality should just live inside env_quote? I don't
anticipate us using it anywhere else...

+
+    if not isinstance(cmd, basestring):
+      cmd = ' '.join(cmd_helper.quote(s) for s in cmd)

[jbudorick, 2014/10/17 09:04:53]

I really don't like the 'quote = pipes.quote' in practice.

     if as_root and not self._HasRootImpl():
       cmd = 'su -c %s' % cmd
-    if env:
-      cmd = '%s %s' % (
-          ' '.join('%s=%s' % (k, v) for k, v in env.iteritems()), cmd)
-    if cwd:
-      cmd = 'cd %s && %s' % (cwd, cmd)
-    if check_return:
-      code, output = self.old_interface.GetShellCommandStatusAndOutput(
-          cmd, timeout_time=timeout)
-      if int(code) != 0:
-        raise device_errors.AdbCommandFailedError(
-            cmd.split(), 'Nonzero exit code (%d)' % code, device=str(self))
-    else:
-      output = self.old_interface.RunShellCommand(cmd, timeout_time=timeout)
-    return output
+    if env is not None:

[jbudorick, 2014/10/17 09:04:53]

As below, this change only matters if env is empty, and I don't think we want to
do the below in that case.

[perezju, 2014/10/17 11:17:09]

Acknowledged.

+      env = ' '.join(env_quote(k, v) for k, v in env.iteritems())
+      cmd = '%s %s' % (env, cmd)
+    if cwd is not None:

[jbudorick, 2014/10/17 09:04:53]

The only time when this change matters is if cwd is the empty string, and I
don't think we want to add the cwd logic in that case.

+      cmd = 'cd %s && %s' % (cmd_helper.quote(cwd), cmd)
+    # TODO(perezju) still need to make sure that we call a version of adb.Shell
+    # without a timeout-and-retries wrapper.
+    return self.adb.Shell(cmd, expect_rc=0)
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def KillAll(self, process_name, signum=9, as_root=False, blocking=False,
               timeout=None, retries=None):
     """Kill all processes with the given name on the device.
 
     Args:
       process_name: A string containing the name of the process to kill.
       signum: An integer containing the signal number to send to kill. Defaults
               to 9 (SIGKILL).
       as_root: A boolean indicating whether the kill should be executed with
                root privileges.
       blocking: A boolean indicating whether we should wait until all processes
                 with the given |process_name| are dead.
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
       CommandFailedError if no process was killed.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     pids = self._GetPidsImpl(process_name)
     if not pids:
       raise device_errors.CommandFailedError(
           'No process "%s"' % process_name, device=str(self))
 
-    cmd = 'kill -%d %s' % (signum, ' '.join(pids.values()))
+    cmd = ['kill', '-%d' % signum] + pids.values()
     self._RunShellCommandImpl(cmd, as_root=as_root)
 
     if blocking:
       wait_period = 0.1
       while self._GetPidsImpl(process_name):
         time.sleep(wait_period)
 
     return len(pids)
 
   @decorators.WithTimeoutAndRetriesFromInstance()
@@ skipping 131 matching lines @@
 
     Raises:
       CommandFailedError on failure.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
 
     files = []
     for h, d in host_device_tuples:
       if os.path.isdir(h):
-        self._RunShellCommandImpl(['mkdir', '-p', '"%s"' % d],
-                                  check_return=True)
+        self._RunShellCommandImpl(['mkdir', '-p', d])

[jbudorick, 2014/10/17 09:04:53]

following from my comments on the _Impl split, restore check_return=True here.

[perezju, 2014/10/17 11:17:09]

All Done.

       files += self._GetChangedFilesImpl(h, d)
 
     if not files:
       return
 
     size = sum(host_utils.GetRecursiveDiskUsage(h) for h, _ in files)
     file_count = len(files)
     dir_size = sum(host_utils.GetRecursiveDiskUsage(h)
                    for h, _ in host_device_tuples)
     dir_file_count = 0
@@ skipping 19 matching lines @@
     else:
       self._PushChangedFilesZipped(files)
       self._RunShellCommandImpl(
           ['chmod', '-R', '777'] + [d for _, d in host_device_tuples],
           as_root=True)
 
   def _GetChangedFilesImpl(self, host_path, device_path):
     real_host_path = os.path.realpath(host_path)
     try:
       real_device_path = self._RunShellCommandImpl(
-          ['realpath', device_path], check_return=True)
-      real_device_path = real_device_path[0]
+          ['realpath', device_path]).rstrip()

[jbudorick, 2014/10/17 09:04:53]

and here.

     except device_errors.CommandFailedError:
       return [(host_path, device_path)]
 
     # TODO(jbudorick): Move the md5 logic up into DeviceUtils or base
     # this function on mtime.
     # pylint: disable=W0212
     host_hash_tuples, device_hash_tuples = self.old_interface._RunMd5Sum(
         real_host_path, real_device_path)
     # pylint: enable=W0212
 
@@ skipping 71 matching lines @@
           target=DeviceUtils._CreateDeviceZip,
           args=(zip_file.name, files))
       zip_proc.start()
       zip_proc.join()
 
       zip_on_device = '%s/tmp.zip' % self._GetExternalStoragePathImpl()
       try:
         self.adb.Push(zip_file.name, zip_on_device)
         self._RunShellCommandImpl(
             ['unzip', zip_on_device],
-            as_root=True, check_return=True,
+            as_root=True,

[jbudorick, 2014/10/17 09:04:53]

and here.

             env={'PATH': '$PATH:%s' % install_commands.BIN_DIR})
       finally:
         if zip_proc.is_alive():
           zip_proc.terminate()
         if self._IsOnlineImpl():
           self._RunShellCommandImpl(['rm', zip_on_device])
 
   @staticmethod
   def _CreateDeviceZip(zip_path, host_device_tuples):
     with zipfile.ZipFile(zip_path, 'w') as zip_file:
@@ skipping 121 matching lines @@
       as_root: A boolean indicating whether the write should be executed with
                root privileges.
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
       CommandFailedError if the file could not be written on the device.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
-    self._RunShellCommandImpl('echo {1} > {0}'.format(device_path,
-        pipes.quote(text)), check_return=True, as_root=as_root)
+    cmd = 'echo %s > %s' % (cmd_helper.quote(text),
+                            cmd_helper.quote(device_path))
+    self._RunShellCommandImpl(cmd, as_root=as_root)

[jbudorick, 2014/10/17 09:04:53]

and here.

 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def Ls(self, device_path, timeout=None, retries=None):
     """Lists the contents of a directory on the device.
 
     Args:
       device_path: A string containing the path of the directory on the device
                    to list.
       timeout: timeout in seconds
       retries: number of retries
@@ skipping 96 matching lines @@
       provided |process_name|.
 
     Raises:
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     return self._GetPidsImpl(process_name)
 
   def _GetPidsImpl(self, process_name):
     procs_pids = {}
-    for line in self._RunShellCommandImpl('ps'):
+    for line in self._RunShellCommandImpl('ps').splitlines():
       try:
         ps_data = line.split()
         if process_name in ps_data[-1]:
           procs_pids[ps_data[-1]] = ps_data[1]
       except IndexError:
         pass
     return procs_pids
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def TakeScreenshot(self, host_path=None, timeout=None, retries=None):
@@ skipping 77 matching lines @@
     Returns:
       A Parallelizer operating over |devices|.
     """
     if not devices or len(devices) == 0:
       devices = pylib.android_commands.GetAttachedDevices()
     parallelizer_type = (parallelizer.Parallelizer if async
                          else parallelizer.SyncParallelizer)
     return parallelizer_type([
         d if isinstance(d, DeviceUtils) else DeviceUtils(d)
         for d in devices])