Reland of Allow RunShellCommand to work even with very large commands

build/android/pylib/device/device_utils.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Provides a variety of device interactions based on adb.
 
 Eventually, this will be based on adb_wrapper.
 """
 # pylint: disable=W0613
 
@@ skipping 56 matching lines @@
   """Restarts the adb server.
 
   Raises:
     CommandFailedError if we fail to kill or restart the server.
   """
   pylib.android_commands.AndroidCommands().RestartAdbServer()
 
 
 class DeviceUtils(object):
 
+  _MAX_ADB_COMMAND_LENGTH = 512
   _VALID_SHELL_VARIABLE = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
 
   def __init__(self, device, default_timeout=_DEFAULT_TIMEOUT,
                default_retries=_DEFAULT_RETRIES):
     """DeviceUtils constructor.
 
     Args:
       device: Either a device serial, an existing AdbWrapper instance, or an
               an existing AndroidCommands instance.
       default_timeout: An integer containing the default number of seconds to
@@ skipping 341 matching lines @@
         more lines.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
     def env_quote(key, value):
       if not DeviceUtils._VALID_SHELL_VARIABLE.match(key):
         raise KeyError('Invalid shell variable name %r' % key)
       # using double quotes here to allow interpolation of shell variables
       return '%s=%s' % (key, cmd_helper.DoubleQuote(value))
 
+    def do_run(cmd):
+      try:
+        return self.adb.Shell(cmd)
+      except device_errors.AdbCommandFailedError as exc:
+        if check_return:
+          raise
+        else:
+          return exc.output
+
     if not isinstance(cmd, basestring):
       cmd = ' '.join(cmd_helper.SingleQuote(s) for s in cmd)
     if env:
       env = ' '.join(env_quote(k, v) for k, v in env.iteritems())
       cmd = '%s %s' % (env, cmd)
     if cwd:
       cmd = 'cd %s && %s' % (cmd_helper.SingleQuote(cwd), cmd)
     if as_root and self.NeedsSU():
       # "su -c sh -c" allows using shell features in |cmd|
       cmd = 'su -c sh -c %s' % cmd_helper.SingleQuote(cmd)
     if timeout is None:
       timeout = self._default_timeout
 
-    try:
-      output = self.adb.Shell(cmd)
-    except device_errors.AdbCommandFailedError as e:
-      if check_return:
-        raise
-      else:
-        output = e.output
+    if len(cmd) < self._MAX_ADB_COMMAND_LENGTH:
+      output = do_run(cmd)
+    else:
+      with device_temp_file.DeviceTempFile(self.adb, suffix='.sh') as script:
+        self._WriteFileWithPush(script.name, cmd)
+        logging.info('Large shell command will be run from file: %s ...',
+                     cmd[:100])

[perezju, 2014/12/15 15:48:59]

I've noted that running large shell commands in this way (like the ones already
done by hand for PushChanged files) annoyingly hide the actual command being run
under "adb shell 'sh -c /path/to/temp123456.sh'".

So I thought it's best to log at least a portion of the command, so that we can
get an idea of what's going on when reading the logs.

[jbudorick, 2014/12/15 23:18:28]

sgtm

+        output = do_run('sh %s' % script.name_quoted)
 
     output = output.splitlines()
     if single_line:
       if not output:
         return ''
       elif len(output) == 1:
         return output[0]
       else:
         msg = 'one line of output was expected, but got: %s'
         raise device_errors.CommandFailedError(msg % output, str(self))
@@ skipping 425 matching lines @@
     # TODO(jbudorick) Evaluate whether we want to return a list of lines after
     # the implementation switch, and if file not found should raise exception.
     if as_root:
       if not self.old_interface.CanAccessProtectedFileContents():
         raise device_errors.CommandFailedError(
           'Cannot read from %s with root privileges.' % device_path)
       return self.old_interface.GetProtectedFileContents(device_path)
     else:
       return self.old_interface.GetFileContents(device_path)
 
+  def _WriteFileWithPush(self, device_path, contents):
+    with tempfile.NamedTemporaryFile() as host_temp:
+      host_temp.write(contents)
+      host_temp.flush()
+      self.adb.Push(host_temp.name, device_path)
+
   @decorators.WithTimeoutAndRetriesFromInstance()
   def WriteFile(self, device_path, contents, as_root=False, force_push=False,
                 timeout=None, retries=None):
     """Writes |contents| to a file on the device.
 
     Args:
       device_path: A string containing the absolute path to the file to write
           on the device.
       contents: A string containing the data to write to the device.
       as_root: A boolean indicating whether the write should be executed with
           root privileges (if available).
       force_push: A boolean indicating whether to force the operation to be
           performed by pushing a file to the device. The default is, when the
           contents are short, to pass the contents using a shell script instead.
       timeout: timeout in seconds
       retries: number of retries
 
     Raises:
       CommandFailedError if the file could not be written on the device.
       CommandTimeoutError on timeout.
       DeviceUnreachableError on missing device.
     """
-    if len(contents) < 512 and not force_push:
+    if not force_push and len(contents) < self._MAX_ADB_COMMAND_LENGTH:
+      # If the contents are small, for efficieny we write the contents with
+      # a shell command rather than pushing a file.
       cmd = 'echo -n %s > %s' % (cmd_helper.SingleQuote(contents),
                                  cmd_helper.SingleQuote(device_path))
       self.RunShellCommand(cmd, as_root=as_root, check_return=True)
+    elif as_root and self.NeedsSU():
+      # Adb does not allow to "push with su", so we first push to a temp file
+      # on a safe location, and then copy it to the desired location with su.
+      with device_temp_file.DeviceTempFile(self.adb) as device_temp:
+        self._WriteFileWithPush(device_temp.name, contents)
+        # Here we need 'cp' rather than 'mv' because the temp and
+        # destination files might be on different file systems (e.g.
+        # on internal storage and an external sd card).
+        self.RunShellCommand(['cp', device_temp.name, device_path],
+                             as_root=True, check_return=True)
     else:
-      with tempfile.NamedTemporaryFile() as host_temp:
-        host_temp.write(contents)
-        host_temp.flush()
-        if as_root and self.NeedsSU():
-          with device_temp_file.DeviceTempFile(self.adb) as device_temp:
-            self.adb.Push(host_temp.name, device_temp.name)
-            # Here we need 'cp' rather than 'mv' because the temp and
-            # destination files might be on different file systems (e.g.
-            # on internal storage and an external sd card)
-            self.RunShellCommand(['cp', device_temp.name, device_path],
-                                 as_root=True, check_return=True)
-        else:
-          self.adb.Push(host_temp.name, device_path)
+      # If root is not needed, we can push directly to the desired location.
+      self._WriteFileWithPush(device_path, contents)
 
   @decorators.WithTimeoutAndRetriesFromInstance()
   def Ls(self, device_path, timeout=None, retries=None):
     """Lists the contents of a directory on the device.
 
     Args:
       device_path: A string containing the path of the directory on the device
                    to list.
       timeout: timeout in seconds
       retries: number of retries
@@ skipping 316 matching lines @@
     Returns:
       A Parallelizer operating over |devices|.
     """
     if not devices:
       devices = adb_wrapper.AdbWrapper.GetDevices()
     devices = [d if isinstance(d, cls) else cls(d) for d in devices]
     if async:
       return parallelizer.Parallelizer(devices)
     else:
       return parallelizer.SyncParallelizer(devices)