Allow the forwarder to recover if the device_forwarder crashes or hangs

build/android/pylib/forwarder.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # pylint: disable=W0212
 
 import fcntl
 import logging
 import os
 import psutil
 
 from pylib import cmd_helper
 from pylib import constants
 from pylib import valgrind_tools
-from pylib.device import device_errors
 
 # TODO(jbudorick) Remove once telemetry gets switched over.
 import pylib.android_commands
 import pylib.device.device_utils
 
 
 def _GetProcessStartTime(pid):
   return psutil.Process(pid).create_time
 
 
@@ skipping 58 matching lines @@
     if isinstance(device, pylib.android_commands.AndroidCommands):
       device = pylib.device.device_utils.DeviceUtils(device)
     if not tool:
       tool = valgrind_tools.CreateTool(None, device)
     with _FileLock(Forwarder._LOCK_PATH):
       instance = Forwarder._GetInstanceLocked(tool)
       instance._InitDeviceLocked(device, tool)
 
       device_serial = str(device)
       redirection_commands = [
-          ['--serial-id=' + device_serial, '--map', str(device),
-           str(host)] for device, host in port_pairs]
+          ['--serial-id=' + device_serial, '--map', str(device_port),
+           str(host_port)] for device_port, host_port in port_pairs]
       logging.info('Forwarding using commands: %s', redirection_commands)
 
       for redirection_command in redirection_commands:
         try:
           (exit_code, output) = cmd_helper.GetCmdStatusAndOutput(
               [instance._host_forwarder_path] + redirection_command)
         except OSError as e:
           if e.errno == 2:
             raise Exception('Unable to start host forwarder. Make sure you have'
                             ' built host_forwarder.')
           else: raise
         if exit_code != 0:
+          Forwarder._KillDeviceLocked(device, tool)
           raise Exception('%s exited with %d:\n%s' % (
               instance._host_forwarder_path, exit_code, '\n'.join(output)))
         tokens = output.split(':')
         if len(tokens) != 2:
-          raise Exception(('Unexpected host forwarder output "%s", ' +
-                          'expected "device_port:host_port"') % output)
+          raise Exception('Unexpected host forwarder output "%s", '
+                          'expected "device_port:host_port"' % output)
         device_port = int(tokens[0])
         host_port = int(tokens[1])
         serial_with_port = (device_serial, device_port)
         instance._device_to_host_port_map[serial_with_port] = host_port
         instance._host_to_device_port_map[host_port] = serial_with_port
         logging.info('Forwarding device port: %d to host port: %d.',
                      device_port, host_port)
 
   @staticmethod
   def UnmapDevicePort(device_port, device):
@@ skipping 27 matching lines @@
       if adb_serial not in Forwarder._instance._initialized_devices:
         return
       port_map = Forwarder._GetInstanceLocked(
           None)._device_to_host_port_map
       for (device_serial, device_port) in port_map.keys():
         if adb_serial == device_serial:
           Forwarder._UnmapDevicePortLocked(device_port, device)
       # There are no more ports mapped, kill the device_forwarder.
       tool = valgrind_tools.CreateTool(None, device)
       Forwarder._KillDeviceLocked(device, tool)
-      Forwarder._instance._initialized_devices.remove(adb_serial)
-
 
   @staticmethod
   def DevicePortForHostPort(host_port):
     """Returns the device port that corresponds to a given host port."""
     with _FileLock(Forwarder._LOCK_PATH):
       (_device_serial, device_port) = Forwarder._GetInstanceLocked(
           None)._host_to_device_port_map.get(host_port)
       return device_port
 
   @staticmethod
@@ skipping 148 matching lines @@
     """Kills the forwarder process running on the device.
 
     Note that the global lock must be acquired before calling this method.
 
     Args:
       device: Instance of DeviceUtils for talking to the device.
       tool: Wrapper tool (e.g. valgrind) that can be used to execute the device
             forwarder (see valgrind_tools.py).
     """
     logging.info('Killing device_forwarder.')
+    Forwarder._instance._initialized_devices.discard(str(device))
     if not device.FileExists(Forwarder._DEVICE_FORWARDER_PATH):
       return
 
     cmd = '%s %s --kill-server' % (tool.GetUtilWrapper(),
                                    Forwarder._DEVICE_FORWARDER_PATH)
     device.old_interface.GetAndroidToolStatusAndOutput(
         cmd, lib_path=Forwarder._DEVICE_FORWARDER_FOLDER)
-
-    # TODO(pliard): Remove the following call to KillAllBlocking() when we are
-    # sure that the old version of device_forwarder (not supporting
-    # 'kill-server') is not running on the bots anymore.
-    timeout_sec = 5
-    try:
-      device.KillAll('device_forwarder', blocking=True, timeout=timeout_sec)
-    except device_errors.CommandFailedError:
-      if device.GetPids('device_forwarder'):
-        raise