| OLD | NEW | 
|---|
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be | 
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. | 
| 4 | 4 | 
| 5 """Defines the task controller library.""" | 5 """Defines the client library.""" | 
| 6 | 6 | 
| 7 import argparse | 7 import argparse | 
| 8 import datetime | 8 import datetime | 
| 9 import logging | 9 import logging | 
| 10 import os | 10 import os | 
| 11 import socket | 11 import socket | 
| 12 import subprocess | 12 import subprocess | 
| 13 import sys | 13 import sys | 
| 14 import tempfile | 14 import tempfile | 
| 15 import threading | 15 import threading | 
| 16 import xmlrpclib | 16 import xmlrpclib | 
| 17 | 17 | 
| 18 #pylint: disable=relative-import | 18 #pylint: disable=relative-import | 
| 19 import common_lib | 19 import common_lib | 
| 20 | 20 | 
| 21 ISOLATE_PY = os.path.join(common_lib.SWARMING_DIR, 'isolate.py') | 21 ISOLATE_PY = os.path.join(common_lib.SWARMING_DIR, 'isolate.py') | 
| 22 SWARMING_PY = os.path.join(common_lib.SWARMING_DIR, 'swarming.py') | 22 SWARMING_PY = os.path.join(common_lib.SWARMING_DIR, 'swarming.py') | 
| 23 | 23 | 
| 24 | 24 | 
| 25 class Error(Exception): | 25 class Error(Exception): | 
| 26   pass | 26   pass | 
| 27 | 27 | 
| 28 | 28 | 
| 29 class ConnectionTimeoutError(Error): | 29 class ConnectionTimeoutError(Error): | 
| 30   pass | 30   pass | 
| 31 | 31 | 
| 32 | 32 | 
| 33 class TaskController(object): | 33 class ClientController(object): | 
| 34   """Provisions, configures, and controls a task machine. | 34   """Creates, configures, and controls a client machine.""" | 
| 35 | 35 | 
| 36   This class is an abstraction of a physical task machine. It provides an | 36   _client_count = 0 | 
| 37   end to end API for controlling a task machine. Operations on the task machine | 37   _controllers = [] | 
| 38   are performed using the instance's "rpc" property. A simple end to end |  | 
| 39   scenario is as follows: |  | 
| 40 |  | 
| 41   task = TaskController(...) |  | 
| 42   task.Create() |  | 
| 43   task.WaitForConnection() |  | 
| 44   proc = task.rpc.subprocess.Popen(['ls']) |  | 
| 45   print task.rpc.subprocess.GetStdout(proc) |  | 
| 46   task.Release() |  | 
| 47   """ |  | 
| 48 |  | 
| 49   _task_count = 0 |  | 
| 50   _tasks = [] |  | 
| 51 | 38 | 
| 52   def __init__(self, isolate_file, config_vars, dimensions, priority=100, | 39   def __init__(self, isolate_file, config_vars, dimensions, priority=100, | 
| 53                idle_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS, | 40                idle_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS, | 
| 54                connection_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS, | 41                connection_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS, | 
| 55                verbosity='ERROR', name=None): | 42                verbosity='ERROR', name=None): | 
| 56     assert isinstance(config_vars, dict) | 43     assert isinstance(config_vars, dict) | 
| 57     assert isinstance(dimensions, dict) | 44     assert isinstance(dimensions, dict) | 
| 58     type(self)._tasks.append(self) | 45     type(self)._controllers.append(self) | 
| 59     type(self)._task_count += 1 | 46     type(self)._client_count += 1 | 
| 60     self.verbosity = verbosity | 47     self.verbosity = verbosity | 
| 61     self._name = name or 'Task%d' % type(self)._task_count | 48     self._name = name or 'Client%d' % type(self)._client_count | 
| 62     self._priority = priority | 49     self._priority = priority | 
| 63     self._isolate_file = isolate_file | 50     self._isolate_file = isolate_file | 
| 64     self._isolated_file = isolate_file + 'd' | 51     self._isolated_file = isolate_file + 'd' | 
| 65     self._idle_timeout_secs = idle_timeout_secs | 52     self._idle_timeout_secs = idle_timeout_secs | 
| 66     self._config_vars = config_vars | 53     self._config_vars = config_vars | 
| 67     self._dimensions = dimensions | 54     self._dimensions = dimensions | 
| 68     self._connect_event = threading.Event() | 55     self._connect_event = threading.Event() | 
| 69     self._connected = False | 56     self._connected = False | 
| 70     self._ip_address = None | 57     self._ip_address = None | 
| 71     self._otp = self._CreateOTP() | 58     self._otp = self._CreateOTP() | 
| 72     self._rpc = None | 59     self._rpc = None | 
| 73 | 60 | 
| 74     parser = argparse.ArgumentParser() | 61     parser = argparse.ArgumentParser() | 
| 75     parser.add_argument('--isolate-server') | 62     parser.add_argument('--isolate-server') | 
| 76     parser.add_argument('--swarming-server') | 63     parser.add_argument('--swarming-server') | 
| 77     parser.add_argument('--task-connection-timeout-secs', | 64     parser.add_argument('--client-connection-timeout-secs', | 
| 78                         default=common_lib.DEFAULT_TIMEOUT_SECS) | 65                         default=common_lib.DEFAULT_TIMEOUT_SECS) | 
| 79     args, _ = parser.parse_known_args() | 66     args, _ = parser.parse_known_args() | 
| 80 | 67 | 
| 81     self._isolate_server = args.isolate_server | 68     self._isolate_server = args.isolate_server | 
| 82     self._swarming_server = args.swarming_server | 69     self._swarming_server = args.swarming_server | 
| 83     self._connection_timeout_secs = (connection_timeout_secs or | 70     self._connection_timeout_secs = (connection_timeout_secs or | 
| 84                                     args.task_connection_timeout_secs) | 71                                     args.client_connection_timeout_secs) | 
| 85 | 72 | 
| 86   @property | 73   @property | 
| 87   def name(self): | 74   def name(self): | 
| 88     return self._name | 75     return self._name | 
| 89 | 76 | 
| 90   @property | 77   @property | 
| 91   def otp(self): | 78   def otp(self): | 
| 92     return self._otp | 79     return self._otp | 
| 93 | 80 | 
| 94   @property | 81   @property | 
| (...skipping 18 matching lines...) Expand all  Loading... | 
| 113 | 100 | 
| 114     Either a string ('INFO', 'DEBUG', etc) or a logging level (logging.INFO, | 101     Either a string ('INFO', 'DEBUG', etc) or a logging level (logging.INFO, | 
| 115     logging.DEBUG, etc) is allowed. | 102     logging.DEBUG, etc) is allowed. | 
| 116     """ | 103     """ | 
| 117     assert isinstance(level, (str, int)) | 104     assert isinstance(level, (str, int)) | 
| 118     if isinstance(level, int): | 105     if isinstance(level, int): | 
| 119       level = logging.getLevelName(level) | 106       level = logging.getLevelName(level) | 
| 120     self._verbosity = level  #pylint: disable=attribute-defined-outside-init | 107     self._verbosity = level  #pylint: disable=attribute-defined-outside-init | 
| 121 | 108 | 
| 122   @classmethod | 109   @classmethod | 
| 123   def ReleaseAllTasks(cls): | 110   def ReleaseAllControllers(cls): | 
| 124     for task in cls._tasks: | 111     for controller in cls._controllers: | 
| 125       task.Release() | 112       controller.Release() | 
| 126 | 113 | 
| 127   def _CreateOTP(self): | 114   def _CreateOTP(self): | 
| 128     """Creates the OTP.""" | 115     """Creates the OTP.""" | 
| 129     controller_name = socket.gethostname() | 116     host_name = socket.gethostname() | 
| 130     test_name = os.path.basename(sys.argv[0]) | 117     test_name = os.path.basename(sys.argv[0]) | 
| 131     creation_time = datetime.datetime.utcnow() | 118     creation_time = datetime.datetime.utcnow() | 
| 132     otp = 'task:%s controller:%s test:%s creation:%s' % ( | 119     otp = 'client:%s-host:%s-test:%s-creation:%s' % ( | 
| 133         self._name, controller_name, test_name, creation_time) | 120         self._name, host_name, test_name, creation_time) | 
| 134     return otp | 121     return otp | 
| 135 | 122 | 
| 136   def Create(self): | 123   def Create(self): | 
| 137     """Creates the task machine.""" | 124     """Creates the client machine.""" | 
| 138     logging.info('Creating %s', self.name) | 125     logging.info('Creating %s', self.name) | 
| 139     self._connect_event.clear() | 126     self._connect_event.clear() | 
| 140     self._ExecuteIsolate() | 127     self._ExecuteIsolate() | 
| 141     self._ExecuteSwarming() | 128     self._ExecuteSwarming() | 
| 142 | 129 | 
| 143   def WaitForConnection(self): | 130   def WaitForConnection(self): | 
| 144     """Waits for the task machine to connect. | 131     """Waits for the client machine to connect. | 
| 145 | 132 | 
| 146     Raises: | 133     Raises: | 
| 147       ConnectionTimeoutError if the task doesn't connect in time. | 134       ConnectionTimeoutError if the client doesn't connect in time. | 
| 148     """ | 135     """ | 
| 149     logging.info('Waiting for %s to connect with a timeout of %d seconds', | 136     logging.info('Waiting for %s to connect with a timeout of %d seconds', | 
| 150                  self._name, self._connection_timeout_secs) | 137                  self._name, self._connection_timeout_secs) | 
| 151     self._connect_event.wait(self._connection_timeout_secs) | 138     self._connect_event.wait(self._connection_timeout_secs) | 
| 152     if not self._connect_event.is_set(): | 139     if not self._connect_event.is_set(): | 
| 153       raise ConnectionTimeoutError('%s failed to connect' % self.name) | 140       raise ConnectionTimeoutError('%s failed to connect' % self.name) | 
| 154 | 141 | 
| 155   def Release(self): | 142   def Release(self): | 
| 156     """Quits the task's RPC server so it can release the machine.""" | 143     """Quits the client's RPC server so it can release the machine.""" | 
| 157     if self._rpc is not None and self._connected: | 144     if self._rpc is not None and self._connected: | 
| 158       logging.info('Releasing %s', self._name) | 145       logging.info('Releasing %s', self._name) | 
| 159       try: | 146       try: | 
| 160         self._rpc.Quit() | 147         self._rpc.Quit() | 
| 161       except (socket.error, xmlrpclib.Fault): | 148       except (socket.error, xmlrpclib.Fault): | 
| 162         logging.error('Unable to connect to %s to call Quit', self.name) | 149         logging.error('Unable to connect to %s to call Quit', self.name) | 
| 163       self._rpc = None | 150       self._rpc = None | 
| 164       self._connected = False | 151       self._connected = False | 
| 165 | 152 | 
| 166   def _ExecuteIsolate(self): | 153   def _ExecuteIsolate(self): | 
| (...skipping 25 matching lines...) Expand all  Loading... | 
| 192 | 179 | 
| 193     if self._isolate_server: | 180     if self._isolate_server: | 
| 194       cmd.extend(['--isolate-server', self._isolate_server]) | 181       cmd.extend(['--isolate-server', self._isolate_server]) | 
| 195     if self._swarming_server: | 182     if self._swarming_server: | 
| 196       cmd.extend(['--swarming', self._swarming_server]) | 183       cmd.extend(['--swarming', self._swarming_server]) | 
| 197     for key, value in self._dimensions.iteritems(): | 184     for key, value in self._dimensions.iteritems(): | 
| 198       cmd.extend(['--dimension', key, value]) | 185       cmd.extend(['--dimension', key, value]) | 
| 199 | 186 | 
| 200     cmd.extend([ | 187     cmd.extend([ | 
| 201         '--', | 188         '--', | 
| 202         '--controller', common_lib.MY_IP, | 189         '--host', common_lib.MY_IP, | 
| 203         '--otp', self._otp, | 190         '--otp', self._otp, | 
| 204         '--verbosity', self._verbosity, | 191         '--verbosity', self._verbosity, | 
| 205         '--idle-timeout', str(self._idle_timeout_secs), | 192         '--idle-timeout', str(self._idle_timeout_secs), | 
| 206         ]) | 193         ]) | 
| 207 | 194 | 
| 208     self._ExecuteProcess(cmd) | 195     self._ExecuteProcess(cmd) | 
| 209 | 196 | 
| 210   def _ExecuteProcess(self, cmd): | 197   def _ExecuteProcess(self, cmd): | 
| 211     """Executes a process, waits for it to complete, and checks for success.""" | 198     """Executes a process, waits for it to complete, and checks for success.""" | 
| 212     logging.debug('Running %s', ' '.join(cmd)) | 199     logging.debug('Running %s', ' '.join(cmd)) | 
| 213     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | 200     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | 
| 214     _, stderr = p.communicate() | 201     _, stderr = p.communicate() | 
| 215     if p.returncode != 0: | 202     if p.returncode != 0: | 
| 216       raise Error(stderr) | 203       raise Error(stderr) | 
| 217 | 204 | 
| 218   def OnConnect(self, ip_address): | 205   def OnConnect(self, ip_address): | 
| 219     """Receives task ip address on connection.""" | 206     """Receives client ip address on connection.""" | 
| 220     self._ip_address = ip_address | 207     self._ip_address = ip_address | 
| 221     self._connected = True | 208     self._connected = True | 
| 222     self._rpc = common_lib.ConnectToServer(self._ip_address) | 209     self._rpc = common_lib.ConnectToServer(self._ip_address) | 
| 223     logging.info('%s connected from %s', self._name, ip_address) | 210     logging.info('%s connected from %s', self._name, ip_address) | 
| 224     self._connect_event.set() | 211     self._connect_event.set() | 
| OLD | NEW | 
|---|