Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(139)

Side by Side Diff: testing/legion/client_lib.py

Issue 951673002: Revert "Pull chromium at 2c3ffb2355a27c32f45e508ef861416b820c823b" (Closed) Base URL: git@github.com:domokit/mojo.git@master
Patch Set: Created 5 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « testing/legion/client_controller.py ('k') | testing/legion/client_rpc_methods.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2015 The Chromium Authors. All rights reserved. 1 # Copyright 2015 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Defines the task controller library.""" 5 """Defines the client library."""
6 6
7 import argparse 7 import argparse
8 import datetime 8 import datetime
9 import logging 9 import logging
10 import os 10 import os
11 import socket 11 import socket
12 import subprocess 12 import subprocess
13 import sys 13 import sys
14 import tempfile 14 import tempfile
15 import threading 15 import threading
16 import xmlrpclib 16 import xmlrpclib
17 17
18 #pylint: disable=relative-import 18 #pylint: disable=relative-import
19 import common_lib 19 import common_lib
20 20
21 ISOLATE_PY = os.path.join(common_lib.SWARMING_DIR, 'isolate.py') 21 ISOLATE_PY = os.path.join(common_lib.SWARMING_DIR, 'isolate.py')
22 SWARMING_PY = os.path.join(common_lib.SWARMING_DIR, 'swarming.py') 22 SWARMING_PY = os.path.join(common_lib.SWARMING_DIR, 'swarming.py')
23 23
24 24
25 class Error(Exception): 25 class Error(Exception):
26 pass 26 pass
27 27
28 28
29 class ConnectionTimeoutError(Error): 29 class ConnectionTimeoutError(Error):
30 pass 30 pass
31 31
32 32
33 class TaskController(object): 33 class ClientController(object):
34 """Provisions, configures, and controls a task machine. 34 """Creates, configures, and controls a client machine."""
35 35
36 This class is an abstraction of a physical task machine. It provides an 36 _client_count = 0
37 end to end API for controlling a task machine. Operations on the task machine 37 _controllers = []
38 are performed using the instance's "rpc" property. A simple end to end
39 scenario is as follows:
40
41 task = TaskController(...)
42 task.Create()
43 task.WaitForConnection()
44 proc = task.rpc.subprocess.Popen(['ls'])
45 print task.rpc.subprocess.GetStdout(proc)
46 task.Release()
47 """
48
49 _task_count = 0
50 _tasks = []
51 38
52 def __init__(self, isolate_file, config_vars, dimensions, priority=100, 39 def __init__(self, isolate_file, config_vars, dimensions, priority=100,
53 idle_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS, 40 idle_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS,
54 connection_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS, 41 connection_timeout_secs=common_lib.DEFAULT_TIMEOUT_SECS,
55 verbosity='ERROR', name=None): 42 verbosity='ERROR', name=None):
56 assert isinstance(config_vars, dict) 43 assert isinstance(config_vars, dict)
57 assert isinstance(dimensions, dict) 44 assert isinstance(dimensions, dict)
58 type(self)._tasks.append(self) 45 type(self)._controllers.append(self)
59 type(self)._task_count += 1 46 type(self)._client_count += 1
60 self.verbosity = verbosity 47 self.verbosity = verbosity
61 self._name = name or 'Task%d' % type(self)._task_count 48 self._name = name or 'Client%d' % type(self)._client_count
62 self._priority = priority 49 self._priority = priority
63 self._isolate_file = isolate_file 50 self._isolate_file = isolate_file
64 self._isolated_file = isolate_file + 'd' 51 self._isolated_file = isolate_file + 'd'
65 self._idle_timeout_secs = idle_timeout_secs 52 self._idle_timeout_secs = idle_timeout_secs
66 self._config_vars = config_vars 53 self._config_vars = config_vars
67 self._dimensions = dimensions 54 self._dimensions = dimensions
68 self._connect_event = threading.Event() 55 self._connect_event = threading.Event()
69 self._connected = False 56 self._connected = False
70 self._ip_address = None 57 self._ip_address = None
71 self._otp = self._CreateOTP() 58 self._otp = self._CreateOTP()
72 self._rpc = None 59 self._rpc = None
73 60
74 parser = argparse.ArgumentParser() 61 parser = argparse.ArgumentParser()
75 parser.add_argument('--isolate-server') 62 parser.add_argument('--isolate-server')
76 parser.add_argument('--swarming-server') 63 parser.add_argument('--swarming-server')
77 parser.add_argument('--task-connection-timeout-secs', 64 parser.add_argument('--client-connection-timeout-secs',
78 default=common_lib.DEFAULT_TIMEOUT_SECS) 65 default=common_lib.DEFAULT_TIMEOUT_SECS)
79 args, _ = parser.parse_known_args() 66 args, _ = parser.parse_known_args()
80 67
81 self._isolate_server = args.isolate_server 68 self._isolate_server = args.isolate_server
82 self._swarming_server = args.swarming_server 69 self._swarming_server = args.swarming_server
83 self._connection_timeout_secs = (connection_timeout_secs or 70 self._connection_timeout_secs = (connection_timeout_secs or
84 args.task_connection_timeout_secs) 71 args.client_connection_timeout_secs)
85 72
86 @property 73 @property
87 def name(self): 74 def name(self):
88 return self._name 75 return self._name
89 76
90 @property 77 @property
91 def otp(self): 78 def otp(self):
92 return self._otp 79 return self._otp
93 80
94 @property 81 @property
(...skipping 18 matching lines...) Expand all
113 100
114 Either a string ('INFO', 'DEBUG', etc) or a logging level (logging.INFO, 101 Either a string ('INFO', 'DEBUG', etc) or a logging level (logging.INFO,
115 logging.DEBUG, etc) is allowed. 102 logging.DEBUG, etc) is allowed.
116 """ 103 """
117 assert isinstance(level, (str, int)) 104 assert isinstance(level, (str, int))
118 if isinstance(level, int): 105 if isinstance(level, int):
119 level = logging.getLevelName(level) 106 level = logging.getLevelName(level)
120 self._verbosity = level #pylint: disable=attribute-defined-outside-init 107 self._verbosity = level #pylint: disable=attribute-defined-outside-init
121 108
122 @classmethod 109 @classmethod
123 def ReleaseAllTasks(cls): 110 def ReleaseAllControllers(cls):
124 for task in cls._tasks: 111 for controller in cls._controllers:
125 task.Release() 112 controller.Release()
126 113
127 def _CreateOTP(self): 114 def _CreateOTP(self):
128 """Creates the OTP.""" 115 """Creates the OTP."""
129 controller_name = socket.gethostname() 116 host_name = socket.gethostname()
130 test_name = os.path.basename(sys.argv[0]) 117 test_name = os.path.basename(sys.argv[0])
131 creation_time = datetime.datetime.utcnow() 118 creation_time = datetime.datetime.utcnow()
132 otp = 'task:%s controller:%s test:%s creation:%s' % ( 119 otp = 'client:%s-host:%s-test:%s-creation:%s' % (
133 self._name, controller_name, test_name, creation_time) 120 self._name, host_name, test_name, creation_time)
134 return otp 121 return otp
135 122
136 def Create(self): 123 def Create(self):
137 """Creates the task machine.""" 124 """Creates the client machine."""
138 logging.info('Creating %s', self.name) 125 logging.info('Creating %s', self.name)
139 self._connect_event.clear() 126 self._connect_event.clear()
140 self._ExecuteIsolate() 127 self._ExecuteIsolate()
141 self._ExecuteSwarming() 128 self._ExecuteSwarming()
142 129
143 def WaitForConnection(self): 130 def WaitForConnection(self):
144 """Waits for the task machine to connect. 131 """Waits for the client machine to connect.
145 132
146 Raises: 133 Raises:
147 ConnectionTimeoutError if the task doesn't connect in time. 134 ConnectionTimeoutError if the client doesn't connect in time.
148 """ 135 """
149 logging.info('Waiting for %s to connect with a timeout of %d seconds', 136 logging.info('Waiting for %s to connect with a timeout of %d seconds',
150 self._name, self._connection_timeout_secs) 137 self._name, self._connection_timeout_secs)
151 self._connect_event.wait(self._connection_timeout_secs) 138 self._connect_event.wait(self._connection_timeout_secs)
152 if not self._connect_event.is_set(): 139 if not self._connect_event.is_set():
153 raise ConnectionTimeoutError('%s failed to connect' % self.name) 140 raise ConnectionTimeoutError('%s failed to connect' % self.name)
154 141
155 def Release(self): 142 def Release(self):
156 """Quits the task's RPC server so it can release the machine.""" 143 """Quits the client's RPC server so it can release the machine."""
157 if self._rpc is not None and self._connected: 144 if self._rpc is not None and self._connected:
158 logging.info('Releasing %s', self._name) 145 logging.info('Releasing %s', self._name)
159 try: 146 try:
160 self._rpc.Quit() 147 self._rpc.Quit()
161 except (socket.error, xmlrpclib.Fault): 148 except (socket.error, xmlrpclib.Fault):
162 logging.error('Unable to connect to %s to call Quit', self.name) 149 logging.error('Unable to connect to %s to call Quit', self.name)
163 self._rpc = None 150 self._rpc = None
164 self._connected = False 151 self._connected = False
165 152
166 def _ExecuteIsolate(self): 153 def _ExecuteIsolate(self):
(...skipping 25 matching lines...) Expand all
192 179
193 if self._isolate_server: 180 if self._isolate_server:
194 cmd.extend(['--isolate-server', self._isolate_server]) 181 cmd.extend(['--isolate-server', self._isolate_server])
195 if self._swarming_server: 182 if self._swarming_server:
196 cmd.extend(['--swarming', self._swarming_server]) 183 cmd.extend(['--swarming', self._swarming_server])
197 for key, value in self._dimensions.iteritems(): 184 for key, value in self._dimensions.iteritems():
198 cmd.extend(['--dimension', key, value]) 185 cmd.extend(['--dimension', key, value])
199 186
200 cmd.extend([ 187 cmd.extend([
201 '--', 188 '--',
202 '--controller', common_lib.MY_IP, 189 '--host', common_lib.MY_IP,
203 '--otp', self._otp, 190 '--otp', self._otp,
204 '--verbosity', self._verbosity, 191 '--verbosity', self._verbosity,
205 '--idle-timeout', str(self._idle_timeout_secs), 192 '--idle-timeout', str(self._idle_timeout_secs),
206 ]) 193 ])
207 194
208 self._ExecuteProcess(cmd) 195 self._ExecuteProcess(cmd)
209 196
210 def _ExecuteProcess(self, cmd): 197 def _ExecuteProcess(self, cmd):
211 """Executes a process, waits for it to complete, and checks for success.""" 198 """Executes a process, waits for it to complete, and checks for success."""
212 logging.debug('Running %s', ' '.join(cmd)) 199 logging.debug('Running %s', ' '.join(cmd))
213 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 200 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
214 _, stderr = p.communicate() 201 _, stderr = p.communicate()
215 if p.returncode != 0: 202 if p.returncode != 0:
216 raise Error(stderr) 203 raise Error(stderr)
217 204
218 def OnConnect(self, ip_address): 205 def OnConnect(self, ip_address):
219 """Receives task ip address on connection.""" 206 """Receives client ip address on connection."""
220 self._ip_address = ip_address 207 self._ip_address = ip_address
221 self._connected = True 208 self._connected = True
222 self._rpc = common_lib.ConnectToServer(self._ip_address) 209 self._rpc = common_lib.ConnectToServer(self._ip_address)
223 logging.info('%s connected from %s', self._name, ip_address) 210 logging.info('%s connected from %s', self._name, ip_address)
224 self._connect_event.set() 211 self._connect_event.set()
OLDNEW
« no previous file with comments | « testing/legion/client_controller.py ('k') | testing/legion/client_rpc_methods.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698