| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Triggers a Swarm request based off of a .isolated file. | |
| 7 | |
| 8 This script takes a .isolated file, packages it, and sends a Swarm manifest file | |
| 9 to the Swarm server. This is expected to be called as a build step with the cwd | |
| 10 as the parent of the src/ directory. | |
| 11 """ | |
| 12 | |
| 13 import hashlib | |
| 14 import json | |
| 15 import optparse | |
| 16 import os | |
| 17 import StringIO | |
| 18 import sys | |
| 19 import time | |
| 20 import urllib | |
| 21 import zipfile | |
| 22 | |
| 23 import run_isolated | |
| 24 | |
| 25 | |
| 26 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| 27 TOOLS_PATH = os.path.join(ROOT_DIR, 'tools') | |
| 28 | |
| 29 | |
| 30 PLATFORM_MAPPING = { | |
| 31 'cygwin': 'Windows', | |
| 32 'darwin': 'Mac', | |
| 33 'linux2': 'Linux', | |
| 34 'win32': 'Windows', | |
| 35 } | |
| 36 | |
| 37 | |
| 38 class Failure(Exception): | |
| 39 pass | |
| 40 | |
| 41 | |
| 42 class Manifest(object): | |
| 43 def __init__( | |
| 44 self, manifest_hash, test_name, shards, test_filter, os_image, | |
| 45 working_dir, data_server, verbose, profile, priority): | |
| 46 """Populates a manifest object. | |
| 47 Args: | |
| 48 manifest_hash - The manifest's sha-1 that the slave is going to fetch. | |
| 49 test_name - The name to give the test request. | |
| 50 shards - The number of swarm shards to request. | |
| 51 test_filter - The gtest filter to apply when running the test. | |
| 52 os_image - OS to run on. | |
| 53 working_dir - Relative working directory to start the script. | |
| 54 data_server - isolate server url. | |
| 55 verbose - if True, have the slave print more details. | |
| 56 profile - if True, have the slave print more timing data. | |
| 57 priority - int between 0 and 1000, lower the higher priority | |
| 58 """ | |
| 59 self.manifest_hash = manifest_hash | |
| 60 self._test_name = test_name | |
| 61 self._shards = shards | |
| 62 self._test_filter = test_filter | |
| 63 self._target_platform = PLATFORM_MAPPING[os_image] | |
| 64 self._working_dir = working_dir | |
| 65 | |
| 66 base_url = data_server.rstrip('/') | |
| 67 self.data_server_retrieval = base_url + '/content/retrieve/default/' | |
| 68 self._data_server_storage = base_url + '/content/store/default/' | |
| 69 self._data_server_has = base_url + '/content/contains/default' | |
| 70 self._data_server_get_token = base_url + '/content/get_token' | |
| 71 | |
| 72 self.verbose = bool(verbose) | |
| 73 self.profile = bool(profile) | |
| 74 self.priority = priority | |
| 75 | |
| 76 self._zip_file_hash = '' | |
| 77 self._tasks = [] | |
| 78 self._files = {} | |
| 79 self._token_cache = None | |
| 80 | |
| 81 def _token(self): | |
| 82 if not self._token_cache: | |
| 83 result = run_isolated.url_open(self._data_server_get_token) | |
| 84 if not result: | |
| 85 # TODO(maruel): Implement authentication. | |
| 86 raise Failure('Failed to get token, need authentication') | |
| 87 # Quote it right away, so creating the urls is simpler. | |
| 88 self._token_cache = urllib.quote(result.read()) | |
| 89 return self._token_cache | |
| 90 | |
| 91 def add_task(self, task_name, actions, time_out=600): | |
| 92 """Appends a new task to the swarm manifest file.""" | |
| 93 # See swarming/src/common/test_request_message.py TestObject constructor for | |
| 94 # the valid flags. | |
| 95 self._tasks.append( | |
| 96 { | |
| 97 'action': actions, | |
| 98 'decorate_output': self.verbose, | |
| 99 'test_name': task_name, | |
| 100 'time_out': time_out, | |
| 101 }) | |
| 102 | |
| 103 def add_file(self, source_path, rel_path): | |
| 104 self._files[source_path] = rel_path | |
| 105 | |
| 106 def zip_and_upload(self): | |
| 107 """Zips up all the files necessary to run a shard and uploads to Swarming | |
| 108 master. | |
| 109 """ | |
| 110 assert not self._zip_file_hash | |
| 111 start_time = time.time() | |
| 112 | |
| 113 zip_memory_file = StringIO.StringIO() | |
| 114 zip_file = zipfile.ZipFile(zip_memory_file, 'w') | |
| 115 | |
| 116 for source, relpath in self._files.iteritems(): | |
| 117 zip_file.write(source, relpath) | |
| 118 | |
| 119 zip_file.close() | |
| 120 print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) | |
| 121 | |
| 122 zip_memory_file.flush() | |
| 123 zip_contents = zip_memory_file.getvalue() | |
| 124 zip_memory_file.close() | |
| 125 | |
| 126 self._zip_file_hash = hashlib.sha1(zip_contents).hexdigest() | |
| 127 | |
| 128 response = run_isolated.url_open( | |
| 129 self._data_server_has + '?token=%s' % self._token(), | |
| 130 data=self._zip_file_hash, | |
| 131 content_type='application/octet-stream') | |
| 132 if response is None: | |
| 133 print >> sys.stderr, ( | |
| 134 'Unable to query server for zip file presence, aborting.') | |
| 135 return False | |
| 136 | |
| 137 if response.read(1) == chr(1): | |
| 138 print 'Zip file already on server, no need to reupload.' | |
| 139 return True | |
| 140 | |
| 141 print 'Zip file not on server, starting uploading.' | |
| 142 | |
| 143 url = '%s%s?priority=0&token=%s' % ( | |
| 144 self._data_server_storage, self._zip_file_hash, self._token()) | |
| 145 response = run_isolated.url_open( | |
| 146 url, data=zip_contents, content_type='application/octet-stream') | |
| 147 if response is None: | |
| 148 print >> sys.stderr, 'Failed to upload the zip file: %s' % url | |
| 149 return False | |
| 150 | |
| 151 return True | |
| 152 | |
| 153 def to_json(self): | |
| 154 """Exports the current configuration into a swarm-readable manifest file. | |
| 155 | |
| 156 This function doesn't mutate the object. | |
| 157 """ | |
| 158 test_case = { | |
| 159 'test_case_name': self._test_name, | |
| 160 'data': [ | |
| 161 [self.data_server_retrieval + urllib.quote(self._zip_file_hash), | |
| 162 'swarm_data.zip'], | |
| 163 ], | |
| 164 'tests': self._tasks, | |
| 165 'env_vars': {}, | |
| 166 'configurations': [ | |
| 167 { | |
| 168 'min_instances': self._shards, | |
| 169 'config_name': self._target_platform, | |
| 170 'dimensions': { | |
| 171 'os': self._target_platform, | |
| 172 }, | |
| 173 }, | |
| 174 ], | |
| 175 'working_dir': self._working_dir, | |
| 176 'restart_on_failure': True, | |
| 177 'cleanup': 'root', | |
| 178 'priority': self.priority, | |
| 179 } | |
| 180 | |
| 181 # These flags are googletest specific. | |
| 182 if self._test_filter and self._test_filter != '*': | |
| 183 test_case['env_vars']['GTEST_FILTER'] = self._test_filter | |
| 184 if self._shards > 1: | |
| 185 test_case['env_vars']['GTEST_SHARD_INDEX'] = '%(instance_index)s' | |
| 186 test_case['env_vars']['GTEST_TOTAL_SHARDS'] = '%(num_instances)s' | |
| 187 | |
| 188 return json.dumps(test_case, separators=(',',':')) | |
| 189 | |
| 190 | |
| 191 def chromium_setup(manifest): | |
| 192 """Sets up the commands to run. | |
| 193 | |
| 194 Highly chromium specific. | |
| 195 """ | |
| 196 cleanup_script_name = 'swarm_cleanup.py' | |
| 197 cleanup_script_path = os.path.join(TOOLS_PATH, cleanup_script_name) | |
| 198 run_test_name = 'run_isolated.py' | |
| 199 run_test_path = os.path.join(ROOT_DIR, run_test_name) | |
| 200 | |
| 201 manifest.add_file(run_test_path, run_test_name) | |
| 202 manifest.add_file(cleanup_script_path, cleanup_script_name) | |
| 203 run_cmd = [ | |
| 204 'python', run_test_name, | |
| 205 '--hash', manifest.manifest_hash, | |
| 206 '--remote', manifest.data_server_retrieval.rstrip('/') + '-gzip/', | |
| 207 ] | |
| 208 if manifest.verbose or manifest.profile: | |
| 209 # Have it print the profiling section. | |
| 210 run_cmd.append('--verbose') | |
| 211 manifest.add_task('Run Test', run_cmd) | |
| 212 | |
| 213 # Clean up | |
| 214 manifest.add_task('Clean Up', ['python', cleanup_script_name]) | |
| 215 | |
| 216 | |
| 217 def process_manifest( | |
| 218 file_sha1, test_name, shards, test_filter, os_image, working_dir, | |
| 219 data_server, swarm_url, verbose, profile, priority): | |
| 220 """Process the manifest file and send off the swarm test request.""" | |
| 221 try: | |
| 222 manifest = Manifest( | |
| 223 file_sha1, test_name, shards, test_filter, os_image, working_dir, | |
| 224 data_server, verbose, profile, priority) | |
| 225 except ValueError as e: | |
| 226 print >> sys.stderr, 'Unable to process %s: %s' % (test_name, e) | |
| 227 return 1 | |
| 228 | |
| 229 chromium_setup(manifest) | |
| 230 | |
| 231 # Zip up relevent files | |
| 232 print "Zipping up files..." | |
| 233 if not manifest.zip_and_upload(): | |
| 234 return 1 | |
| 235 | |
| 236 # Send test requests off to swarm. | |
| 237 print('Sending test requests to swarm.') | |
| 238 print('Server: %s' % swarm_url) | |
| 239 print('Job name: %s' % test_name) | |
| 240 test_url = swarm_url.rstrip('/') + '/test' | |
| 241 manifest_text = manifest.to_json() | |
| 242 result = run_isolated.url_open(test_url, data={'request': manifest_text}) | |
| 243 if not result: | |
| 244 print >> sys.stderr, 'Failed to send test for %s\n%s' % ( | |
| 245 test_name, test_url) | |
| 246 return 1 | |
| 247 try: | |
| 248 json.load(result) | |
| 249 except (ValueError, TypeError) as e: | |
| 250 print >> sys.stderr, 'Failed to send test for %s' % test_name | |
| 251 print >> sys.stderr, 'Manifest: %s' % manifest_text | |
| 252 print >> sys.stderr, str(e) | |
| 253 return 1 | |
| 254 return 0 | |
| 255 | |
| 256 | |
| 257 def main(argv): | |
| 258 run_isolated.disable_buffering() | |
| 259 parser = optparse.OptionParser( | |
| 260 usage='%prog [options]', description=sys.modules[__name__].__doc__) | |
| 261 parser.add_option('-w', '--working_dir', default='swarm_tests', | |
| 262 help='Desired working direction on the swarm slave side. ' | |
| 263 'Defaults to %default.') | |
| 264 parser.add_option('-o', '--os_image', | |
| 265 help='Swarm OS image to request.') | |
| 266 parser.add_option('-u', '--swarm-url', default='http://localhost:8080', | |
| 267 help='Specify the url of the Swarm server. ' | |
| 268 'Defaults to %default') | |
| 269 parser.add_option('-d', '--data-server', | |
| 270 help='The server where all the test data is stored.') | |
| 271 parser.add_option('-t', '--test-name-prefix', default='', | |
| 272 help='Specify the prefix to give the swarm test request. ' | |
| 273 'Defaults to %default') | |
| 274 parser.add_option('--run_from_hash', nargs=4, action='append', default=[], | |
| 275 help='Specify a hash to run on swarm. The format is ' | |
| 276 '(hash, hash_test_name, shards, test_filter). This may be ' | |
| 277 'used multiple times to send multiple hashes.') | |
| 278 parser.add_option('-v', '--verbose', action='store_true', | |
| 279 help='Print verbose logging') | |
| 280 parser.add_option('--profile', action='store_true', | |
| 281 default=bool(os.environ.get('ISOLATE_DEBUG')), | |
| 282 help='Have run_isolated.py print profiling info') | |
| 283 parser.add_option('--priority', type='int', default=100, | |
| 284 help='The lower value, the more important the task is') | |
| 285 (options, args) = parser.parse_args(argv) | |
| 286 | |
| 287 if args: | |
| 288 parser.error('Unknown args: %s' % args) | |
| 289 | |
| 290 if not options.os_image or options.os_image == 'None': | |
| 291 # This means the Try Server/user wants to use the current OS. | |
| 292 options.os_image = sys.platform | |
| 293 if not options.data_server: | |
| 294 parser.error('Must specify the data directory') | |
| 295 | |
| 296 if not options.run_from_hash: | |
| 297 parser.error('At least one --run_from_hash is required.') | |
| 298 | |
| 299 highest_exit_code = 0 | |
| 300 try: | |
| 301 # Send off the hash swarm test requests. | |
| 302 for (file_sha1, test_name, shards, testfilter) in options.run_from_hash: | |
| 303 exit_code = process_manifest( | |
| 304 file_sha1, | |
| 305 options.test_name_prefix + test_name, | |
| 306 int(shards), | |
| 307 testfilter, | |
| 308 options.os_image, | |
| 309 options.working_dir, | |
| 310 options.data_server, | |
| 311 options.swarm_url, | |
| 312 options.verbose, | |
| 313 options.profile, | |
| 314 options.priority) | |
| 315 highest_exit_code = max(highest_exit_code, exit_code) | |
| 316 except Failure as e: | |
| 317 print >> sys.stderr, e.args[0] | |
| 318 highest_exit_code = max(1, highest_exit_code) | |
| 319 return highest_exit_code | |
| 320 | |
| 321 | |
| 322 if __name__ == '__main__': | |
| 323 sys.exit(main(None)) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 22980008:
Merge all swarm_*.py scripts into swarming.py. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/swarm_client