| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Triggers a Swarm request based off of a .isolated file. | 6 """Triggers a Swarm request based off of a .isolated file. |
| 7 | 7 |
| 8 This script takes a .isolated file, packages it, and sends a Swarm manifest file | 8 This script takes a .isolated file, packages it, and sends a Swarm manifest file |
| 9 to the Swarm server. This is expected to be called as a build step with the cwd | 9 to the Swarm server. This is expected to be called as a build step with the cwd |
| 10 as the parent of the src/ directory. | 10 as the parent of the src/ directory. |
| 11 """ | 11 """ |
| 12 | 12 |
| 13 import binascii | 13 import binascii |
| 14 import hashlib | 14 import hashlib |
| 15 import json | 15 import json |
| 16 import optparse | 16 import optparse |
| 17 import os | 17 import os |
| 18 import sys | 18 import sys |
| 19 import time | 19 import time |
| 20 import urllib | 20 import urllib |
| 21 | 21 |
| 22 import run_isolated | 22 from utils import net |
| 23 | |
| 24 from utils import tools | 23 from utils import tools |
| 25 from utils import zip_package | 24 from utils import zip_package |
| 26 | 25 |
| 27 | 26 |
| 28 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) | 27 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 29 TOOLS_PATH = os.path.join(ROOT_DIR, 'tools') | 28 TOOLS_PATH = os.path.join(ROOT_DIR, 'tools') |
| 30 | 29 |
| 31 | 30 |
| 32 PLATFORM_MAPPING = { | 31 PLATFORM_MAPPING = { |
| 33 'cygwin': 'Windows', | 32 'cygwin': 'Windows', |
| (...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 76 self.verbose = bool(verbose) | 75 self.verbose = bool(verbose) |
| 77 self.profile = bool(profile) | 76 self.profile = bool(profile) |
| 78 self.priority = priority | 77 self.priority = priority |
| 79 | 78 |
| 80 self._zip_file_hash = '' | 79 self._zip_file_hash = '' |
| 81 self._tasks = [] | 80 self._tasks = [] |
| 82 self._token_cache = None | 81 self._token_cache = None |
| 83 | 82 |
| 84 def _token(self): | 83 def _token(self): |
| 85 if not self._token_cache: | 84 if not self._token_cache: |
| 86 result = run_isolated.url_open(self._data_server_get_token) | 85 result = net.url_open(self._data_server_get_token) |
| 87 if not result: | 86 if not result: |
| 88 # TODO(maruel): Implement authentication. | 87 # TODO(maruel): Implement authentication. |
| 89 raise Failure('Failed to get token, need authentication') | 88 raise Failure('Failed to get token, need authentication') |
| 90 # Quote it right away, so creating the urls is simpler. | 89 # Quote it right away, so creating the urls is simpler. |
| 91 self._token_cache = urllib.quote(result.read()) | 90 self._token_cache = urllib.quote(result.read()) |
| 92 return self._token_cache | 91 return self._token_cache |
| 93 | 92 |
| 94 def add_task(self, task_name, actions, time_out=600): | 93 def add_task(self, task_name, actions, time_out=600): |
| 95 """Appends a new task to the swarm manifest file.""" | 94 """Appends a new task to the swarm manifest file.""" |
| 96 # See swarming/src/common/test_request_message.py TestObject constructor for | 95 # See swarming/src/common/test_request_message.py TestObject constructor for |
| (...skipping 10 matching lines...) Expand all Loading... |
| 107 """Zips up all the files necessary to run a shard and uploads to Swarming | 106 """Zips up all the files necessary to run a shard and uploads to Swarming |
| 108 master. | 107 master. |
| 109 """ | 108 """ |
| 110 assert not self._zip_file_hash | 109 assert not self._zip_file_hash |
| 111 | 110 |
| 112 start_time = time.time() | 111 start_time = time.time() |
| 113 zip_contents = self.bundle.zip_into_buffer() | 112 zip_contents = self.bundle.zip_into_buffer() |
| 114 self._zip_file_hash = hashlib.sha1(zip_contents).hexdigest() | 113 self._zip_file_hash = hashlib.sha1(zip_contents).hexdigest() |
| 115 print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) | 114 print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) |
| 116 | 115 |
| 117 response = run_isolated.url_open( | 116 response = net.url_open( |
| 118 self._data_server_has + '?token=%s' % self._token(), | 117 self._data_server_has + '?token=%s' % self._token(), |
| 119 data=binascii.unhexlify(self._zip_file_hash), | 118 data=binascii.unhexlify(self._zip_file_hash), |
| 120 content_type='application/octet-stream') | 119 content_type='application/octet-stream') |
| 121 if response is None: | 120 if response is None: |
| 122 print >> sys.stderr, ( | 121 print >> sys.stderr, ( |
| 123 'Unable to query server for zip file presence, aborting.') | 122 'Unable to query server for zip file presence, aborting.') |
| 124 return False | 123 return False |
| 125 | 124 |
| 126 if response.read(1) == chr(1): | 125 if response.read(1) == chr(1): |
| 127 print 'Zip file already on server, no need to reupload.' | 126 print 'Zip file already on server, no need to reupload.' |
| 128 return True | 127 return True |
| 129 | 128 |
| 130 print 'Zip file not on server, starting uploading.' | 129 print 'Zip file not on server, starting uploading.' |
| 131 | 130 |
| 132 url = '%s%s?priority=0&token=%s' % ( | 131 url = '%s%s?priority=0&token=%s' % ( |
| 133 self._data_server_storage, self._zip_file_hash, self._token()) | 132 self._data_server_storage, self._zip_file_hash, self._token()) |
| 134 response = run_isolated.url_open( | 133 response = net.url_open( |
| 135 url, data=zip_contents, content_type='application/octet-stream') | 134 url, data=zip_contents, content_type='application/octet-stream') |
| 136 if response is None: | 135 if response is None: |
| 137 print >> sys.stderr, 'Failed to upload the zip file: %s' % url | 136 print >> sys.stderr, 'Failed to upload the zip file: %s' % url |
| 138 return False | 137 return False |
| 139 | 138 |
| 140 return True | 139 return True |
| 141 | 140 |
| 142 def to_json(self): | 141 def to_json(self): |
| 143 """Exports the current configuration into a swarm-readable manifest file. | 142 """Exports the current configuration into a swarm-readable manifest file. |
| 144 | 143 |
| (...skipping 30 matching lines...) Expand all Loading... |
| 175 test_case['env_vars']['GTEST_TOTAL_SHARDS'] = '%(num_instances)s' | 174 test_case['env_vars']['GTEST_TOTAL_SHARDS'] = '%(num_instances)s' |
| 176 | 175 |
| 177 return json.dumps(test_case, separators=(',',':')) | 176 return json.dumps(test_case, separators=(',',':')) |
| 178 | 177 |
| 179 | 178 |
| 180 def chromium_setup(manifest): | 179 def chromium_setup(manifest): |
| 181 """Sets up the commands to run. | 180 """Sets up the commands to run. |
| 182 | 181 |
| 183 Highly chromium specific. | 182 Highly chromium specific. |
| 184 """ | 183 """ |
| 184 import run_isolated |
| 185 |
| 185 # Add uncompressed zip here. It'll be compressed as part of the package sent | 186 # Add uncompressed zip here. It'll be compressed as part of the package sent |
| 186 # to Swarming server. | 187 # to Swarming server. |
| 187 run_test_name = 'run_isolated.zip' | 188 run_test_name = 'run_isolated.zip' |
| 188 manifest.bundle.add_buffer(run_test_name, | 189 manifest.bundle.add_buffer(run_test_name, |
| 189 run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) | 190 run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) |
| 190 | 191 |
| 191 cleanup_script_name = 'swarm_cleanup.py' | 192 cleanup_script_name = 'swarm_cleanup.py' |
| 192 manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name), | 193 manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name), |
| 193 cleanup_script_name) | 194 cleanup_script_name) |
| 194 | 195 |
| (...skipping 29 matching lines...) Expand all Loading... |
| 224 print "Zipping up files..." | 225 print "Zipping up files..." |
| 225 if not manifest.zip_and_upload(): | 226 if not manifest.zip_and_upload(): |
| 226 return 1 | 227 return 1 |
| 227 | 228 |
| 228 # Send test requests off to swarm. | 229 # Send test requests off to swarm. |
| 229 print('Sending test requests to swarm.') | 230 print('Sending test requests to swarm.') |
| 230 print('Server: %s' % swarm_url) | 231 print('Server: %s' % swarm_url) |
| 231 print('Job name: %s' % test_name) | 232 print('Job name: %s' % test_name) |
| 232 test_url = swarm_url.rstrip('/') + '/test' | 233 test_url = swarm_url.rstrip('/') + '/test' |
| 233 manifest_text = manifest.to_json() | 234 manifest_text = manifest.to_json() |
| 234 result = run_isolated.url_open(test_url, data={'request': manifest_text}) | 235 result = net.url_open(test_url, data={'request': manifest_text}) |
| 235 if not result: | 236 if not result: |
| 236 print >> sys.stderr, 'Failed to send test for %s\n%s' % ( | 237 print >> sys.stderr, 'Failed to send test for %s\n%s' % ( |
| 237 test_name, test_url) | 238 test_name, test_url) |
| 238 return 1 | 239 return 1 |
| 239 try: | 240 try: |
| 240 json.load(result) | 241 json.load(result) |
| 241 except (ValueError, TypeError) as e: | 242 except (ValueError, TypeError) as e: |
| 242 print >> sys.stderr, 'Failed to send test for %s' % test_name | 243 print >> sys.stderr, 'Failed to send test for %s' % test_name |
| 243 print >> sys.stderr, 'Manifest: %s' % manifest_text | 244 print >> sys.stderr, 'Manifest: %s' % manifest_text |
| 244 print >> sys.stderr, str(e) | 245 print >> sys.stderr, str(e) |
| (...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 306 options.priority) | 307 options.priority) |
| 307 highest_exit_code = max(highest_exit_code, exit_code) | 308 highest_exit_code = max(highest_exit_code, exit_code) |
| 308 except Failure as e: | 309 except Failure as e: |
| 309 print >> sys.stderr, e.args[0] | 310 print >> sys.stderr, e.args[0] |
| 310 highest_exit_code = max(1, highest_exit_code) | 311 highest_exit_code = max(1, highest_exit_code) |
| 311 return highest_exit_code | 312 return highest_exit_code |
| 312 | 313 |
| 313 | 314 |
| 314 if __name__ == '__main__': | 315 if __name__ == '__main__': |
| 315 sys.exit(main(None)) | 316 sys.exit(main(None)) |
| OLD | NEW |