Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(74)

Side by Side Diff: swarm_trigger_step.py

Issue 23431002: [Abandoned] Move url_open with dependencies to utils.net module. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/swarm_client
Patch Set: Created 7 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « swarm_get_results.py ('k') | tests/isolateserver_archive_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Triggers a Swarm request based off of a .isolated file. 6 """Triggers a Swarm request based off of a .isolated file.
7 7
8 This script takes a .isolated file, packages it, and sends a Swarm manifest file 8 This script takes a .isolated file, packages it, and sends a Swarm manifest file
9 to the Swarm server. This is expected to be called as a build step with the cwd 9 to the Swarm server. This is expected to be called as a build step with the cwd
10 as the parent of the src/ directory. 10 as the parent of the src/ directory.
11 """ 11 """
12 12
13 import binascii 13 import binascii
14 import hashlib 14 import hashlib
15 import json 15 import json
16 import optparse 16 import optparse
17 import os 17 import os
18 import sys 18 import sys
19 import time 19 import time
20 import urllib 20 import urllib
21 21
22 import run_isolated 22 from utils import net
23
24 from utils import tools 23 from utils import tools
25 from utils import zip_package 24 from utils import zip_package
26 25
27 26
28 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 27 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
29 TOOLS_PATH = os.path.join(ROOT_DIR, 'tools') 28 TOOLS_PATH = os.path.join(ROOT_DIR, 'tools')
30 29
31 30
32 PLATFORM_MAPPING = { 31 PLATFORM_MAPPING = {
33 'cygwin': 'Windows', 32 'cygwin': 'Windows',
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
76 self.verbose = bool(verbose) 75 self.verbose = bool(verbose)
77 self.profile = bool(profile) 76 self.profile = bool(profile)
78 self.priority = priority 77 self.priority = priority
79 78
80 self._zip_file_hash = '' 79 self._zip_file_hash = ''
81 self._tasks = [] 80 self._tasks = []
82 self._token_cache = None 81 self._token_cache = None
83 82
84 def _token(self): 83 def _token(self):
85 if not self._token_cache: 84 if not self._token_cache:
86 result = run_isolated.url_open(self._data_server_get_token) 85 result = net.url_open(self._data_server_get_token)
87 if not result: 86 if not result:
88 # TODO(maruel): Implement authentication. 87 # TODO(maruel): Implement authentication.
89 raise Failure('Failed to get token, need authentication') 88 raise Failure('Failed to get token, need authentication')
90 # Quote it right away, so creating the urls is simpler. 89 # Quote it right away, so creating the urls is simpler.
91 self._token_cache = urllib.quote(result.read()) 90 self._token_cache = urllib.quote(result.read())
92 return self._token_cache 91 return self._token_cache
93 92
94 def add_task(self, task_name, actions, time_out=600): 93 def add_task(self, task_name, actions, time_out=600):
95 """Appends a new task to the swarm manifest file.""" 94 """Appends a new task to the swarm manifest file."""
96 # See swarming/src/common/test_request_message.py TestObject constructor for 95 # See swarming/src/common/test_request_message.py TestObject constructor for
(...skipping 10 matching lines...) Expand all
107 """Zips up all the files necessary to run a shard and uploads to Swarming 106 """Zips up all the files necessary to run a shard and uploads to Swarming
108 master. 107 master.
109 """ 108 """
110 assert not self._zip_file_hash 109 assert not self._zip_file_hash
111 110
112 start_time = time.time() 111 start_time = time.time()
113 zip_contents = self.bundle.zip_into_buffer() 112 zip_contents = self.bundle.zip_into_buffer()
114 self._zip_file_hash = hashlib.sha1(zip_contents).hexdigest() 113 self._zip_file_hash = hashlib.sha1(zip_contents).hexdigest()
115 print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) 114 print 'Zipping completed, time elapsed: %f' % (time.time() - start_time)
116 115
117 response = run_isolated.url_open( 116 response = net.url_open(
118 self._data_server_has + '?token=%s' % self._token(), 117 self._data_server_has + '?token=%s' % self._token(),
119 data=binascii.unhexlify(self._zip_file_hash), 118 data=binascii.unhexlify(self._zip_file_hash),
120 content_type='application/octet-stream') 119 content_type='application/octet-stream')
121 if response is None: 120 if response is None:
122 print >> sys.stderr, ( 121 print >> sys.stderr, (
123 'Unable to query server for zip file presence, aborting.') 122 'Unable to query server for zip file presence, aborting.')
124 return False 123 return False
125 124
126 if response.read(1) == chr(1): 125 if response.read(1) == chr(1):
127 print 'Zip file already on server, no need to reupload.' 126 print 'Zip file already on server, no need to reupload.'
128 return True 127 return True
129 128
130 print 'Zip file not on server, starting uploading.' 129 print 'Zip file not on server, starting uploading.'
131 130
132 url = '%s%s?priority=0&token=%s' % ( 131 url = '%s%s?priority=0&token=%s' % (
133 self._data_server_storage, self._zip_file_hash, self._token()) 132 self._data_server_storage, self._zip_file_hash, self._token())
134 response = run_isolated.url_open( 133 response = net.url_open(
135 url, data=zip_contents, content_type='application/octet-stream') 134 url, data=zip_contents, content_type='application/octet-stream')
136 if response is None: 135 if response is None:
137 print >> sys.stderr, 'Failed to upload the zip file: %s' % url 136 print >> sys.stderr, 'Failed to upload the zip file: %s' % url
138 return False 137 return False
139 138
140 return True 139 return True
141 140
142 def to_json(self): 141 def to_json(self):
143 """Exports the current configuration into a swarm-readable manifest file. 142 """Exports the current configuration into a swarm-readable manifest file.
144 143
(...skipping 30 matching lines...) Expand all
175 test_case['env_vars']['GTEST_TOTAL_SHARDS'] = '%(num_instances)s' 174 test_case['env_vars']['GTEST_TOTAL_SHARDS'] = '%(num_instances)s'
176 175
177 return json.dumps(test_case, separators=(',',':')) 176 return json.dumps(test_case, separators=(',',':'))
178 177
179 178
180 def chromium_setup(manifest): 179 def chromium_setup(manifest):
181 """Sets up the commands to run. 180 """Sets up the commands to run.
182 181
183 Highly chromium specific. 182 Highly chromium specific.
184 """ 183 """
184 import run_isolated
185
185 # Add uncompressed zip here. It'll be compressed as part of the package sent 186 # Add uncompressed zip here. It'll be compressed as part of the package sent
186 # to Swarming server. 187 # to Swarming server.
187 run_test_name = 'run_isolated.zip' 188 run_test_name = 'run_isolated.zip'
188 manifest.bundle.add_buffer(run_test_name, 189 manifest.bundle.add_buffer(run_test_name,
189 run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) 190 run_isolated.get_as_zip_package().zip_into_buffer(compress=False))
190 191
191 cleanup_script_name = 'swarm_cleanup.py' 192 cleanup_script_name = 'swarm_cleanup.py'
192 manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name), 193 manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name),
193 cleanup_script_name) 194 cleanup_script_name)
194 195
(...skipping 29 matching lines...) Expand all
224 print "Zipping up files..." 225 print "Zipping up files..."
225 if not manifest.zip_and_upload(): 226 if not manifest.zip_and_upload():
226 return 1 227 return 1
227 228
228 # Send test requests off to swarm. 229 # Send test requests off to swarm.
229 print('Sending test requests to swarm.') 230 print('Sending test requests to swarm.')
230 print('Server: %s' % swarm_url) 231 print('Server: %s' % swarm_url)
231 print('Job name: %s' % test_name) 232 print('Job name: %s' % test_name)
232 test_url = swarm_url.rstrip('/') + '/test' 233 test_url = swarm_url.rstrip('/') + '/test'
233 manifest_text = manifest.to_json() 234 manifest_text = manifest.to_json()
234 result = run_isolated.url_open(test_url, data={'request': manifest_text}) 235 result = net.url_open(test_url, data={'request': manifest_text})
235 if not result: 236 if not result:
236 print >> sys.stderr, 'Failed to send test for %s\n%s' % ( 237 print >> sys.stderr, 'Failed to send test for %s\n%s' % (
237 test_name, test_url) 238 test_name, test_url)
238 return 1 239 return 1
239 try: 240 try:
240 json.load(result) 241 json.load(result)
241 except (ValueError, TypeError) as e: 242 except (ValueError, TypeError) as e:
242 print >> sys.stderr, 'Failed to send test for %s' % test_name 243 print >> sys.stderr, 'Failed to send test for %s' % test_name
243 print >> sys.stderr, 'Manifest: %s' % manifest_text 244 print >> sys.stderr, 'Manifest: %s' % manifest_text
244 print >> sys.stderr, str(e) 245 print >> sys.stderr, str(e)
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after
306 options.priority) 307 options.priority)
307 highest_exit_code = max(highest_exit_code, exit_code) 308 highest_exit_code = max(highest_exit_code, exit_code)
308 except Failure as e: 309 except Failure as e:
309 print >> sys.stderr, e.args[0] 310 print >> sys.stderr, e.args[0]
310 highest_exit_code = max(1, highest_exit_code) 311 highest_exit_code = max(1, highest_exit_code)
311 return highest_exit_code 312 return highest_exit_code
312 313
313 314
314 if __name__ == '__main__': 315 if __name__ == '__main__':
315 sys.exit(main(None)) 316 sys.exit(main(None))
OLDNEW
« no previous file with comments | « swarm_get_results.py ('k') | tests/isolateserver_archive_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698