Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(684)

Side by Side Diff: scripts/slave/build_scan.py

Issue 2465493002: build_scan: Fix exception handling (Closed)
Patch Set: Move exception logic Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Scans a list of masters and saves information in a build_db.""" 6 """Scans a list of masters and saves information in a build_db."""
7 7
8 from contextlib import closing 8 from contextlib import closing
9 import json 9 import json
10 import logging 10 import logging
(...skipping 26 matching lines...) Expand all
37 } 37 }
38 38
39 attempts = 0 39 attempts = 0
40 while True: 40 while True:
41 try: 41 try:
42 r = urllib2.Request(url, headers=headers) 42 r = urllib2.Request(url, headers=headers)
43 with closing(urllib2.urlopen(r, timeout=URL_TIMEOUT)) as f: 43 with closing(urllib2.urlopen(r, timeout=URL_TIMEOUT)) as f:
44 return json.load(f) 44 return json.load(f)
45 except (urllib2.URLError, IOError) as f: 45 except (urllib2.URLError, IOError) as f:
46 if attempts > MAX_ATTEMPTS: 46 if attempts > MAX_ATTEMPTS:
47 raise 47 # Raise a ValueError because this can be called from multiprocessing,
48 # which can't pickle SSLContext objects, which apparently are
49 # properties of urllib2.URLError (it seems).
50 msg = "Error encountered during URL Fetch: %s" % f
51 logging.error(msg)
52 raise ValueError(msg)
48 53
49 attempts += 1 54 attempts += 1
50 time_to_sleep = 2 ** attempts 55 time_to_sleep = 2 ** attempts
51 logging.info( 56 logging.info(
52 "url fetch encountered %s, sleeping for %d seconds and retrying..." % ( 57 "url fetch encountered %s, sleeping for %d seconds and retrying..." % (
53 f, time_to_sleep)) 58 f, time_to_sleep))
54 59
55 time.sleep(time_to_sleep) 60 time.sleep(time_to_sleep)
56 61
57 62
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after
156 161
157 # Assumes we have something like https://build.chromium.org/p/chromium.perf 162 # Assumes we have something like https://build.chromium.org/p/chromium.perf
158 name = master.rstrip('/').split('/')[-1] 163 name = master.rstrip('/').split('/')[-1]
159 if name in CBE_WHITELIST: 164 if name in CBE_WHITELIST:
160 url = '%s/p/%s/builders/%s/builds/%d?json=1' % ( 165 url = '%s/p/%s/builders/%s/builds/%d?json=1' % (
161 CBE_URL, name, urllib.quote(builder), buildnum) 166 CBE_URL, name, urllib.quote(builder), buildnum)
162 167
163 logging.debug('opening %s...' % url) 168 logging.debug('opening %s...' % url)
164 return _url_open_json(url), master, builder, buildnum 169 return _url_open_json(url), master, builder, buildnum
165 170
166
167 def get_build_jsons(master_builds, processes): 171 def get_build_jsons(master_builds, processes):
168 """Get all new builds on specified masters. 172 """Get all new builds on specified masters.
169 173
170 This takes a dict in the form of [master][builder][build], formats that URL 174 This takes a dict in the form of [master][builder][build], formats that URL
171 and appends that to url_list. Then, it forks out and queries each build_url 175 and appends that to url_list. Then, it forks out and queries each build_url
172 for build information. 176 for build information.
173 """ 177 """
174 url_list = [] 178 url_list = []
175 for master, builder_dict in master_builds.iteritems(): 179 for master, builder_dict in master_builds.iteritems():
176 for builder, new_builds in builder_dict.iteritems(): 180 for builder, new_builds in builder_dict.iteritems():
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
229 233
230 if not args: 234 if not args:
231 parser.error('you need to specify at least one master URL') 235 parser.error('you need to specify at least one master URL')
232 236
233 args = [url.rstrip('/') for url in args] 237 args = [url.rstrip('/') for url in args]
234 238
235 return options, args 239 return options, args
236 240
237 241
238 def get_updated_builds(masters, build_db, parallelism): 242 def get_updated_builds(masters, build_db, parallelism):
239 try: 243 new_builds, master_jsons = find_new_builds_per_master(masters, build_db)
240 new_builds, master_jsons = find_new_builds_per_master(masters, build_db) 244 build_jsons = get_build_jsons(new_builds, parallelism)
241 build_jsons = get_build_jsons(new_builds, parallelism) 245 propagate_build_json_to_db(build_db, build_jsons)
242 propagate_build_json_to_db(build_db, build_jsons) 246 return master_jsons, build_jsons
243 return master_jsons, build_jsons
244 247
245 # Catch this and raise a ValueError because this can be called from
246 # mulitprocessing, which can't pickle SSLContext objects, which apparently are
247 # properties of urllib2.URLError (it seems).
248 except (urllib2.URLError, IOError) as f:
249 msg = "Error encountered during URL Fetch: %s" % f
250 logging.error(msg)
251 raise ValueError(msg)
252 248
253 249
254 def main(): 250 def main():
255 options, args = get_options() 251 options, args = get_options()
256 252
257 logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO) 253 logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO)
258 254
259 masters = {} 255 masters = {}
260 for m in set(args): 256 for m in set(args):
261 masters[m] = BUILDER_WILDCARD 257 masters[m] = BUILDER_WILDCARD
(...skipping 11 matching lines...) Expand all
273 print '%s:%s:%s' % (master_url, builder, buildnum) 269 print '%s:%s:%s' % (master_url, builder, buildnum)
274 270
275 if not options.skip_build_db_update: 271 if not options.skip_build_db_update:
276 build_scan_db.save_build_db(build_db, {}, options.build_db) 272 build_scan_db.save_build_db(build_db, {}, options.build_db)
277 273
278 return 0 274 return 0
279 275
280 276
281 if __name__ == '__main__': 277 if __name__ == '__main__':
282 sys.exit(main()) 278 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698