| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Scans a list of masters and saves information in a build_db.""" | 6 """Scans a list of masters and saves information in a build_db.""" |
| 7 | 7 |
| 8 from contextlib import closing | 8 from contextlib import closing |
| 9 import json | 9 import json |
| 10 import logging | 10 import logging |
| (...skipping 26 matching lines...) Expand all Loading... |
| 37 } | 37 } |
| 38 | 38 |
| 39 attempts = 0 | 39 attempts = 0 |
| 40 while True: | 40 while True: |
| 41 try: | 41 try: |
| 42 r = urllib2.Request(url, headers=headers) | 42 r = urllib2.Request(url, headers=headers) |
| 43 with closing(urllib2.urlopen(r, timeout=URL_TIMEOUT)) as f: | 43 with closing(urllib2.urlopen(r, timeout=URL_TIMEOUT)) as f: |
| 44 return json.load(f) | 44 return json.load(f) |
| 45 except (urllib2.URLError, IOError) as f: | 45 except (urllib2.URLError, IOError) as f: |
| 46 if attempts > MAX_ATTEMPTS: | 46 if attempts > MAX_ATTEMPTS: |
| 47 raise | 47 # Raise a ValueError because this can be called from multiprocessing, |
| 48 # which can't pickle SSLContext objects, which apparently are |
| 49 # properties of urllib2.URLError (it seems). |
| 50 msg = "Error encountered during URL Fetch: %s" % f |
| 51 logging.error(msg) |
| 52 raise ValueError(msg) |
| 48 | 53 |
| 49 attempts += 1 | 54 attempts += 1 |
| 50 time_to_sleep = 2 ** attempts | 55 time_to_sleep = 2 ** attempts |
| 51 logging.info( | 56 logging.info( |
| 52 "url fetch encountered %s, sleeping for %d seconds and retrying..." % ( | 57 "url fetch encountered %s, sleeping for %d seconds and retrying..." % ( |
| 53 f, time_to_sleep)) | 58 f, time_to_sleep)) |
| 54 | 59 |
| 55 time.sleep(time_to_sleep) | 60 time.sleep(time_to_sleep) |
| 56 | 61 |
| 57 | 62 |
| (...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 156 | 161 |
| 157 # Assumes we have something like https://build.chromium.org/p/chromium.perf | 162 # Assumes we have something like https://build.chromium.org/p/chromium.perf |
| 158 name = master.rstrip('/').split('/')[-1] | 163 name = master.rstrip('/').split('/')[-1] |
| 159 if name in CBE_WHITELIST: | 164 if name in CBE_WHITELIST: |
| 160 url = '%s/p/%s/builders/%s/builds/%d?json=1' % ( | 165 url = '%s/p/%s/builders/%s/builds/%d?json=1' % ( |
| 161 CBE_URL, name, urllib.quote(builder), buildnum) | 166 CBE_URL, name, urllib.quote(builder), buildnum) |
| 162 | 167 |
| 163 logging.debug('opening %s...' % url) | 168 logging.debug('opening %s...' % url) |
| 164 return _url_open_json(url), master, builder, buildnum | 169 return _url_open_json(url), master, builder, buildnum |
| 165 | 170 |
| 166 | |
| 167 def get_build_jsons(master_builds, processes): | 171 def get_build_jsons(master_builds, processes): |
| 168 """Get all new builds on specified masters. | 172 """Get all new builds on specified masters. |
| 169 | 173 |
| 170 This takes a dict in the form of [master][builder][build], formats that URL | 174 This takes a dict in the form of [master][builder][build], formats that URL |
| 171 and appends that to url_list. Then, it forks out and queries each build_url | 175 and appends that to url_list. Then, it forks out and queries each build_url |
| 172 for build information. | 176 for build information. |
| 173 """ | 177 """ |
| 174 url_list = [] | 178 url_list = [] |
| 175 for master, builder_dict in master_builds.iteritems(): | 179 for master, builder_dict in master_builds.iteritems(): |
| 176 for builder, new_builds in builder_dict.iteritems(): | 180 for builder, new_builds in builder_dict.iteritems(): |
| (...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 229 | 233 |
| 230 if not args: | 234 if not args: |
| 231 parser.error('you need to specify at least one master URL') | 235 parser.error('you need to specify at least one master URL') |
| 232 | 236 |
| 233 args = [url.rstrip('/') for url in args] | 237 args = [url.rstrip('/') for url in args] |
| 234 | 238 |
| 235 return options, args | 239 return options, args |
| 236 | 240 |
| 237 | 241 |
| 238 def get_updated_builds(masters, build_db, parallelism): | 242 def get_updated_builds(masters, build_db, parallelism): |
| 239 try: | 243 new_builds, master_jsons = find_new_builds_per_master(masters, build_db) |
| 240 new_builds, master_jsons = find_new_builds_per_master(masters, build_db) | 244 build_jsons = get_build_jsons(new_builds, parallelism) |
| 241 build_jsons = get_build_jsons(new_builds, parallelism) | 245 propagate_build_json_to_db(build_db, build_jsons) |
| 242 propagate_build_json_to_db(build_db, build_jsons) | 246 return master_jsons, build_jsons |
| 243 return master_jsons, build_jsons | |
| 244 | 247 |
| 245 # Catch this and raise a ValueError because this can be called from | |
| 246 # mulitprocessing, which can't pickle SSLContext objects, which apparently are | |
| 247 # properties of urllib2.URLError (it seems). | |
| 248 except (urllib2.URLError, IOError) as f: | |
| 249 msg = "Error encountered during URL Fetch: %s" % f | |
| 250 logging.error(msg) | |
| 251 raise ValueError(msg) | |
| 252 | 248 |
| 253 | 249 |
| 254 def main(): | 250 def main(): |
| 255 options, args = get_options() | 251 options, args = get_options() |
| 256 | 252 |
| 257 logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO) | 253 logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO) |
| 258 | 254 |
| 259 masters = {} | 255 masters = {} |
| 260 for m in set(args): | 256 for m in set(args): |
| 261 masters[m] = BUILDER_WILDCARD | 257 masters[m] = BUILDER_WILDCARD |
| (...skipping 11 matching lines...) Expand all Loading... |
| 273 print '%s:%s:%s' % (master_url, builder, buildnum) | 269 print '%s:%s:%s' % (master_url, builder, buildnum) |
| 274 | 270 |
| 275 if not options.skip_build_db_update: | 271 if not options.skip_build_db_update: |
| 276 build_scan_db.save_build_db(build_db, {}, options.build_db) | 272 build_scan_db.save_build_db(build_db, {}, options.build_db) |
| 277 | 273 |
| 278 return 0 | 274 return 0 |
| 279 | 275 |
| 280 | 276 |
| 281 if __name__ == '__main__': | 277 if __name__ == '__main__': |
| 282 sys.exit(main()) | 278 sys.exit(main()) |
| OLD | NEW |