| OLD | NEW |
| (Empty) |
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 """Define the supported projects.""" | |
| 5 | |
| 6 import json | |
| 7 import logging | |
| 8 import os | |
| 9 import re | |
| 10 import sys | |
| 11 import urllib2 | |
| 12 | |
| 13 import find_depot_tools # pylint: disable=W0611 | |
| 14 import checkout | |
| 15 | |
| 16 import async_push | |
| 17 import context | |
| 18 import errors | |
| 19 import pending_manager | |
| 20 from post_processors import chromium_copyright | |
| 21 from verification import presubmit_check | |
| 22 from verification import project_base | |
| 23 from verification import reviewer_lgtm | |
| 24 from verification import tree_status | |
| 25 from verification import trigger_experimental_try_job | |
| 26 from verification import try_job_steps | |
| 27 from verification import try_job_on_rietveld | |
| 28 from verification import try_server | |
| 29 | |
| 30 | |
| 31 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| 32 INTERNAL_DIR = os.path.abspath( | |
| 33 os.path.join(ROOT_DIR, os.pardir, 'commit-queue-internal')) | |
| 34 | |
| 35 # These come from commit-queue in the internal repo. | |
| 36 if os.path.isdir(INTERNAL_DIR): | |
| 37 sys.path.insert(0, INTERNAL_DIR) | |
| 38 import chromium_committers # pylint: disable=F0401 | |
| 39 import gyp_committers # pylint: disable=F0401 | |
| 40 import nacl_committers # pylint: disable=F0401 | |
| 41 import skia_committers # pylint: disable=F0401 | |
| 42 else: | |
| 43 print >> sys.stderr, ( | |
| 44 'Failed to find commit-queue-internal; will fail to start!') | |
| 45 chromium_committers = None | |
| 46 gyp_committers = None | |
| 47 nacl_committers = None | |
| 48 skia_committers = None | |
| 49 | |
| 50 | |
| 51 # It's tricky here because 'chrome' is remapped to 'svn' on src.chromium.org but | |
| 52 # the other repositories keep their repository name. So don't list it here. | |
| 53 SVN_HOST_ALIASES = [ | |
| 54 'svn://svn.chromium.org', | |
| 55 'svn://chrome-svn', | |
| 56 'svn://chrome-svn.corp', | |
| 57 'svn://chrome-svn.corp.google.com' | |
| 58 ] | |
| 59 | |
| 60 CHROME_SVN_BASES = [item + '/chrome' for item in SVN_HOST_ALIASES] + [ | |
| 61 'http://src.chromium.org/svn', | |
| 62 'https://src.chromium.org/svn', | |
| 63 'http://src.chromium.org/chrome', | |
| 64 'https://src.chromium.org/chrome', | |
| 65 ] | |
| 66 | |
| 67 BLINK_SVN_BASES = [item + '/blink' for item in SVN_HOST_ALIASES] + [ | |
| 68 'http://src.chromium.org/blink', | |
| 69 'https://src.chromium.org/blink', | |
| 70 ] | |
| 71 | |
| 72 # Steps that are never considered to determine the try job success. | |
| 73 IGNORED_STEPS = ( | |
| 74 'svnkill', 'update_scripts', 'taskkill', 'cleanup_temp', 'process_dumps') | |
| 75 | |
| 76 # To be used in a regexp to match the branch part of an git url. | |
| 77 BRANCH_MATCH = r'\@[a-zA-Z0-9\-_\.]+' | |
| 78 | |
| 79 | |
| 80 def _read_lines(filepath, what): | |
| 81 try: | |
| 82 return open(filepath).readlines() | |
| 83 except IOError: | |
| 84 raise errors.ConfigurationError('Put the %s in %s' % (what, filepath)) | |
| 85 | |
| 86 | |
| 87 def _get_chromium_committers(): | |
| 88 """Gets the list of all allowed committers.""" | |
| 89 if not chromium_committers: | |
| 90 # Fake values. | |
| 91 entries = ['georges'] | |
| 92 else: | |
| 93 entries = chromium_committers.get_list() | |
| 94 logging.info('Found %d committers' % len(entries)) | |
| 95 return ['^%s$' % re.escape(i) for i in entries] | |
| 96 | |
| 97 | |
| 98 def _get_skia_committers(): | |
| 99 """Gets the list of all allowed committers.""" | |
| 100 if not skia_committers: | |
| 101 # Fake values. | |
| 102 entries = ['georges'] | |
| 103 else: | |
| 104 entries = skia_committers.get_list() | |
| 105 logging.info('Found %d committers' % len(entries)) | |
| 106 return ['^%s$' % re.escape(i) for i in entries] | |
| 107 | |
| 108 | |
| 109 def _get_nacl_committers(): | |
| 110 """Gets the list of all allowed committers.""" | |
| 111 if not nacl_committers: | |
| 112 # Fake values. | |
| 113 entries = ['georges'] | |
| 114 else: | |
| 115 entries = nacl_committers.get_list() | |
| 116 logging.info('Found %d committers' % len(entries)) | |
| 117 return ['^%s$' % re.escape(i) for i in entries] | |
| 118 | |
| 119 | |
| 120 def _get_gyp_committers(): | |
| 121 """Gets the list of all allowed committers.""" | |
| 122 if not gyp_committers: | |
| 123 # Fake values. | |
| 124 entries = ['georges'] | |
| 125 else: | |
| 126 entries = gyp_committers.get_list() | |
| 127 logging.info('Found %d committers' % len(entries)) | |
| 128 return ['^%s$' % re.escape(i) for i in entries] | |
| 129 | |
| 130 | |
| 131 def _chromium_lkgr(): | |
| 132 try: | |
| 133 return int( | |
| 134 urllib2.urlopen('https://chromium-status.appspot.com/lkgr').read()) | |
| 135 except (ValueError, IOError): | |
| 136 return None | |
| 137 | |
| 138 | |
| 139 def _nacl_lkgr(): | |
| 140 try: | |
| 141 return int( | |
| 142 urllib2.urlopen('https://nativeclient-status.appspot.com/lkgr').read()) | |
| 143 except (ValueError, IOError): | |
| 144 return None | |
| 145 | |
| 146 | |
| 147 def _chromium_status_pwd(root_dir): | |
| 148 filepath = os.path.join(root_dir, '.chromium_status_pwd') | |
| 149 return _read_lines(filepath, 'chromium-status password')[0].strip() | |
| 150 | |
| 151 | |
| 152 def _gen_blink(user, root_dir, rietveld_obj, no_try): | |
| 153 """Generates a PendingManager commit queue for blink/trunk.""" | |
| 154 local_checkout = checkout.SvnCheckout( | |
| 155 root_dir, | |
| 156 'blink', | |
| 157 user, | |
| 158 None, | |
| 159 'svn://svn.chromium.org/blink/trunk', | |
| 160 []) | |
| 161 context_obj = context.Context( | |
| 162 rietveld_obj, | |
| 163 local_checkout, | |
| 164 async_push.AsyncPush( | |
| 165 'https://chromium-status.appspot.com/cq', | |
| 166 _chromium_status_pwd(root_dir))) | |
| 167 | |
| 168 project_bases = [ | |
| 169 '^%s/trunk(|/.*)$' % re.escape(base) for base in BLINK_SVN_BASES] | |
| 170 project_bases.append( | |
| 171 r'^https?\:\/\/chromium.googlesource.com\/chromium\/blink(?:\.git)?%s$' % | |
| 172 BRANCH_MATCH) | |
| 173 verifiers_no_patch = [ | |
| 174 project_base.ProjectBaseUrlVerifier(project_bases), | |
| 175 reviewer_lgtm.ReviewerLgtmVerifier( | |
| 176 _get_chromium_committers(), | |
| 177 [re.escape(user)]), | |
| 178 ] | |
| 179 verifiers = [] | |
| 180 prereq_builder = 'blink_presubmit' | |
| 181 prereq_tests = ['presubmit'] | |
| 182 step_verifiers = [ | |
| 183 try_job_steps.TryJobSteps(builder_name=prereq_builder, | |
| 184 steps=prereq_tests)] | |
| 185 if not no_try: | |
| 186 blink_tests = [ | |
| 187 'blink_heap_unittests', | |
| 188 'blink_platform_unittests', | |
| 189 'webkit_lint', | |
| 190 'webkit_python_tests', | |
| 191 'webkit_tests', | |
| 192 'webkit_unit_tests', | |
| 193 'wtf_unittests', | |
| 194 ] | |
| 195 | |
| 196 # A "compile-only" bot runs the webkit_lint tests (which are fast) | |
| 197 # in order to pick up the default build targets. We don't use the | |
| 198 # "compile" step because that will build all the chromium targets, not | |
| 199 # just the blink-specific ones. | |
| 200 compile_only = [ 'webkit_lint' ] | |
| 201 | |
| 202 builders_and_tests = { | |
| 203 'mac_layout': compile_only, | |
| 204 'win_layout': compile_only, | |
| 205 | |
| 206 'linux_blink': blink_tests, | |
| 207 'linux_blink_rel': blink_tests, | |
| 208 'mac_blink_rel': blink_tests, | |
| 209 'win_blink_rel': blink_tests, | |
| 210 } | |
| 211 | |
| 212 step_verifiers += [ | |
| 213 try_job_steps.TryJobSteps(builder_name=b, prereq_builder=prereq_builder, | |
| 214 prereq_tests=prereq_tests, steps=s) | |
| 215 for b, s in builders_and_tests.iteritems() | |
| 216 ] | |
| 217 | |
| 218 verifiers.append(try_job_on_rietveld.TryRunnerRietveld( | |
| 219 context_obj, | |
| 220 'http://build.chromium.org/p/tryserver.chromium/', | |
| 221 user, | |
| 222 step_verifiers, | |
| 223 IGNORED_STEPS, | |
| 224 'src')) | |
| 225 | |
| 226 verifiers.append(tree_status.TreeStatusVerifier( | |
| 227 'https://blink-status.appspot.com')) | |
| 228 return pending_manager.PendingManager( | |
| 229 context_obj, | |
| 230 verifiers_no_patch, | |
| 231 verifiers) | |
| 232 | |
| 233 | |
| 234 def _gen_chromium(user, root_dir, rietveld_obj, no_try): | |
| 235 """Generates a PendingManager commit queue for chrome/trunk/src.""" | |
| 236 local_checkout = checkout.SvnCheckout( | |
| 237 root_dir, | |
| 238 'chromium', | |
| 239 user, | |
| 240 None, | |
| 241 'svn://svn.chromium.org/chrome/trunk/src', | |
| 242 [chromium_copyright.process]) | |
| 243 context_obj = context.Context( | |
| 244 rietveld_obj, | |
| 245 local_checkout, | |
| 246 async_push.AsyncPush( | |
| 247 'https://chromium-status.appspot.com/cq', | |
| 248 _chromium_status_pwd(root_dir))) | |
| 249 | |
| 250 project_bases = [ | |
| 251 '^%s/trunk/src(|/.*)$' % re.escape(base) for base in CHROME_SVN_BASES] | |
| 252 | |
| 253 aliases = ( | |
| 254 # Old path. | |
| 255 'git.chromium.org/git/chromium', | |
| 256 # New path. | |
| 257 'git.chromium.org/chromium/src', | |
| 258 'chromium.googlesource.com/chromium/src', | |
| 259 'chromium.googlesource.com/a/chromium/src', | |
| 260 ) | |
| 261 project_bases.extend( | |
| 262 r'^https?\:\/\/%s(?:\.git)?%s$' % (re.escape(i), BRANCH_MATCH) | |
| 263 for i in aliases) | |
| 264 verifiers_no_patch = [ | |
| 265 project_base.ProjectBaseUrlVerifier(project_bases), | |
| 266 reviewer_lgtm.ReviewerLgtmVerifier( | |
| 267 _get_chromium_committers(), | |
| 268 [re.escape(user)]), | |
| 269 ] | |
| 270 verifiers = [] | |
| 271 prereq_builder = 'chromium_presubmit' | |
| 272 prereq_tests = ['presubmit'] | |
| 273 step_verifiers = [ | |
| 274 try_job_steps.TryJobSteps(builder_name=prereq_builder, | |
| 275 steps=prereq_tests)] | |
| 276 if not no_try: | |
| 277 # To add tests to this list, they MUST be in | |
| 278 # /chrome/trunk/tools/build/masters/master.chromium/master_gatekeeper_cfg.py | |
| 279 # or somehow close the tree whenever they break. | |
| 280 standard_tests = [ | |
| 281 'base_unittests', | |
| 282 'browser_tests', | |
| 283 'cacheinvalidation_unittests', | |
| 284 'check_deps', | |
| 285 'check_deps2git', | |
| 286 'content_browsertests', | |
| 287 'content_unittests', | |
| 288 'crypto_unittests', | |
| 289 #'gfx_unittests', | |
| 290 # Broken in release. | |
| 291 #'url_unittests', | |
| 292 'gpu_unittests', | |
| 293 'ipc_tests', | |
| 294 'interactive_ui_tests', | |
| 295 'jingle_unittests', | |
| 296 'media_unittests', | |
| 297 'net_unittests', | |
| 298 'ppapi_unittests', | |
| 299 'printing_unittests', | |
| 300 'sql_unittests', | |
| 301 'sync_unit_tests', | |
| 302 'unit_tests', | |
| 303 #'webkit_unit_tests', | |
| 304 ] | |
| 305 # Use a smaller set of tests for *_aura, since there's a lot of overlap with | |
| 306 # the corresponding *_rel builders. | |
| 307 # Note: *_aura are Release builders even if their names convey otherwise. | |
| 308 linux_aura_tests = [ | |
| 309 'app_list_unittests', | |
| 310 'aura_unittests', | |
| 311 'browser_tests', | |
| 312 'compositor_unittests', | |
| 313 'content_browsertests', | |
| 314 'content_unittests', | |
| 315 'events_unittests', | |
| 316 'interactive_ui_tests', | |
| 317 'unit_tests', | |
| 318 ] | |
| 319 builders_and_tests = { | |
| 320 # TODO(maruel): Figure out a way to run 'sizes' where people can | |
| 321 # effectively update the perf expectation correctly. This requires a | |
| 322 # clobber=True build running 'sizes'. 'sizes' is not accurate with | |
| 323 # incremental build. Reference: | |
| 324 # http://chromium.org/developers/tree-sheriffs/perf-sheriffs. | |
| 325 # TODO(maruel): An option would be to run 'sizes' but not count a failure | |
| 326 # of this step as a try job failure. | |
| 327 'android_dbg': ['slave_steps'], | |
| 328 'android_clang_dbg': ['slave_steps'], | |
| 329 'android_aosp': ['compile'], | |
| 330 'ios_dbg_simulator': [ | |
| 331 'compile', | |
| 332 'base_unittests', | |
| 333 'components_unittests', | |
| 334 'content_unittests', | |
| 335 'crypto_unittests', | |
| 336 'url_unittests', | |
| 337 'net_unittests', | |
| 338 'sql_unittests', | |
| 339 'ui_unittests', | |
| 340 ], | |
| 341 'ios_rel_device': ['compile'], | |
| 342 'ios_rel_device_ninja': ['compile'], | |
| 343 'linux_aura': linux_aura_tests, | |
| 344 'linux_clang': ['compile'], | |
| 345 'linux_chromeos_clang': ['compile'], | |
| 346 # Note: It is a Release builder even if its name convey otherwise. | |
| 347 'linux_chromeos': standard_tests + [ | |
| 348 'app_list_unittests', | |
| 349 'aura_unittests', | |
| 350 'ash_unittests', | |
| 351 'chromeos_unittests', | |
| 352 'components_unittests', | |
| 353 'dbus_unittests', | |
| 354 'device_unittests', | |
| 355 'events_unittests', | |
| 356 'google_apis_unittests', | |
| 357 'sandbox_linux_unittests', | |
| 358 ], | |
| 359 'linux_rel': standard_tests + [ | |
| 360 'cc_unittests', | |
| 361 'chromedriver_unittests', | |
| 362 'components_unittests', | |
| 363 'google_apis_unittests', | |
| 364 'nacl_integration', | |
| 365 'remoting_unittests', | |
| 366 'sandbox_linux_unittests', | |
| 367 'sync_integration_tests', | |
| 368 'telemetry_perf_unittests', | |
| 369 'telemetry_unittests', | |
| 370 ], | |
| 371 'mac': ['compile'], | |
| 372 'mac_rel': standard_tests + [ | |
| 373 'app_list_unittests', | |
| 374 'cc_unittests', | |
| 375 'chromedriver_unittests', | |
| 376 'components_unittests', | |
| 377 'google_apis_unittests', | |
| 378 'message_center_unittests', | |
| 379 'nacl_integration', | |
| 380 'remoting_unittests', | |
| 381 'sync_integration_tests', | |
| 382 'telemetry_perf_unittests', | |
| 383 'telemetry_unittests', | |
| 384 ], | |
| 385 'win': ['compile'], | |
| 386 'win_rel': standard_tests + [ | |
| 387 'app_list_unittests', | |
| 388 'ash_unittests', | |
| 389 'aura_unittests', | |
| 390 'cc_unittests', | |
| 391 'chrome_elf_unittests', | |
| 392 'chromedriver_unittests', | |
| 393 'components_unittests', | |
| 394 'compositor_unittests', | |
| 395 'events_unittests', | |
| 396 'google_apis_unittests', | |
| 397 'installer_util_unittests', | |
| 398 'mini_installer_test', | |
| 399 'nacl_integration', | |
| 400 'remoting_unittests', | |
| 401 'sync_integration_tests', | |
| 402 'telemetry_perf_unittests', | |
| 403 'telemetry_unittests', | |
| 404 'views_unittests', | |
| 405 ], | |
| 406 'win_x64_rel': [ | |
| 407 'base_unittests', | |
| 408 ], | |
| 409 } | |
| 410 | |
| 411 swarm_enabled_tests = ( | |
| 412 'base_unittests', | |
| 413 'browser_tests', | |
| 414 'interactive_ui_tests', | |
| 415 'net_unittests', | |
| 416 'unit_tests', | |
| 417 ) | |
| 418 | |
| 419 # pylint: disable=W0612 | |
| 420 swarm_test_map = dict( | |
| 421 (test, test + '_swarm') for test in swarm_enabled_tests) | |
| 422 | |
| 423 # Commenting out the items below will make the CQ not use swarm for its | |
| 424 # execution. Uncomment to make the CQ use Swarming again. | |
| 425 swarm_enabled_builders_and_tests = { | |
| 426 ('linux_rel', 'linux_swarm_triggered'): swarm_test_map, | |
| 427 ('mac_rel', 'mac_swarm_triggered'): swarm_test_map, | |
| 428 ('win_rel', 'win_swarm_triggered'): swarm_test_map, | |
| 429 } | |
| 430 | |
| 431 step_verifiers += [ | |
| 432 try_job_steps.TryJobSteps( | |
| 433 builder_name=b, prereq_builder=prereq_builder, | |
| 434 prereq_tests=prereq_tests, steps=s) | |
| 435 for b, s in builders_and_tests.iteritems() | |
| 436 if b not in swarm_enabled_builders_and_tests | |
| 437 ] + [ | |
| 438 try_job_steps.TryJobTriggeredSteps( | |
| 439 builder_name='android_dbg_triggered_tests', | |
| 440 trigger_name='android_dbg', | |
| 441 prereq_builder=prereq_builder, | |
| 442 prereq_tests=prereq_tests, | |
| 443 steps={'slave_steps': 'slave_steps'}), | |
| 444 ] | |
| 445 | |
| 446 # Add the swarm enabled builders with swarm accepted tests. | |
| 447 for (builder, triggered), builder_swarm_enabled_tests in ( | |
| 448 swarm_enabled_builders_and_tests.iteritems()): | |
| 449 regular_tests = list(set(builders_and_tests[builder]) - | |
| 450 set(builder_swarm_enabled_tests)) | |
| 451 | |
| 452 step_verifiers.append( | |
| 453 try_job_steps.TryJobTriggeredOrNormalSteps( | |
| 454 builder_name=triggered, | |
| 455 trigger_name=builder, | |
| 456 prereq_builder=prereq_builder, | |
| 457 prereq_tests=prereq_tests, | |
| 458 steps=builder_swarm_enabled_tests, | |
| 459 trigger_bot_steps=regular_tests, | |
| 460 use_triggered_bot=False)) | |
| 461 | |
| 462 # Experimental recipe-based Chromium trybots. To avoid possible capacity | |
| 463 # problems, only enable for a small percentage of try runs. | |
| 464 # | |
| 465 # Note how we pass revision=None . In presence of safesync_url | |
| 466 # this makes a difference: no revision uses safesync_url, | |
| 467 # while HEAD ignores it and always fetches latest revision. | |
| 468 verifiers.append( | |
| 469 trigger_experimental_try_job.TriggerExperimentalTryJobVerifier( | |
| 470 context_obj, | |
| 471 percentage=0.25, | |
| 472 revision=None, | |
| 473 try_job_description={ | |
| 474 'linux_chromium_rel': ['defaulttests'], | |
| 475 'mac_chromium_rel': ['defaulttests'], | |
| 476 })) | |
| 477 | |
| 478 verifiers.append(try_job_on_rietveld.TryRunnerRietveld( | |
| 479 context_obj, | |
| 480 'http://build.chromium.org/p/tryserver.chromium/', | |
| 481 user, | |
| 482 step_verifiers, | |
| 483 IGNORED_STEPS, | |
| 484 'src')) | |
| 485 | |
| 486 verifiers.append(tree_status.TreeStatusVerifier( | |
| 487 'https://chromium-status.appspot.com')) | |
| 488 return pending_manager.PendingManager( | |
| 489 context_obj, | |
| 490 verifiers_no_patch, | |
| 491 verifiers) | |
| 492 | |
| 493 | |
| 494 def _skia_status_pwd(root_dir): | |
| 495 filepath = os.path.join(root_dir, '.skia_status_pwd') | |
| 496 return _read_lines(filepath, 'skia-status password')[0].strip() | |
| 497 | |
| 498 | |
| 499 def _gen_skia(user, root_dir, rietveld_obj, no_try): | |
| 500 """Generates a PendingManager commit queue for Skia. | |
| 501 | |
| 502 Adds the following verifiers to the PendingManager: | |
| 503 * ProjectBaseUrlVerifier | |
| 504 * ReviewerLgtmVerifier | |
| 505 * PresubmitCheckVerifier | |
| 506 * TreeStatusVerifier | |
| 507 * TryRunnerRietveld (runs compile trybots) | |
| 508 """ | |
| 509 naked_url = '://skia.googlecode.com/svn/trunk' | |
| 510 local_checkout = checkout.SvnCheckout( | |
| 511 root_dir, | |
| 512 'skia', | |
| 513 user, | |
| 514 None, | |
| 515 'https' + naked_url) | |
| 516 context_obj = context.Context( | |
| 517 rietveld_obj, | |
| 518 local_checkout, | |
| 519 async_push.AsyncPush( | |
| 520 'https://skia-tree-status.appspot.com/cq', | |
| 521 _skia_status_pwd(root_dir)), | |
| 522 server_hooks_missing=True) | |
| 523 | |
| 524 project_bases = [ | |
| 525 '^%s(|/.*)$' % re.escape(base + naked_url) for base in ('http', 'https') | |
| 526 ] | |
| 527 project_bases.append( | |
| 528 r'^https?\:\/\/skia.googlesource.com\/skia(?:\.git)?%s$' % | |
| 529 BRANCH_MATCH) | |
| 530 verifiers_no_patch = [ | |
| 531 project_base.ProjectBaseUrlVerifier(project_bases), | |
| 532 reviewer_lgtm.ReviewerLgtmVerifier( | |
| 533 _get_skia_committers(), | |
| 534 [re.escape(user)]), | |
| 535 ] | |
| 536 verifiers = [ | |
| 537 presubmit_check.PresubmitCheckVerifier(context_obj), | |
| 538 tree_status.TreeStatusVerifier( | |
| 539 'https://skia-tree-status.appspot.com') | |
| 540 ] | |
| 541 | |
| 542 if not no_try: | |
| 543 # TODO(skia-infrastructure-team): Use skia.org instead of the below when it | |
| 544 # is ready. | |
| 545 try_server_url = 'http://108.170.219.164:10117/' | |
| 546 | |
| 547 # Get the required build steps and builder names from the try server. | |
| 548 compile_required_build_steps = json.load( | |
| 549 urllib2.urlopen( | |
| 550 try_server_url + 'json/cq_required_steps'))['cq_required_steps'] | |
| 551 builder_names = list( | |
| 552 json.load(urllib2.urlopen(try_server_url + 'json/cqtrybots'))) | |
| 553 | |
| 554 step_verifiers = [] | |
| 555 for builder_name in builder_names: | |
| 556 step_verifiers.append( | |
| 557 try_job_steps.TryJobSteps( | |
| 558 builder_name=builder_name, | |
| 559 steps=compile_required_build_steps)) | |
| 560 verifiers.append(try_job_on_rietveld.TryRunnerRietveld( | |
| 561 context_obj=context_obj, | |
| 562 try_server_url=try_server_url, | |
| 563 commit_user=user, | |
| 564 step_verifiers=step_verifiers, | |
| 565 ignored_steps=[], | |
| 566 solution='src')) | |
| 567 | |
| 568 return pending_manager.PendingManager( | |
| 569 context_obj, | |
| 570 verifiers_no_patch, | |
| 571 verifiers) | |
| 572 | |
| 573 | |
| 574 def _gen_nacl(user, root_dir, rietveld_obj, no_try): | |
| 575 """Generates a PendingManager commit queue for Native Client.""" | |
| 576 offset = 'trunk/src/native_client' | |
| 577 local_checkout = checkout.SvnCheckout( | |
| 578 root_dir, | |
| 579 'nacl', | |
| 580 user, | |
| 581 None, | |
| 582 'svn://svn.chromium.org/native_client/' + offset) | |
| 583 context_obj = context.Context( | |
| 584 rietveld_obj, | |
| 585 local_checkout, | |
| 586 async_push.AsyncPush( | |
| 587 'https://nativeclient-status.appspot.com/cq', | |
| 588 _chromium_status_pwd(root_dir))) | |
| 589 | |
| 590 host_aliases = SVN_HOST_ALIASES + [ | |
| 591 'http://src.chromium.org', 'https://src.chromium.org'] | |
| 592 svn_bases = [i + '/native_client' for i in host_aliases] | |
| 593 project_bases = [ | |
| 594 '^%s/%s(|/.*)$' % (re.escape(base), offset) for base in svn_bases | |
| 595 ] | |
| 596 aliases = ( | |
| 597 'git.chromium.org/native_client/src/native_client', | |
| 598 'chromium.googlesource.com/native_client/src/native_client', | |
| 599 ) | |
| 600 project_bases.extend( | |
| 601 r'^https?\:\/\/%s(?:\.git)?%s$' % (re.escape(i), BRANCH_MATCH) | |
| 602 for i in aliases) | |
| 603 verifiers_no_patch = [ | |
| 604 project_base.ProjectBaseUrlVerifier(project_bases), | |
| 605 reviewer_lgtm.ReviewerLgtmVerifier( | |
| 606 _get_nacl_committers(), | |
| 607 [re.escape(user)]), | |
| 608 ] | |
| 609 verifiers = [ | |
| 610 presubmit_check.PresubmitCheckVerifier(context_obj), | |
| 611 ] | |
| 612 if not no_try: | |
| 613 # Grab the list of all the builders here. The commit queue needs to know | |
| 614 # which builders were triggered. TODO: makes this more automatic. | |
| 615 url = 'http://build.chromium.org/p/tryserver.nacl/json/builders' | |
| 616 builders_and_tests = dict( | |
| 617 (key, []) for key in json.load(urllib2.urlopen(url)) | |
| 618 if (key.startswith('nacl-') and | |
| 619 'toolchain' not in key and | |
| 620 'valgrind' not in key and | |
| 621 'perf_panda' not in key and | |
| 622 'arm_hw' not in key and | |
| 623 'shared' not in key and | |
| 624 'coverage' not in key) | |
| 625 ) | |
| 626 verifiers.append(try_server.TryRunnerSvn( | |
| 627 context_obj, | |
| 628 'http://build.chromium.org/p/tryserver.nacl/', | |
| 629 user, | |
| 630 builders_and_tests, | |
| 631 IGNORED_STEPS, | |
| 632 'native_client', | |
| 633 ['--root', 'native_client'], | |
| 634 _nacl_lkgr)) | |
| 635 | |
| 636 verifiers.append(tree_status.TreeStatusVerifier( | |
| 637 'https://nativeclient-status.appspot.com')) | |
| 638 return pending_manager.PendingManager( | |
| 639 context_obj, | |
| 640 verifiers_no_patch, | |
| 641 verifiers) | |
| 642 | |
| 643 | |
| 644 def _gen_gyp(user, root_dir, rietveld_obj, no_try): | |
| 645 """Generates a PendingManager commit queue for GYP.""" | |
| 646 naked_url = '://gyp.googlecode.com/svn/trunk' | |
| 647 local_checkout = checkout.SvnCheckout( | |
| 648 root_dir, | |
| 649 'gyp', | |
| 650 user, | |
| 651 None, | |
| 652 'https' + naked_url) | |
| 653 context_obj = context.Context( | |
| 654 rietveld_obj, | |
| 655 local_checkout, | |
| 656 async_push.AsyncPush( | |
| 657 'https://chromium-status.appspot.com/cq/receiver', | |
| 658 _chromium_status_pwd(root_dir))) | |
| 659 | |
| 660 project_bases = [ | |
| 661 '^%s(|/.*)$' % re.escape(base + naked_url) for base in ('http', 'https') | |
| 662 ] | |
| 663 verifiers_no_patch = [ | |
| 664 project_base.ProjectBaseUrlVerifier(project_bases), | |
| 665 reviewer_lgtm.ReviewerLgtmVerifier( | |
| 666 _get_gyp_committers(), | |
| 667 [re.escape(user)]), | |
| 668 ] | |
| 669 verifiers = [] | |
| 670 if not no_try: | |
| 671 # Grab the list of all the builders here. The commit queue needs to know | |
| 672 # which builders were triggered. TODO: makes this more automatic. | |
| 673 # GYP is using the Nacl try server. | |
| 674 url = 'http://build.chromium.org/p/tryserver.nacl/json/builders' | |
| 675 builders_and_tests = dict( | |
| 676 (key, []) for key in json.load(urllib2.urlopen(url)) | |
| 677 if key.startswith('gyp-') | |
| 678 ) | |
| 679 verifiers.append(try_server.TryRunnerSvn( | |
| 680 context_obj, | |
| 681 'http://build.chromium.org/p/tryserver.nacl/', | |
| 682 user, | |
| 683 builders_and_tests, | |
| 684 IGNORED_STEPS, | |
| 685 'gyp', | |
| 686 ['--root', 'gyp'], | |
| 687 lambda: None)) | |
| 688 | |
| 689 verifiers.append(tree_status.TreeStatusVerifier( | |
| 690 'https://gyp-status.appspot.com/status')) | |
| 691 return pending_manager.PendingManager( | |
| 692 context_obj, | |
| 693 verifiers_no_patch, | |
| 694 verifiers) | |
| 695 | |
| 696 | |
| 697 def _gen_tools(user, root_dir, rietveld_obj, _no_try): | |
| 698 """Generates a PendingManager commit queue for everything under | |
| 699 chrome/trunk/tools. | |
| 700 | |
| 701 These don't have a try server but have presubmit checks. | |
| 702 """ | |
| 703 # Ignore no_try. | |
| 704 path = 'tools' | |
| 705 project_bases = [ | |
| 706 '^%s/trunk/%s(|/.*)$' % (re.escape(base), path) | |
| 707 for base in CHROME_SVN_BASES | |
| 708 ] | |
| 709 aliases = ( | |
| 710 # Old path. | |
| 711 'git.chromium.org/git/chromium/tools', | |
| 712 # New path. | |
| 713 'git.chromium.org/chromium/tools', | |
| 714 'chromium.googlesource.com/chromium/tools', | |
| 715 'chromium.googlesource.com/a/chromium/tools', | |
| 716 ) | |
| 717 project_bases.extend( | |
| 718 r'^https?\:\/\/%s\/([a-z0-9\-_]+)(?:\.git)?%s$' % ( | |
| 719 re.escape(i), BRANCH_MATCH) for i in aliases) | |
| 720 return _internal_simple(path, project_bases, user, root_dir, rietveld_obj) | |
| 721 | |
| 722 | |
| 723 def _gen_chromium_deps(user, root_dir, rietveld_obj, _no_try): | |
| 724 """Generates a PendingManager commit queue for | |
| 725 chrome/trunk/deps/. | |
| 726 """ | |
| 727 # Ignore no_try. | |
| 728 path = 'deps' | |
| 729 project_bases = [ | |
| 730 '^%s/trunk/%s(|/.*)$' % (re.escape(base), path) | |
| 731 for base in CHROME_SVN_BASES | |
| 732 ] | |
| 733 return _internal_simple(path, project_bases, user, root_dir, rietveld_obj) | |
| 734 | |
| 735 | |
| 736 def _internal_simple(path, project_bases, user, root_dir, rietveld_obj): | |
| 737 """Generates a PendingManager commit queue for chrome/trunk/tools/build.""" | |
| 738 local_checkout = checkout.SvnCheckout( | |
| 739 root_dir, | |
| 740 os.path.basename(path), | |
| 741 user, | |
| 742 None, | |
| 743 'svn://svn.chromium.org/chrome/trunk/' + path, | |
| 744 [chromium_copyright.process]) | |
| 745 context_obj = context.Context( | |
| 746 rietveld_obj, | |
| 747 local_checkout, | |
| 748 async_push.AsyncPush( | |
| 749 'https://chromium-status.appspot.com/cq', | |
| 750 _chromium_status_pwd(root_dir))) | |
| 751 | |
| 752 verifiers_no_patch = [ | |
| 753 project_base.ProjectBaseUrlVerifier(project_bases), | |
| 754 reviewer_lgtm.ReviewerLgtmVerifier( | |
| 755 _get_chromium_committers(), | |
| 756 [re.escape(user)]), | |
| 757 ] | |
| 758 verifiers = [ | |
| 759 presubmit_check.PresubmitCheckVerifier(context_obj, timeout=900), | |
| 760 ] | |
| 761 | |
| 762 return pending_manager.PendingManager( | |
| 763 context_obj, | |
| 764 verifiers_no_patch, | |
| 765 verifiers) | |
| 766 | |
| 767 | |
| 768 def supported_projects(): | |
| 769 """List the projects that can be managed by the commit queue.""" | |
| 770 return sorted( | |
| 771 x[5:] for x in dir(sys.modules[__name__]) if x.startswith('_gen_')) | |
| 772 | |
| 773 | |
| 774 def load_project(project, user, root_dir, rietveld_obj, no_try): | |
| 775 """Loads the specified project.""" | |
| 776 assert os.path.isabs(root_dir) | |
| 777 return getattr(sys.modules[__name__], '_gen_' + project)( | |
| 778 user, root_dir, rietveld_obj, no_try) | |
| OLD | NEW |