OLD | NEW |
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 | 5 |
6 """Miscellaneous utilities needed by the Skia buildbot master.""" | 6 """Miscellaneous utilities needed by the Skia buildbot master.""" |
7 | 7 |
8 | 8 |
9 import httplib2 | 9 import httplib2 |
10 import re | 10 import re |
(...skipping 16 matching lines...) Expand all Loading... |
27 from skia_master_scripts import chromeos_factory | 27 from skia_master_scripts import chromeos_factory |
28 from skia_master_scripts import factory as skia_factory | 28 from skia_master_scripts import factory as skia_factory |
29 from skia_master_scripts import housekeeping_percommit_factory, \ | 29 from skia_master_scripts import housekeeping_percommit_factory, \ |
30 housekeeping_periodic_factory | 30 housekeeping_periodic_factory |
31 from skia_master_scripts import ios_factory | 31 from skia_master_scripts import ios_factory |
32 from skia_master_scripts import nacl_factory | 32 from skia_master_scripts import nacl_factory |
33 | 33 |
34 import config_private | 34 import config_private |
35 | 35 |
36 | 36 |
| 37 BUILDER_NAME_SEP = '-' |
37 CATEGORY_BUILD = ' Build' | 38 CATEGORY_BUILD = ' Build' |
38 TRYBOT_NAME_SUFFIX = '_Trybot' | 39 TRYBOT_NAME_SUFFIX = 'Trybot' |
39 TRY_SCHEDULER_SVN = 'skia_try_svn' | 40 TRY_SCHEDULER_SVN = 'skia_try_svn' |
40 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' | 41 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' |
41 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] | 42 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] |
42 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) | 43 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) |
43 | 44 |
44 | 45 |
45 def IsTrybot(builder_name): | 46 def IsTrybot(builder_name): |
46 return builder_name.endswith(TRYBOT_NAME_SUFFIX) | 47 return builder_name.endswith(TRYBOT_NAME_SUFFIX) |
47 | 48 |
48 | 49 |
(...skipping 311 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
360 # been defined. | 361 # been defined. |
361 # pylint: disable=W0601 | 362 # pylint: disable=W0601 |
362 global skia_all_subdirs | 363 global skia_all_subdirs |
363 try: | 364 try: |
364 if skia_all_subdirs: | 365 if skia_all_subdirs: |
365 raise Exception('skia_all_subdirs has already been defined!') | 366 raise Exception('skia_all_subdirs has already been defined!') |
366 except NameError: | 367 except NameError: |
367 skia_all_subdirs = all_subdirs | 368 skia_all_subdirs = all_subdirs |
368 | 369 |
369 | 370 |
370 def MakeBuilderName(builder_base_name, config): | 371 def MakeBuilderName(os, version, arch_width, hardware, role, extra_config=None, |
371 """ Inserts config into builder_base_name at '%s', or if builder_base_name | 372 is_trybot=False): |
372 does not contain '%s', appends config to the end of builder_base_name, | 373 """ Construct a builder name from an OS, version, hardware, configuration, and |
373 separated by an underscore. """ | 374 builder role. """ |
374 try: | 375 default_val = 'x' |
375 return builder_base_name % config | 376 name_parts = [os or default_val, version or default_val, |
376 except TypeError: | 377 hardware or default_val, arch_width or default_val, role] |
377 # If builder_base_name does not contain '%s' | 378 if extra_config: |
378 return '%s_%s' % (builder_base_name, config) | 379 name_parts.append(extra_config) |
| 380 if is_trybot: |
| 381 name_parts.append(TRYBOT_NAME_SUFFIX) |
| 382 for part in name_parts: |
| 383 if BUILDER_NAME_SEP in part: |
| 384 raise ValueError('%s is not legal in %s' % (BUILDER_NAME_SEP, part)) |
| 385 return BUILDER_NAME_SEP.join(name_parts) |
379 | 386 |
380 | 387 |
381 def MakeCompileBuilderName(builder_base_name, release=False): | 388 def MakeCompileBuilderName(release=False, **kwargs): |
382 if release: | 389 if release: |
383 compile_name = 'Compile_Release' | 390 compile_name = 'ReleaseCompile' |
384 else: | 391 else: |
385 compile_name = 'Compile_Debug' | 392 compile_name = 'DebugCompile' |
386 return MakeBuilderName(builder_base_name, compile_name) | 393 return MakeBuilderName(role=compile_name, **kwargs) |
387 | 394 |
388 | 395 |
389 def MakeDebugBuilderName(builder_base_name): | 396 def MakeDebugBuilderName(**kwargs): |
390 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_DEBUG) | 397 return MakeBuilderName(role=skia_factory.CONFIG_DEBUG, **kwargs) |
391 | 398 |
392 | 399 |
393 def MakeReleaseBuilderName(builder_base_name): | 400 def MakeReleaseBuilderName(**kwargs): |
394 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_RELEASE) | 401 return MakeBuilderName(role=skia_factory.CONFIG_RELEASE, **kwargs) |
395 | 402 |
396 | 403 |
397 def MakeBenchBuilderName(builder_base_name): | 404 def MakeBenchBuilderName(**kwargs): |
398 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_BENCH) | 405 return MakeBuilderName(role=skia_factory.CONFIG_BENCH, **kwargs) |
399 | 406 |
400 | 407 |
401 def MakeSchedulerName(builder_base_name): | 408 def MakeSchedulerName(**kwargs): |
402 return MakeBuilderName(builder_base_name, 'Scheduler') | 409 return MakeBuilderName(role='Scheduler', **kwargs) |
403 | 410 |
404 | 411 |
405 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, | 412 def _MakeBuilderSet(helper, os, version, arch_width, hardware, |
| 413 gm_image_subdir, extra_config=None, |
406 perf_output_basedir=None, extra_branches=None, | 414 perf_output_basedir=None, extra_branches=None, |
407 factory_type=None, do_compile=True, do_debug=True, | 415 factory_type=None, do_compile=True, do_debug=True, |
408 do_release=True, do_bench=True, try_schedulers=None, | 416 do_release=True, do_bench=True, try_schedulers=None, |
409 compile_bot_warnings_as_errors=True, | 417 compile_bot_warnings_as_errors=True, |
410 **kwargs): | 418 **kwargs): |
411 """ Creates a trio of builders for a given platform: | 419 """ Creates a trio of builders for a given platform: |
412 1. Debug mode builder which runs all steps | 420 1. Debug mode builder which runs all steps |
413 2. Release mode builder which runs all steps EXCEPT benchmarks | 421 2. Release mode builder which runs all steps EXCEPT benchmarks |
414 3. Release mode builder which runs ONLY benchmarks. | 422 3. Release mode builder which runs ONLY benchmarks. |
415 """ | 423 """ |
416 B = helper.Builder | 424 B = helper.Builder |
417 F = helper.Factory | 425 F = helper.Factory |
418 | 426 |
419 if not extra_branches: | 427 if not extra_branches: |
420 extra_branches = [] | 428 extra_branches = [] |
421 subdirs_to_checkout = set(extra_branches) | 429 subdirs_to_checkout = set(extra_branches) |
| 430 is_trybot = try_schedulers is not None |
422 if gm_image_subdir: | 431 if gm_image_subdir: |
423 gm_image_branch = 'gm-expected/%s' % gm_image_subdir | 432 gm_image_branch = 'gm-expected/%s' % gm_image_subdir |
424 subdirs_to_checkout.add(gm_image_branch) | 433 subdirs_to_checkout.add(gm_image_branch) |
425 | |
426 if try_schedulers: | 434 if try_schedulers: |
427 scheduler_name = '|'.join(try_schedulers) | 435 scheduler_name = '|'.join(try_schedulers) |
428 builder_base_name = builder_base_name + TRYBOT_NAME_SUFFIX | |
429 else: | 436 else: |
430 scheduler_name = MakeSchedulerName(builder_base_name) | 437 scheduler_name = MakeSchedulerName(os=os, |
| 438 version=version, |
| 439 arch_width=arch_width, |
| 440 hardware=hardware, |
| 441 extra_config=extra_config) |
431 branches = list(subdirs_to_checkout.union(SKIA_PRIMARY_SUBDIRS)) | 442 branches = list(subdirs_to_checkout.union(SKIA_PRIMARY_SUBDIRS)) |
432 helper.AnyBranchScheduler(scheduler_name, branches=branches) | 443 helper.AnyBranchScheduler(scheduler_name, branches=branches) |
433 | 444 |
434 if do_compile: | 445 if do_compile: |
435 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name, | 446 compile_debug_builder_name = MakeCompileBuilderName( |
436 release=False) | 447 os=os, |
| 448 version=version, |
| 449 arch_width=arch_width, |
| 450 hardware=hardware, |
| 451 extra_config=extra_config, |
| 452 is_trybot=is_trybot, |
| 453 release=False) |
437 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, | 454 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, |
438 # Do not add gatekeeper for trybots. | 455 # Do not add gatekeeper for trybots. |
439 gatekeeper='GateKeeper' if try_schedulers is None else None, | 456 gatekeeper='GateKeeper' if try_schedulers is None else None, |
440 scheduler=scheduler_name, override_category=CATEGORY_BUILD) | 457 scheduler=scheduler_name, override_category=CATEGORY_BUILD) |
441 F('f_%s' % compile_debug_builder_name, factory_type( | 458 F('f_%s' % compile_debug_builder_name, factory_type( |
442 builder_name=compile_debug_builder_name, | 459 builder_name=compile_debug_builder_name, |
443 other_subdirs=subdirs_to_checkout, | 460 other_subdirs=subdirs_to_checkout, |
444 configuration=skia_factory.CONFIG_DEBUG, | 461 configuration=skia_factory.CONFIG_DEBUG, |
445 gm_image_subdir=gm_image_subdir, | 462 gm_image_subdir=gm_image_subdir, |
446 do_patch_step=(try_schedulers is not None), | 463 do_patch_step=is_trybot, |
447 perf_output_basedir=None, | 464 perf_output_basedir=None, |
448 compile_warnings_as_errors=compile_bot_warnings_as_errors, | 465 compile_warnings_as_errors=compile_bot_warnings_as_errors, |
449 **kwargs | 466 **kwargs |
450 ).BuildCompileOnly()) | 467 ).BuildCompileOnly()) |
451 compile_release_builder_name = MakeCompileBuilderName(builder_base_name, | 468 compile_release_builder_name = MakeCompileBuilderName( |
452 release=True) | 469 os=os, |
| 470 version=version, |
| 471 arch_width=arch_width, |
| 472 hardware=hardware, |
| 473 extra_config=extra_config, |
| 474 is_trybot=is_trybot, |
| 475 release=True) |
453 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, | 476 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, |
454 # Do not add gatekeeper for trybots. | 477 # Do not add gatekeeper for trybots. |
455 gatekeeper='GateKeeper' if try_schedulers is None else None, | 478 gatekeeper='GateKeeper' if try_schedulers is None else None, |
456 scheduler=scheduler_name, override_category=CATEGORY_BUILD) | 479 scheduler=scheduler_name, override_category=CATEGORY_BUILD) |
457 F('f_%s' % compile_release_builder_name, factory_type( | 480 F('f_%s' % compile_release_builder_name, factory_type( |
458 builder_name=compile_release_builder_name, | 481 builder_name=compile_release_builder_name, |
459 other_subdirs=subdirs_to_checkout, | 482 other_subdirs=subdirs_to_checkout, |
460 configuration=skia_factory.CONFIG_RELEASE, | 483 configuration=skia_factory.CONFIG_RELEASE, |
461 gm_image_subdir=gm_image_subdir, | 484 gm_image_subdir=gm_image_subdir, |
462 do_patch_step=(try_schedulers is not None), | 485 do_patch_step=is_trybot, |
463 perf_output_basedir=None, | 486 perf_output_basedir=None, |
464 compile_warnings_as_errors=compile_bot_warnings_as_errors, | 487 compile_warnings_as_errors=compile_bot_warnings_as_errors, |
465 **kwargs | 488 **kwargs |
466 ).BuildCompileOnly()) | 489 ).BuildCompileOnly()) |
467 | 490 |
468 if do_debug: | 491 if do_debug: |
469 debug_builder_name = MakeDebugBuilderName(builder_base_name) | 492 debug_builder_name = MakeDebugBuilderName(os=os, |
| 493 version=version, |
| 494 arch_width=arch_width, |
| 495 hardware=hardware, |
| 496 extra_config=extra_config, |
| 497 is_trybot=is_trybot) |
470 B(debug_builder_name, 'f_%s' % debug_builder_name, | 498 B(debug_builder_name, 'f_%s' % debug_builder_name, |
471 scheduler=scheduler_name) | 499 scheduler=scheduler_name) |
472 F('f_%s' % debug_builder_name, factory_type( | 500 F('f_%s' % debug_builder_name, factory_type( |
473 builder_name=debug_builder_name, | 501 builder_name=debug_builder_name, |
474 other_subdirs=subdirs_to_checkout, | 502 other_subdirs=subdirs_to_checkout, |
475 configuration=skia_factory.CONFIG_DEBUG, | 503 configuration=skia_factory.CONFIG_DEBUG, |
476 gm_image_subdir=gm_image_subdir, | 504 gm_image_subdir=gm_image_subdir, |
477 do_patch_step=(try_schedulers is not None), | 505 do_patch_step=is_trybot, |
478 perf_output_basedir=None, | 506 perf_output_basedir=None, |
479 compile_warnings_as_errors=False, | 507 compile_warnings_as_errors=False, |
480 **kwargs | 508 **kwargs |
481 ).Build()) | 509 ).Build()) |
482 | 510 |
483 if do_release: | 511 if do_release: |
484 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name) | 512 no_perf_builder_name = MakeReleaseBuilderName(os=os, |
| 513 version=version, |
| 514 arch_width=arch_width, |
| 515 hardware=hardware, |
| 516 extra_config=extra_config, |
| 517 is_trybot=is_trybot) |
485 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, | 518 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, |
486 scheduler=scheduler_name) | 519 scheduler=scheduler_name) |
487 F('f_%s' % no_perf_builder_name, factory_type( | 520 F('f_%s' % no_perf_builder_name, factory_type( |
488 builder_name=no_perf_builder_name, | 521 builder_name=no_perf_builder_name, |
489 other_subdirs=subdirs_to_checkout, | 522 other_subdirs=subdirs_to_checkout, |
490 configuration=skia_factory.CONFIG_RELEASE, | 523 configuration=skia_factory.CONFIG_RELEASE, |
491 gm_image_subdir=gm_image_subdir, | 524 gm_image_subdir=gm_image_subdir, |
492 do_patch_step=(try_schedulers is not None), | 525 do_patch_step=is_trybot, |
493 perf_output_basedir=None, | 526 perf_output_basedir=None, |
494 compile_warnings_as_errors=False, | 527 compile_warnings_as_errors=False, |
495 **kwargs | 528 **kwargs |
496 ).BuildNoPerf()) | 529 ).BuildNoPerf()) |
497 | 530 |
498 if do_bench: | 531 if do_bench: |
499 perf_builder_name = MakeBenchBuilderName(builder_base_name) | 532 perf_builder_name = MakeBenchBuilderName(os=os, |
| 533 version=version, |
| 534 arch_width=arch_width, |
| 535 hardware=hardware, |
| 536 extra_config=extra_config, |
| 537 is_trybot=is_trybot) |
500 B(perf_builder_name, 'f_%s' % perf_builder_name, | 538 B(perf_builder_name, 'f_%s' % perf_builder_name, |
501 scheduler=scheduler_name) | 539 scheduler=scheduler_name) |
502 F('f_%s' % perf_builder_name, factory_type( | 540 F('f_%s' % perf_builder_name, factory_type( |
503 builder_name=perf_builder_name, | 541 builder_name=perf_builder_name, |
504 other_subdirs=subdirs_to_checkout, | 542 other_subdirs=subdirs_to_checkout, |
505 configuration=skia_factory.CONFIG_RELEASE, | 543 configuration=skia_factory.CONFIG_RELEASE, |
506 gm_image_subdir=gm_image_subdir, | 544 gm_image_subdir=gm_image_subdir, |
507 do_patch_step=(try_schedulers is not None), | 545 do_patch_step=is_trybot, |
508 perf_output_basedir=perf_output_basedir, | 546 perf_output_basedir=perf_output_basedir, |
509 compile_warnings_as_errors=False, | 547 compile_warnings_as_errors=False, |
510 **kwargs | 548 **kwargs |
511 ).BuildPerfOnly()) | 549 ).BuildPerfOnly()) |
512 | 550 |
513 | 551 |
514 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): | 552 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): |
515 _MakeBuilderSet(try_schedulers=None, **kwargs) | 553 _MakeBuilderSet(try_schedulers=None, **kwargs) |
516 if do_trybots: | 554 if do_trybots: |
517 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) | 555 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) |
518 | 556 |
519 | 557 |
520 def MakeBuilderSet(**kwargs): | 558 def MakeBuilderSet(**kwargs): |
521 _MakeBuilderAndMaybeTrybotSet(factory_type=skia_factory.SkiaFactory, **kwargs) | 559 _MakeBuilderAndMaybeTrybotSet(factory_type=skia_factory.SkiaFactory, **kwargs) |
522 | 560 |
523 | 561 |
524 def MakeHousekeeperBuilderSet(helper, do_trybots, do_upload_results): | 562 def MakeHousekeeperBuilderSet(helper, do_trybots, do_upload_results): |
525 B = helper.Builder | 563 B = helper.Builder |
526 F = helper.Factory | 564 F = helper.Factory |
527 | 565 |
528 builder_factory_scheduler = [ | 566 builder_factory_scheduler = [ |
529 # The Percommit housekeeper | 567 # The Percommit housekeeper |
530 ('Skia_PerCommit_House_Keeping', | 568 ('PerCommit-Housekeeping', |
531 housekeeping_percommit_factory.HouseKeepingPerCommitFactory, | 569 housekeeping_percommit_factory.HouseKeepingPerCommitFactory, |
532 'skia_rel'), | 570 'skia_rel'), |
533 # The Periodic housekeeper | 571 # The Periodic housekeeper |
534 ('Skia_Periodic_House_Keeping', | 572 ('Periodic-Housekeeping', |
535 housekeeping_periodic_factory.HouseKeepingPeriodicFactory, | 573 housekeeping_periodic_factory.HouseKeepingPeriodicFactory, |
536 'skia_periodic'), | 574 'skia_periodic'), |
537 ] | 575 ] |
538 if do_trybots: | 576 if do_trybots: |
539 # Add the corresponding trybot builders to the above list. | 577 # Add the corresponding trybot builders to the above list. |
540 builder_factory_scheduler.extend([ | 578 builder_factory_scheduler.extend([ |
541 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR) | 579 (builder + BUILDER_NAME_SEP + TRYBOT_NAME_SUFFIX, factory, |
| 580 TRY_SCHEDULERS_STR) |
542 for (builder, factory, _scheduler) in builder_factory_scheduler]) | 581 for (builder, factory, _scheduler) in builder_factory_scheduler]) |
543 | 582 |
544 for (builder_name, factory, scheduler) in builder_factory_scheduler: | 583 for (builder_name, factory, scheduler) in builder_factory_scheduler: |
545 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) | 584 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) |
546 F('f_%s' % builder_name, | 585 F('f_%s' % builder_name, |
547 factory( | 586 factory( |
548 do_upload_results=do_upload_results, | 587 do_upload_results=do_upload_results, |
549 target_platform=skia_factory.TARGET_PLATFORM_LINUX, | 588 target_platform=skia_factory.TARGET_PLATFORM_LINUX, |
550 builder_name=builder_name, | 589 builder_name=builder_name, |
551 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), | 590 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
614 # request is associated with a change but the revisions match (#5 above). | 653 # request is associated with a change but the revisions match (#5 above). |
615 if req1.source.changes and not req2.source.changes: | 654 if req1.source.changes and not req2.source.changes: |
616 return False | 655 return False |
617 if not req1.source.changes and req2.source.changes: | 656 if not req1.source.changes and req2.source.changes: |
618 return False | 657 return False |
619 if not (req1.source.changes and req2.source.changes): | 658 if not (req1.source.changes and req2.source.changes): |
620 if req1.source.revision != req2.source.revision: | 659 if req1.source.revision != req2.source.revision: |
621 return False | 660 return False |
622 | 661 |
623 return True | 662 return True |
OLD | NEW |