OLD | NEW |
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 | 5 |
6 """Miscellaneous utilities needed by the Skia buildbot master.""" | 6 """Miscellaneous utilities needed by the Skia buildbot master.""" |
7 | 7 |
8 | 8 |
9 import httplib2 | 9 import httplib2 |
10 import re | 10 import re |
11 | 11 |
12 # requires Google APIs client library for Python; see | 12 # requires Google APIs client library for Python; see |
13 # https://code.google.com/p/google-api-python-client/wiki/Installation | 13 # https://code.google.com/p/google-api-python-client/wiki/Installation |
14 from apiclient.discovery import build | 14 from apiclient.discovery import build |
15 from buildbot.scheduler import AnyBranchScheduler | 15 from buildbot.scheduler import AnyBranchScheduler |
16 from buildbot.schedulers import timed | 16 from buildbot.schedulers import timed |
17 from buildbot.schedulers.filter import ChangeFilter | 17 from buildbot.schedulers.filter import ChangeFilter |
18 from buildbot.util import NotABranch | 18 from buildbot.util import NotABranch |
19 from config_private import TRY_SVN_BASEURL | 19 from config_private import TRY_SVN_BASEURL |
20 from master import master_config | 20 from master import master_config |
21 from master import try_job_svn | 21 from master import try_job_svn |
22 from master import try_job_rietveld | 22 from master import try_job_rietveld |
23 from master.builders_pools import BuildersPools | 23 from master.builders_pools import BuildersPools |
24 from oauth2client.client import SignedJwtAssertionCredentials | 24 from oauth2client.client import SignedJwtAssertionCredentials |
25 | 25 |
26 from skia_master_scripts import android_factory | |
27 from skia_master_scripts import chromeos_factory | |
28 from skia_master_scripts import factory as skia_factory | |
29 from skia_master_scripts import housekeeping_percommit_factory, \ | |
30 housekeeping_periodic_factory | |
31 from skia_master_scripts import ios_factory | |
32 from skia_master_scripts import nacl_factory | |
33 | |
34 import config_private | 26 import config_private |
35 | 27 |
36 | 28 |
37 CATEGORY_BUILD = ' Build' | 29 CATEGORY_BUILD = ' Build' |
38 TRYBOT_NAME_SUFFIX = '_Trybot' | 30 TRYBOT_NAME_SUFFIX = '_Trybot' |
39 TRY_SCHEDULER_SVN = 'skia_try_svn' | 31 TRY_SCHEDULER_SVN = 'skia_try_svn' |
40 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' | 32 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' |
41 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] | 33 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] |
42 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) | 34 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) |
43 | 35 |
44 | 36 |
45 def IsTrybot(builder_name): | 37 def IsTrybot(builder_name): |
46 return builder_name.endswith(TRYBOT_NAME_SUFFIX) | 38 return builder_name.endswith(TRYBOT_NAME_SUFFIX) |
47 | 39 |
48 | 40 |
| 41 def _IndentStr(indent): |
| 42 return ' ' * (indent + 1) |
| 43 |
| 44 |
| 45 def ToString(obj, indent=0): |
| 46 """ Returns a string representation of the given object. This differs from the |
| 47 built-in string function in that it does not give memory locations. |
| 48 |
| 49 obj: the object to print. |
| 50 indent: integer; the current indent level. |
| 51 """ |
| 52 if isinstance(obj, list): |
| 53 return _ListToString(obj, indent) |
| 54 elif isinstance(obj, dict): |
| 55 return _DictToString(obj, indent) |
| 56 elif isinstance(obj, tuple): |
| 57 return _ListToString(obj, indent) |
| 58 elif isinstance(obj, str): |
| 59 return '\'%s\'' % obj |
| 60 elif obj is None: |
| 61 return 'None' |
| 62 else: |
| 63 return '<Object>' |
| 64 |
| 65 |
| 66 def _ListToString(list_var, indent): |
| 67 if not list_var: |
| 68 return '[]' |
| 69 indent_str = _IndentStr(indent) |
| 70 val = '[\n' |
| 71 indent += 1 |
| 72 val += ''.join(['%s%s,\n' % (indent_str, ToString(elem, indent)) \ |
| 73 for elem in list_var]) |
| 74 indent -= 1 |
| 75 indent_str = _IndentStr(indent - 1) |
| 76 val += indent_str + ']' |
| 77 return val |
| 78 |
| 79 |
| 80 def _DictToString(d, indent): |
| 81 if not d: |
| 82 return '{}' |
| 83 indent_str = _IndentStr(indent) |
| 84 val = '{\n' |
| 85 indent += 1 |
| 86 val += ''.join(['%s%s: %s,\n' % (indent_str, ToString(k, indent), |
| 87 ToString(d[k], indent)) \ |
| 88 for k in sorted(d.keys())]) |
| 89 indent -= 1 |
| 90 indent_str = _IndentStr(indent - 1) |
| 91 val += indent_str + '}' |
| 92 return val |
| 93 |
| 94 |
49 class SkiaChangeFilter(ChangeFilter): | 95 class SkiaChangeFilter(ChangeFilter): |
50 """Skia specific subclass of ChangeFilter.""" | 96 """Skia specific subclass of ChangeFilter.""" |
51 | 97 |
52 def __init__(self, builders, **kwargs): | 98 def __init__(self, builders, **kwargs): |
53 self._builders = builders | 99 self._builders = builders |
54 ChangeFilter.__init__(self, **kwargs) | 100 ChangeFilter.__init__(self, **kwargs) |
55 | 101 |
56 def filter_change(self, change): | 102 def filter_change(self, change): |
57 """Overrides ChangeFilter.filter_change to pass builders to filter_fn. | 103 """Overrides ChangeFilter.filter_change to pass builders to filter_fn. |
58 | 104 |
(...skipping 321 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
380 | 426 |
381 def MakeCompileBuilderName(builder_base_name, release=False): | 427 def MakeCompileBuilderName(builder_base_name, release=False): |
382 if release: | 428 if release: |
383 compile_name = 'Compile_Release' | 429 compile_name = 'Compile_Release' |
384 else: | 430 else: |
385 compile_name = 'Compile_Debug' | 431 compile_name = 'Compile_Debug' |
386 return MakeBuilderName(builder_base_name, compile_name) | 432 return MakeBuilderName(builder_base_name, compile_name) |
387 | 433 |
388 | 434 |
389 def MakeDebugBuilderName(builder_base_name): | 435 def MakeDebugBuilderName(builder_base_name): |
390 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_DEBUG) | 436 return MakeBuilderName(builder_base_name, 'Debug') |
391 | 437 |
392 | 438 |
393 def MakeReleaseBuilderName(builder_base_name): | 439 def MakeReleaseBuilderName(builder_base_name): |
394 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_RELEASE) | 440 return MakeBuilderName(builder_base_name, 'Release') |
395 | 441 |
396 | 442 |
397 def MakeBenchBuilderName(builder_base_name): | 443 def MakeBenchBuilderName(builder_base_name): |
398 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_BENCH) | 444 return MakeBuilderName(builder_base_name, 'Bench') |
399 | 445 |
400 | 446 |
401 def MakeSchedulerName(builder_base_name): | 447 def MakeSchedulerName(builder_base_name): |
402 return MakeBuilderName(builder_base_name, 'Scheduler') | 448 return MakeBuilderName(builder_base_name, 'Scheduler') |
403 | 449 |
404 | 450 |
405 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, | 451 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, factory_type, |
406 perf_output_basedir=None, extra_branches=None, | 452 perf_output_basedir=None, extra_branches=None, |
407 factory_type=None, do_compile=True, do_debug=True, | 453 do_compile=True, do_debug=True, do_release=True, |
408 do_release=True, do_bench=True, try_schedulers=None, | 454 do_bench=True, try_schedulers=None, |
409 compile_bot_warnings_as_errors=True, | 455 compile_bot_warnings_as_errors=True, **kwargs): |
410 **kwargs): | |
411 """ Creates a trio of builders for a given platform: | 456 """ Creates a trio of builders for a given platform: |
412 1. Debug mode builder which runs all steps | 457 1. Debug mode builder which runs all steps |
413 2. Release mode builder which runs all steps EXCEPT benchmarks | 458 2. Release mode builder which runs all steps EXCEPT benchmarks |
414 3. Release mode builder which runs ONLY benchmarks. | 459 3. Release mode builder which runs ONLY benchmarks. |
415 """ | 460 """ |
416 B = helper.Builder | 461 B = helper.Builder |
417 F = helper.Factory | 462 F = helper.Factory |
418 | 463 |
419 if not extra_branches: | 464 if not extra_branches: |
420 extra_branches = [] | 465 extra_branches = [] |
(...skipping 13 matching lines...) Expand all Loading... |
434 if do_compile: | 479 if do_compile: |
435 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name, | 480 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name, |
436 release=False) | 481 release=False) |
437 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, | 482 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, |
438 # Do not add gatekeeper for trybots. | 483 # Do not add gatekeeper for trybots. |
439 gatekeeper='GateKeeper' if try_schedulers is None else None, | 484 gatekeeper='GateKeeper' if try_schedulers is None else None, |
440 scheduler=scheduler_name, override_category=CATEGORY_BUILD) | 485 scheduler=scheduler_name, override_category=CATEGORY_BUILD) |
441 F('f_%s' % compile_debug_builder_name, factory_type( | 486 F('f_%s' % compile_debug_builder_name, factory_type( |
442 builder_name=compile_debug_builder_name, | 487 builder_name=compile_debug_builder_name, |
443 other_subdirs=subdirs_to_checkout, | 488 other_subdirs=subdirs_to_checkout, |
444 configuration=skia_factory.CONFIG_DEBUG, | 489 configuration='Debug', |
445 gm_image_subdir=gm_image_subdir, | 490 gm_image_subdir=gm_image_subdir, |
446 do_patch_step=(try_schedulers is not None), | 491 do_patch_step=(try_schedulers is not None), |
447 perf_output_basedir=None, | 492 perf_output_basedir=None, |
448 compile_warnings_as_errors=compile_bot_warnings_as_errors, | 493 compile_warnings_as_errors=compile_bot_warnings_as_errors, |
449 **kwargs | 494 **kwargs |
450 ).BuildCompileOnly()) | 495 ).BuildCompileOnly()) |
451 compile_release_builder_name = MakeCompileBuilderName(builder_base_name, | 496 compile_release_builder_name = MakeCompileBuilderName(builder_base_name, |
452 release=True) | 497 release=True) |
453 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, | 498 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, |
454 # Do not add gatekeeper for trybots. | 499 # Do not add gatekeeper for trybots. |
455 gatekeeper='GateKeeper' if try_schedulers is None else None, | 500 gatekeeper='GateKeeper' if try_schedulers is None else None, |
456 scheduler=scheduler_name, override_category=CATEGORY_BUILD) | 501 scheduler=scheduler_name, override_category=CATEGORY_BUILD) |
457 F('f_%s' % compile_release_builder_name, factory_type( | 502 F('f_%s' % compile_release_builder_name, factory_type( |
458 builder_name=compile_release_builder_name, | 503 builder_name=compile_release_builder_name, |
459 other_subdirs=subdirs_to_checkout, | 504 other_subdirs=subdirs_to_checkout, |
460 configuration=skia_factory.CONFIG_RELEASE, | 505 configuration='Release', |
461 gm_image_subdir=gm_image_subdir, | 506 gm_image_subdir=gm_image_subdir, |
462 do_patch_step=(try_schedulers is not None), | 507 do_patch_step=(try_schedulers is not None), |
463 perf_output_basedir=None, | 508 perf_output_basedir=None, |
464 compile_warnings_as_errors=compile_bot_warnings_as_errors, | 509 compile_warnings_as_errors=compile_bot_warnings_as_errors, |
465 **kwargs | 510 **kwargs |
466 ).BuildCompileOnly()) | 511 ).BuildCompileOnly()) |
467 | 512 |
468 if do_debug: | 513 if do_debug: |
469 debug_builder_name = MakeDebugBuilderName(builder_base_name) | 514 debug_builder_name = MakeDebugBuilderName(builder_base_name) |
470 B(debug_builder_name, 'f_%s' % debug_builder_name, | 515 B(debug_builder_name, 'f_%s' % debug_builder_name, |
471 scheduler=scheduler_name) | 516 scheduler=scheduler_name) |
472 F('f_%s' % debug_builder_name, factory_type( | 517 F('f_%s' % debug_builder_name, factory_type( |
473 builder_name=debug_builder_name, | 518 builder_name=debug_builder_name, |
474 other_subdirs=subdirs_to_checkout, | 519 other_subdirs=subdirs_to_checkout, |
475 configuration=skia_factory.CONFIG_DEBUG, | 520 configuration='Debug', |
476 gm_image_subdir=gm_image_subdir, | 521 gm_image_subdir=gm_image_subdir, |
477 do_patch_step=(try_schedulers is not None), | 522 do_patch_step=(try_schedulers is not None), |
478 perf_output_basedir=None, | 523 perf_output_basedir=None, |
479 compile_warnings_as_errors=False, | 524 compile_warnings_as_errors=False, |
480 **kwargs | 525 **kwargs |
481 ).Build()) | 526 ).Build()) |
482 | 527 |
483 if do_release: | 528 if do_release: |
484 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name) | 529 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name) |
485 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, | 530 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, |
486 scheduler=scheduler_name) | 531 scheduler=scheduler_name) |
487 F('f_%s' % no_perf_builder_name, factory_type( | 532 F('f_%s' % no_perf_builder_name, factory_type( |
488 builder_name=no_perf_builder_name, | 533 builder_name=no_perf_builder_name, |
489 other_subdirs=subdirs_to_checkout, | 534 other_subdirs=subdirs_to_checkout, |
490 configuration=skia_factory.CONFIG_RELEASE, | 535 configuration='Release', |
491 gm_image_subdir=gm_image_subdir, | 536 gm_image_subdir=gm_image_subdir, |
492 do_patch_step=(try_schedulers is not None), | 537 do_patch_step=(try_schedulers is not None), |
493 perf_output_basedir=None, | 538 perf_output_basedir=None, |
494 compile_warnings_as_errors=False, | 539 compile_warnings_as_errors=False, |
495 **kwargs | 540 **kwargs |
496 ).BuildNoPerf()) | 541 ).BuildNoPerf()) |
497 | 542 |
498 if do_bench: | 543 if do_bench: |
499 perf_builder_name = MakeBenchBuilderName(builder_base_name) | 544 perf_builder_name = MakeBenchBuilderName(builder_base_name) |
500 B(perf_builder_name, 'f_%s' % perf_builder_name, | 545 B(perf_builder_name, 'f_%s' % perf_builder_name, |
501 scheduler=scheduler_name) | 546 scheduler=scheduler_name) |
502 F('f_%s' % perf_builder_name, factory_type( | 547 F('f_%s' % perf_builder_name, factory_type( |
503 builder_name=perf_builder_name, | 548 builder_name=perf_builder_name, |
504 other_subdirs=subdirs_to_checkout, | 549 other_subdirs=subdirs_to_checkout, |
505 configuration=skia_factory.CONFIG_RELEASE, | 550 configuration='Release', |
506 gm_image_subdir=gm_image_subdir, | 551 gm_image_subdir=gm_image_subdir, |
507 do_patch_step=(try_schedulers is not None), | 552 do_patch_step=(try_schedulers is not None), |
508 perf_output_basedir=perf_output_basedir, | 553 perf_output_basedir=perf_output_basedir, |
509 compile_warnings_as_errors=False, | 554 compile_warnings_as_errors=False, |
510 **kwargs | 555 **kwargs |
511 ).BuildPerfOnly()) | 556 ).BuildPerfOnly()) |
512 | 557 |
513 | 558 |
514 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): | 559 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): |
515 _MakeBuilderSet(try_schedulers=None, **kwargs) | 560 _MakeBuilderSet(try_schedulers=None, **kwargs) |
516 if do_trybots: | 561 if do_trybots: |
517 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) | 562 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) |
518 | 563 |
519 | 564 |
520 def MakeBuilderSet(**kwargs): | 565 def MakeBuilderSet(**kwargs): |
521 _MakeBuilderAndMaybeTrybotSet(factory_type=skia_factory.SkiaFactory, **kwargs) | 566 _MakeBuilderAndMaybeTrybotSet(**kwargs) |
522 | 567 |
523 | 568 |
524 def MakeHousekeeperBuilderSet(helper, do_trybots, do_upload_results): | 569 def MakeHousekeeperBuilderSet(helper, percommit_factory_type, |
| 570 periodic_factory_type, do_trybots, **kwargs): |
525 B = helper.Builder | 571 B = helper.Builder |
526 F = helper.Factory | 572 F = helper.Factory |
527 | 573 |
528 builder_factory_scheduler = [ | 574 builder_factory_scheduler = [ |
529 # The Percommit housekeeper | 575 # The Percommit housekeeper |
530 ('Skia_PerCommit_House_Keeping', | 576 ('Skia_PerCommit_House_Keeping', |
531 housekeeping_percommit_factory.HouseKeepingPerCommitFactory, | 577 percommit_factory_type, |
532 'skia_rel'), | 578 'skia_rel'), |
533 # The Periodic housekeeper | 579 # The Periodic housekeeper |
534 ('Skia_Periodic_House_Keeping', | 580 ('Skia_Periodic_House_Keeping', |
535 housekeeping_periodic_factory.HouseKeepingPeriodicFactory, | 581 periodic_factory_type, |
536 'skia_periodic'), | 582 'skia_periodic'), |
537 ] | 583 ] |
538 if do_trybots: | 584 if do_trybots: |
539 # Add the corresponding trybot builders to the above list. | 585 # Add the corresponding trybot builders to the above list. |
540 builder_factory_scheduler.extend([ | 586 builder_factory_scheduler.extend([ |
541 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR) | 587 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR) |
542 for (builder, factory, _scheduler) in builder_factory_scheduler]) | 588 for (builder, factory, _scheduler) in builder_factory_scheduler]) |
543 | 589 |
544 for (builder_name, factory, scheduler) in builder_factory_scheduler: | 590 for (builder_name, factory, scheduler) in builder_factory_scheduler: |
545 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) | 591 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) |
546 F('f_%s' % builder_name, | 592 F('f_%s' % builder_name, |
547 factory( | 593 factory( |
548 do_upload_results=do_upload_results, | |
549 target_platform=skia_factory.TARGET_PLATFORM_LINUX, | |
550 builder_name=builder_name, | 594 builder_name=builder_name, |
551 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), | 595 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), |
552 ).Build()) | 596 ).Build()) |
553 | 597 |
554 | |
555 def MakeAndroidBuilderSet(extra_branches=None, **kwargs): | |
556 if not extra_branches: | |
557 extra_branches = [] | |
558 extra_branches.append('android') | |
559 _MakeBuilderAndMaybeTrybotSet(factory_type=android_factory.AndroidFactory, | |
560 extra_branches=extra_branches, | |
561 **kwargs) | |
562 | |
563 | |
564 def MakeChromeOSBuilderSet(**kwargs): | |
565 _MakeBuilderAndMaybeTrybotSet(factory_type=chromeos_factory.ChromeOSFactory, | |
566 **kwargs) | |
567 | |
568 | |
569 def MakeIOSBuilderSet(**kwargs): | |
570 _MakeBuilderAndMaybeTrybotSet(factory_type=ios_factory.iOSFactory, **kwargs) | |
571 | |
572 | |
573 def MakeNaClBuilderSet(**kwargs): | |
574 _MakeBuilderAndMaybeTrybotSet(factory_type=nacl_factory.NaClFactory, **kwargs) | |
575 | |
576 | |
577 def CanMergeBuildRequests(req1, req2): | 598 def CanMergeBuildRequests(req1, req2): |
578 """ Determine whether or not two BuildRequests can be merged. Note that the | 599 """ Determine whether or not two BuildRequests can be merged. Note that the |
579 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously | 600 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously |
580 missing. This is because that method verifies that: | 601 missing. This is because that method verifies that: |
581 1. req1.source.repository == req2.source.repository | 602 1. req1.source.repository == req2.source.repository |
582 2. req1.source.project == req2.source.project | 603 2. req1.source.project == req2.source.project |
583 3. req1.source.branch == req2.source.branch | 604 3. req1.source.branch == req2.source.branch |
584 4. req1.patch == None and req2.patch = None | 605 4. req1.patch == None and req2.patch = None |
585 5. (req1.source.changes and req2.source.changes) or \ | 606 5. (req1.source.changes and req2.source.changes) or \ |
586 (not req1.source.changes and not req2.source.changes and \ | 607 (not req1.source.changes and not req2.source.changes and \ |
(...skipping 27 matching lines...) Expand all Loading... |
614 # request is associated with a change but the revisions match (#5 above). | 635 # request is associated with a change but the revisions match (#5 above). |
615 if req1.source.changes and not req2.source.changes: | 636 if req1.source.changes and not req2.source.changes: |
616 return False | 637 return False |
617 if not req1.source.changes and req2.source.changes: | 638 if not req1.source.changes and req2.source.changes: |
618 return False | 639 return False |
619 if not (req1.source.changes and req2.source.changes): | 640 if not (req1.source.changes and req2.source.changes): |
620 if req1.source.revision != req2.source.revision: | 641 if req1.source.revision != req2.source.revision: |
621 return False | 642 return False |
622 | 643 |
623 return True | 644 return True |
OLD | NEW |