OLD | NEW |
---|---|
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 | 5 |
6 """Miscellaneous utilities needed by the Skia buildbot master.""" | 6 """Miscellaneous utilities needed by the Skia buildbot master.""" |
7 | 7 |
8 | 8 |
9 import httplib2 | 9 import httplib2 |
10 import re | 10 import re |
11 | 11 |
12 # requires Google APIs client library for Python; see | 12 # requires Google APIs client library for Python; see |
13 # https://code.google.com/p/google-api-python-client/wiki/Installation | 13 # https://code.google.com/p/google-api-python-client/wiki/Installation |
14 from apiclient.discovery import build | 14 from apiclient.discovery import build |
15 from buildbot.scheduler import AnyBranchScheduler | 15 from buildbot.scheduler import AnyBranchScheduler |
16 from buildbot.schedulers import timed | 16 from buildbot.schedulers import timed |
17 from buildbot.schedulers.filter import ChangeFilter | 17 from buildbot.schedulers.filter import ChangeFilter |
18 from buildbot.util import NotABranch | 18 from buildbot.util import NotABranch |
19 from config_private import TRY_SVN_BASEURL | 19 from config_private import TRY_SVN_BASEURL |
20 from master import master_config | 20 from master import master_config |
21 from master import try_job_svn | 21 from master import try_job_svn |
22 from master import try_job_rietveld | 22 from master import try_job_rietveld |
23 from master.builders_pools import BuildersPools | 23 from master.builders_pools import BuildersPools |
24 from oauth2client.client import SignedJwtAssertionCredentials | 24 from oauth2client.client import SignedJwtAssertionCredentials |
25 | 25 |
26 from skia_master_scripts import android_factory | |
27 from skia_master_scripts import chromeos_factory | |
28 from skia_master_scripts import factory as skia_factory | |
29 from skia_master_scripts import housekeeping_percommit_factory, \ | |
30 housekeeping_periodic_factory | |
31 from skia_master_scripts import ios_factory | |
32 from skia_master_scripts import nacl_factory | |
33 | |
34 import config_private | 26 import config_private |
35 | 27 |
36 | 28 |
37 CATEGORY_BUILD = ' Build' | 29 CATEGORY_BUILD = ' Build' |
38 TRYBOT_NAME_SUFFIX = '_Trybot' | 30 TRYBOT_NAME_SUFFIX = '_Trybot' |
39 TRY_SCHEDULER_SVN = 'skia_try_svn' | 31 TRY_SCHEDULER_SVN = 'skia_try_svn' |
40 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' | 32 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' |
41 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] | 33 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] |
42 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) | 34 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) |
43 | 35 |
44 | 36 |
45 def IsTrybot(builder_name): | 37 def IsTrybot(builder_name): |
46 return builder_name.endswith(TRYBOT_NAME_SUFFIX) | 38 return builder_name.endswith(TRYBOT_NAME_SUFFIX) |
47 | 39 |
48 | 40 |
41 def _IndentStr(indent): | |
42 string = '' | |
epoger
2013/04/25 18:12:29
I think you can implement this as:
return ' '
borenet
2013/04/25 18:28:09
Done.
| |
43 for _ in range(indent + 1): | |
44 string += ' ' | |
45 return string | |
46 | |
47 | |
48 def ToString(o, indent=0): | |
49 """ Returns a string representation of the given object. This differs from the | |
50 built-in string function in that it does not give memory locations. | |
51 z | |
epoger
2013/04/25 18:12:29
z ???
borenet
2013/04/25 18:28:09
Done.
| |
52 o: the object to print. | |
epoger
2013/04/25 18:12:29
suggest o -> obj
borenet
2013/04/25 18:28:09
Done.
| |
53 """ | |
54 if isinstance(o, list): | |
55 return _ListToString(o, indent) | |
56 elif isinstance(o, dict): | |
57 return _DictToString(o, indent) | |
58 elif isinstance(o, tuple): | |
59 return _ListToString(o, indent) | |
60 elif isinstance(o, str): | |
61 return '\'%s\'' % o | |
62 elif o is None: | |
63 return 'None' | |
64 else: | |
65 return '<Object>' | |
66 | |
67 | |
68 def _ListToString(l, indent): | |
epoger
2013/04/25 18:12:29
suggest "list" or a variable name like that; "l" l
borenet
2013/04/25 18:28:09
Changed to list_var to avoid collision with built-
| |
69 if not l: | |
70 return '[]' | |
71 indent_str = _IndentStr(indent) | |
72 val = '[\n' | |
73 indent += 1 | |
74 val += ''.join(['%s%s,\n' % (indent_str, ToString(elem, indent)) \ | |
75 for elem in l]) | |
76 indent -= 1 | |
77 indent_str = _IndentStr(indent - 1) | |
78 val += indent_str + ']' | |
79 return val | |
80 | |
81 | |
82 def _DictToString(d, indent): | |
83 if not d: | |
84 return '{}' | |
85 indent_str = _IndentStr(indent) | |
86 val = '{\n' | |
87 indent += 1 | |
88 val += ''.join(['%s%s: %s,\n' % (indent_str, ToString(k, indent), | |
89 ToString(d[k], indent)) \ | |
90 for k in sorted(d.keys())]) | |
91 indent -= 1 | |
92 indent_str = _IndentStr(indent - 1) | |
93 val += indent_str + '}' | |
94 return val | |
95 | |
96 | |
49 class SkiaChangeFilter(ChangeFilter): | 97 class SkiaChangeFilter(ChangeFilter): |
50 """Skia specific subclass of ChangeFilter.""" | 98 """Skia specific subclass of ChangeFilter.""" |
51 | 99 |
52 def __init__(self, builders, **kwargs): | 100 def __init__(self, builders, **kwargs): |
53 self._builders = builders | 101 self._builders = builders |
54 ChangeFilter.__init__(self, **kwargs) | 102 ChangeFilter.__init__(self, **kwargs) |
55 | 103 |
56 def filter_change(self, change): | 104 def filter_change(self, change): |
57 """Overrides ChangeFilter.filter_change to pass builders to filter_fn. | 105 """Overrides ChangeFilter.filter_change to pass builders to filter_fn. |
58 | 106 |
(...skipping 321 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
380 | 428 |
381 def MakeCompileBuilderName(builder_base_name, release=False): | 429 def MakeCompileBuilderName(builder_base_name, release=False): |
382 if release: | 430 if release: |
383 compile_name = 'Compile_Release' | 431 compile_name = 'Compile_Release' |
384 else: | 432 else: |
385 compile_name = 'Compile_Debug' | 433 compile_name = 'Compile_Debug' |
386 return MakeBuilderName(builder_base_name, compile_name) | 434 return MakeBuilderName(builder_base_name, compile_name) |
387 | 435 |
388 | 436 |
389 def MakeDebugBuilderName(builder_base_name): | 437 def MakeDebugBuilderName(builder_base_name): |
390 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_DEBUG) | 438 return MakeBuilderName(builder_base_name, 'Debug') |
391 | 439 |
392 | 440 |
393 def MakeReleaseBuilderName(builder_base_name): | 441 def MakeReleaseBuilderName(builder_base_name): |
394 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_RELEASE) | 442 return MakeBuilderName(builder_base_name, 'Release') |
395 | 443 |
396 | 444 |
397 def MakeBenchBuilderName(builder_base_name): | 445 def MakeBenchBuilderName(builder_base_name): |
398 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_BENCH) | 446 return MakeBuilderName(builder_base_name, 'Bench') |
399 | 447 |
400 | 448 |
401 def MakeSchedulerName(builder_base_name): | 449 def MakeSchedulerName(builder_base_name): |
402 return MakeBuilderName(builder_base_name, 'Scheduler') | 450 return MakeBuilderName(builder_base_name, 'Scheduler') |
403 | 451 |
404 | 452 |
405 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, | 453 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, factory_type, |
406 perf_output_basedir=None, extra_branches=None, | 454 perf_output_basedir=None, extra_branches=None, |
407 factory_type=None, do_compile=True, do_debug=True, | 455 do_compile=True, do_debug=True, do_release=True, |
408 do_release=True, do_bench=True, try_schedulers=None, | 456 do_bench=True, try_schedulers=None, |
409 compile_bot_warnings_as_errors=True, | 457 compile_bot_warnings_as_errors=True, **kwargs): |
410 **kwargs): | |
411 """ Creates a trio of builders for a given platform: | 458 """ Creates a trio of builders for a given platform: |
412 1. Debug mode builder which runs all steps | 459 1. Debug mode builder which runs all steps |
413 2. Release mode builder which runs all steps EXCEPT benchmarks | 460 2. Release mode builder which runs all steps EXCEPT benchmarks |
414 3. Release mode builder which runs ONLY benchmarks. | 461 3. Release mode builder which runs ONLY benchmarks. |
415 """ | 462 """ |
416 B = helper.Builder | 463 B = helper.Builder |
417 F = helper.Factory | 464 F = helper.Factory |
418 | 465 |
419 if not extra_branches: | 466 if not extra_branches: |
420 extra_branches = [] | 467 extra_branches = [] |
(...skipping 13 matching lines...) Expand all Loading... | |
434 if do_compile: | 481 if do_compile: |
435 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name, | 482 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name, |
436 release=False) | 483 release=False) |
437 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, | 484 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, |
438 # Do not add gatekeeper for trybots. | 485 # Do not add gatekeeper for trybots. |
439 gatekeeper='GateKeeper' if try_schedulers is None else None, | 486 gatekeeper='GateKeeper' if try_schedulers is None else None, |
440 scheduler=scheduler_name, override_category=CATEGORY_BUILD) | 487 scheduler=scheduler_name, override_category=CATEGORY_BUILD) |
441 F('f_%s' % compile_debug_builder_name, factory_type( | 488 F('f_%s' % compile_debug_builder_name, factory_type( |
442 builder_name=compile_debug_builder_name, | 489 builder_name=compile_debug_builder_name, |
443 other_subdirs=subdirs_to_checkout, | 490 other_subdirs=subdirs_to_checkout, |
444 configuration=skia_factory.CONFIG_DEBUG, | 491 configuration='Debug', |
epoger
2013/04/25 18:12:29
I think there's value in using constants to hold t
borenet
2013/04/25 18:28:09
Agreed. Unfortunately, I can't use the constants
| |
445 gm_image_subdir=gm_image_subdir, | 492 gm_image_subdir=gm_image_subdir, |
446 do_patch_step=(try_schedulers is not None), | 493 do_patch_step=(try_schedulers is not None), |
447 perf_output_basedir=None, | 494 perf_output_basedir=None, |
448 compile_warnings_as_errors=compile_bot_warnings_as_errors, | 495 compile_warnings_as_errors=compile_bot_warnings_as_errors, |
449 **kwargs | 496 **kwargs |
450 ).BuildCompileOnly()) | 497 ).BuildCompileOnly()) |
451 compile_release_builder_name = MakeCompileBuilderName(builder_base_name, | 498 compile_release_builder_name = MakeCompileBuilderName(builder_base_name, |
452 release=True) | 499 release=True) |
453 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, | 500 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, |
454 # Do not add gatekeeper for trybots. | 501 # Do not add gatekeeper for trybots. |
455 gatekeeper='GateKeeper' if try_schedulers is None else None, | 502 gatekeeper='GateKeeper' if try_schedulers is None else None, |
456 scheduler=scheduler_name, override_category=CATEGORY_BUILD) | 503 scheduler=scheduler_name, override_category=CATEGORY_BUILD) |
457 F('f_%s' % compile_release_builder_name, factory_type( | 504 F('f_%s' % compile_release_builder_name, factory_type( |
458 builder_name=compile_release_builder_name, | 505 builder_name=compile_release_builder_name, |
459 other_subdirs=subdirs_to_checkout, | 506 other_subdirs=subdirs_to_checkout, |
460 configuration=skia_factory.CONFIG_RELEASE, | 507 configuration='Release', |
461 gm_image_subdir=gm_image_subdir, | 508 gm_image_subdir=gm_image_subdir, |
462 do_patch_step=(try_schedulers is not None), | 509 do_patch_step=(try_schedulers is not None), |
463 perf_output_basedir=None, | 510 perf_output_basedir=None, |
464 compile_warnings_as_errors=compile_bot_warnings_as_errors, | 511 compile_warnings_as_errors=compile_bot_warnings_as_errors, |
465 **kwargs | 512 **kwargs |
466 ).BuildCompileOnly()) | 513 ).BuildCompileOnly()) |
467 | 514 |
468 if do_debug: | 515 if do_debug: |
469 debug_builder_name = MakeDebugBuilderName(builder_base_name) | 516 debug_builder_name = MakeDebugBuilderName(builder_base_name) |
470 B(debug_builder_name, 'f_%s' % debug_builder_name, | 517 B(debug_builder_name, 'f_%s' % debug_builder_name, |
471 scheduler=scheduler_name) | 518 scheduler=scheduler_name) |
472 F('f_%s' % debug_builder_name, factory_type( | 519 F('f_%s' % debug_builder_name, factory_type( |
473 builder_name=debug_builder_name, | 520 builder_name=debug_builder_name, |
474 other_subdirs=subdirs_to_checkout, | 521 other_subdirs=subdirs_to_checkout, |
475 configuration=skia_factory.CONFIG_DEBUG, | 522 configuration='Debug', |
476 gm_image_subdir=gm_image_subdir, | 523 gm_image_subdir=gm_image_subdir, |
477 do_patch_step=(try_schedulers is not None), | 524 do_patch_step=(try_schedulers is not None), |
478 perf_output_basedir=None, | 525 perf_output_basedir=None, |
479 compile_warnings_as_errors=False, | 526 compile_warnings_as_errors=False, |
480 **kwargs | 527 **kwargs |
481 ).Build()) | 528 ).Build()) |
482 | 529 |
483 if do_release: | 530 if do_release: |
484 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name) | 531 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name) |
485 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, | 532 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, |
486 scheduler=scheduler_name) | 533 scheduler=scheduler_name) |
487 F('f_%s' % no_perf_builder_name, factory_type( | 534 F('f_%s' % no_perf_builder_name, factory_type( |
488 builder_name=no_perf_builder_name, | 535 builder_name=no_perf_builder_name, |
489 other_subdirs=subdirs_to_checkout, | 536 other_subdirs=subdirs_to_checkout, |
490 configuration=skia_factory.CONFIG_RELEASE, | 537 configuration='Release', |
491 gm_image_subdir=gm_image_subdir, | 538 gm_image_subdir=gm_image_subdir, |
492 do_patch_step=(try_schedulers is not None), | 539 do_patch_step=(try_schedulers is not None), |
493 perf_output_basedir=None, | 540 perf_output_basedir=None, |
494 compile_warnings_as_errors=False, | 541 compile_warnings_as_errors=False, |
495 **kwargs | 542 **kwargs |
496 ).BuildNoPerf()) | 543 ).BuildNoPerf()) |
497 | 544 |
498 if do_bench: | 545 if do_bench: |
499 perf_builder_name = MakeBenchBuilderName(builder_base_name) | 546 perf_builder_name = MakeBenchBuilderName(builder_base_name) |
500 B(perf_builder_name, 'f_%s' % perf_builder_name, | 547 B(perf_builder_name, 'f_%s' % perf_builder_name, |
501 scheduler=scheduler_name) | 548 scheduler=scheduler_name) |
502 F('f_%s' % perf_builder_name, factory_type( | 549 F('f_%s' % perf_builder_name, factory_type( |
503 builder_name=perf_builder_name, | 550 builder_name=perf_builder_name, |
504 other_subdirs=subdirs_to_checkout, | 551 other_subdirs=subdirs_to_checkout, |
505 configuration=skia_factory.CONFIG_RELEASE, | 552 configuration='Release', |
506 gm_image_subdir=gm_image_subdir, | 553 gm_image_subdir=gm_image_subdir, |
507 do_patch_step=(try_schedulers is not None), | 554 do_patch_step=(try_schedulers is not None), |
508 perf_output_basedir=perf_output_basedir, | 555 perf_output_basedir=perf_output_basedir, |
509 compile_warnings_as_errors=False, | 556 compile_warnings_as_errors=False, |
510 **kwargs | 557 **kwargs |
511 ).BuildPerfOnly()) | 558 ).BuildPerfOnly()) |
512 | 559 |
513 | 560 |
514 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): | 561 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): |
515 _MakeBuilderSet(try_schedulers=None, **kwargs) | 562 _MakeBuilderSet(try_schedulers=None, **kwargs) |
516 if do_trybots: | 563 if do_trybots: |
517 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) | 564 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) |
518 | 565 |
519 | 566 |
520 def MakeBuilderSet(**kwargs): | 567 def MakeBuilderSet(**kwargs): |
521 _MakeBuilderAndMaybeTrybotSet(factory_type=skia_factory.SkiaFactory, **kwargs) | 568 _MakeBuilderAndMaybeTrybotSet(**kwargs) |
522 | 569 |
523 | 570 |
524 def MakeHousekeeperBuilderSet(helper, do_trybots, do_upload_results): | 571 def MakeHousekeeperBuilderSet(helper, percommit_factory_type, |
572 periodic_factory_type, do_trybots, **kwargs): | |
525 B = helper.Builder | 573 B = helper.Builder |
526 F = helper.Factory | 574 F = helper.Factory |
527 | 575 |
528 builder_factory_scheduler = [ | 576 builder_factory_scheduler = [ |
529 # The Percommit housekeeper | 577 # The Percommit housekeeper |
530 ('Skia_PerCommit_House_Keeping', | 578 ('Skia_PerCommit_House_Keeping', |
531 housekeeping_percommit_factory.HouseKeepingPerCommitFactory, | 579 percommit_factory_type, |
532 'skia_rel'), | 580 'skia_rel'), |
533 # The Periodic housekeeper | 581 # The Periodic housekeeper |
534 ('Skia_Periodic_House_Keeping', | 582 ('Skia_Periodic_House_Keeping', |
535 housekeeping_periodic_factory.HouseKeepingPeriodicFactory, | 583 periodic_factory_type, |
536 'skia_periodic'), | 584 'skia_periodic'), |
537 ] | 585 ] |
538 if do_trybots: | 586 if do_trybots: |
539 # Add the corresponding trybot builders to the above list. | 587 # Add the corresponding trybot builders to the above list. |
540 builder_factory_scheduler.extend([ | 588 builder_factory_scheduler.extend([ |
541 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR) | 589 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR) |
542 for (builder, factory, _scheduler) in builder_factory_scheduler]) | 590 for (builder, factory, _scheduler) in builder_factory_scheduler]) |
543 | 591 |
544 for (builder_name, factory, scheduler) in builder_factory_scheduler: | 592 for (builder_name, factory, scheduler) in builder_factory_scheduler: |
545 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) | 593 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) |
546 F('f_%s' % builder_name, | 594 F('f_%s' % builder_name, |
547 factory( | 595 factory( |
548 do_upload_results=do_upload_results, | |
549 target_platform=skia_factory.TARGET_PLATFORM_LINUX, | |
550 builder_name=builder_name, | 596 builder_name=builder_name, |
551 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), | 597 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), |
552 ).Build()) | 598 ).Build()) |
553 | 599 |
554 | |
555 def MakeAndroidBuilderSet(extra_branches=None, **kwargs): | |
556 if not extra_branches: | |
557 extra_branches = [] | |
558 extra_branches.append('android') | |
559 _MakeBuilderAndMaybeTrybotSet(factory_type=android_factory.AndroidFactory, | |
560 extra_branches=extra_branches, | |
561 **kwargs) | |
562 | |
563 | |
564 def MakeChromeOSBuilderSet(**kwargs): | |
565 _MakeBuilderAndMaybeTrybotSet(factory_type=chromeos_factory.ChromeOSFactory, | |
566 **kwargs) | |
567 | |
568 | |
569 def MakeIOSBuilderSet(**kwargs): | |
570 _MakeBuilderAndMaybeTrybotSet(factory_type=ios_factory.iOSFactory, **kwargs) | |
571 | |
572 | |
573 def MakeNaClBuilderSet(**kwargs): | |
574 _MakeBuilderAndMaybeTrybotSet(factory_type=nacl_factory.NaClFactory, **kwargs) | |
575 | |
576 | |
577 def CanMergeBuildRequests(req1, req2): | 600 def CanMergeBuildRequests(req1, req2): |
578 """ Determine whether or not two BuildRequests can be merged. Note that the | 601 """ Determine whether or not two BuildRequests can be merged. Note that the |
579 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously | 602 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously |
580 missing. This is because that method verifies that: | 603 missing. This is because that method verifies that: |
581 1. req1.source.repository == req2.source.repository | 604 1. req1.source.repository == req2.source.repository |
582 2. req1.source.project == req2.source.project | 605 2. req1.source.project == req2.source.project |
583 3. req1.source.branch == req2.source.branch | 606 3. req1.source.branch == req2.source.branch |
584 4. req1.patch == None and req2.patch = None | 607 4. req1.patch == None and req2.patch = None |
585 5. (req1.source.changes and req2.source.changes) or \ | 608 5. (req1.source.changes and req2.source.changes) or \ |
586 (not req1.source.changes and not req2.source.changes and \ | 609 (not req1.source.changes and not req2.source.changes and \ |
(...skipping 27 matching lines...) Expand all Loading... | |
614 # request is associated with a change but the revisions match (#5 above). | 637 # request is associated with a change but the revisions match (#5 above). |
615 if req1.source.changes and not req2.source.changes: | 638 if req1.source.changes and not req2.source.changes: |
616 return False | 639 return False |
617 if not req1.source.changes and req2.source.changes: | 640 if not req1.source.changes and req2.source.changes: |
618 return False | 641 return False |
619 if not (req1.source.changes and req2.source.changes): | 642 if not (req1.source.changes and req2.source.changes): |
620 if req1.source.revision != req2.source.revision: | 643 if req1.source.revision != req2.source.revision: |
621 return False | 644 return False |
622 | 645 |
623 return True | 646 return True |
OLD | NEW |