Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(16)

Side by Side Diff: master/skia_master_scripts/utils.py

Issue 14081044: Add Validation for BuildFactory Configuration (Closed) Base URL: http://skia.googlecode.com/svn/buildbot/
Patch Set: Created 7 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 5
6 """Miscellaneous utilities needed by the Skia buildbot master.""" 6 """Miscellaneous utilities needed by the Skia buildbot master."""
7 7
8 8
9 import httplib2 9 import httplib2
10 import re 10 import re
11 11
12 # requires Google APIs client library for Python; see 12 # requires Google APIs client library for Python; see
13 # https://code.google.com/p/google-api-python-client/wiki/Installation 13 # https://code.google.com/p/google-api-python-client/wiki/Installation
14 from apiclient.discovery import build 14 from apiclient.discovery import build
15 from buildbot.scheduler import AnyBranchScheduler 15 from buildbot.scheduler import AnyBranchScheduler
16 from buildbot.schedulers import timed 16 from buildbot.schedulers import timed
17 from buildbot.schedulers.filter import ChangeFilter 17 from buildbot.schedulers.filter import ChangeFilter
18 from buildbot.util import NotABranch 18 from buildbot.util import NotABranch
19 from config_private import TRY_SVN_BASEURL 19 from config_private import TRY_SVN_BASEURL
20 from master import master_config 20 from master import master_config
21 from master import try_job_svn 21 from master import try_job_svn
22 from master import try_job_rietveld 22 from master import try_job_rietveld
23 from master.builders_pools import BuildersPools 23 from master.builders_pools import BuildersPools
24 from oauth2client.client import SignedJwtAssertionCredentials 24 from oauth2client.client import SignedJwtAssertionCredentials
25 25
26 from skia_master_scripts import android_factory
27 from skia_master_scripts import chromeos_factory
28 from skia_master_scripts import factory as skia_factory
29 from skia_master_scripts import housekeeping_percommit_factory, \
30 housekeeping_periodic_factory
31 from skia_master_scripts import ios_factory
32 from skia_master_scripts import nacl_factory
33
34 import config_private 26 import config_private
35 27
36 28
37 CATEGORY_BUILD = ' Build' 29 CATEGORY_BUILD = ' Build'
38 TRYBOT_NAME_SUFFIX = '_Trybot' 30 TRYBOT_NAME_SUFFIX = '_Trybot'
39 TRY_SCHEDULER_SVN = 'skia_try_svn' 31 TRY_SCHEDULER_SVN = 'skia_try_svn'
40 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' 32 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld'
41 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] 33 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD]
42 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) 34 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS)
43 35
44 36
45 def IsTrybot(builder_name): 37 def IsTrybot(builder_name):
46 return builder_name.endswith(TRYBOT_NAME_SUFFIX) 38 return builder_name.endswith(TRYBOT_NAME_SUFFIX)
47 39
48 40
41 def _IndentStr(indent):
42 return ' ' * (indent + 1)
43
44
45 def ToString(obj, indent=0):
46 """ Returns a string representation of the given object. This differs from the
47 built-in string function in that it does not give memory locations.
48
49 obj: the object to print.
rmistry 2013/04/25 20:53:17 Missing indent parameter, I wonder why the linter
borenet 2013/04/25 21:12:39 Done.
50 """
51 if isinstance(obj, list):
52 return _ListToString(obj, indent)
53 elif isinstance(obj, dict):
54 return _DictToString(obj, indent)
55 elif isinstance(obj, tuple):
56 return _ListToString(obj, indent)
57 elif isinstance(obj, str):
58 return '\'%s\'' % obj
59 elif obj is None:
60 return 'None'
61 else:
62 return '<Object>'
63
64
65 def _ListToString(list_var, indent):
66 if not list_var:
67 return '[]'
68 indent_str = _IndentStr(indent)
69 val = '[\n'
70 indent += 1
71 val += ''.join(['%s%s,\n' % (indent_str, ToString(elem, indent)) \
72 for elem in list_var])
73 indent -= 1
74 indent_str = _IndentStr(indent - 1)
75 val += indent_str + ']'
76 return val
77
78
79 def _DictToString(d, indent):
80 if not d:
81 return '{}'
82 indent_str = _IndentStr(indent)
83 val = '{\n'
84 indent += 1
85 val += ''.join(['%s%s: %s,\n' % (indent_str, ToString(k, indent),
86 ToString(d[k], indent)) \
87 for k in sorted(d.keys())])
88 indent -= 1
89 indent_str = _IndentStr(indent - 1)
90 val += indent_str + '}'
91 return val
92
93
49 class SkiaChangeFilter(ChangeFilter): 94 class SkiaChangeFilter(ChangeFilter):
50 """Skia specific subclass of ChangeFilter.""" 95 """Skia specific subclass of ChangeFilter."""
51 96
52 def __init__(self, builders, **kwargs): 97 def __init__(self, builders, **kwargs):
53 self._builders = builders 98 self._builders = builders
54 ChangeFilter.__init__(self, **kwargs) 99 ChangeFilter.__init__(self, **kwargs)
55 100
56 def filter_change(self, change): 101 def filter_change(self, change):
57 """Overrides ChangeFilter.filter_change to pass builders to filter_fn. 102 """Overrides ChangeFilter.filter_change to pass builders to filter_fn.
58 103
(...skipping 321 matching lines...) Expand 10 before | Expand all | Expand 10 after
380 425
381 def MakeCompileBuilderName(builder_base_name, release=False): 426 def MakeCompileBuilderName(builder_base_name, release=False):
382 if release: 427 if release:
383 compile_name = 'Compile_Release' 428 compile_name = 'Compile_Release'
384 else: 429 else:
385 compile_name = 'Compile_Debug' 430 compile_name = 'Compile_Debug'
386 return MakeBuilderName(builder_base_name, compile_name) 431 return MakeBuilderName(builder_base_name, compile_name)
387 432
388 433
389 def MakeDebugBuilderName(builder_base_name): 434 def MakeDebugBuilderName(builder_base_name):
390 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_DEBUG) 435 return MakeBuilderName(builder_base_name, 'Debug')
391 436
392 437
393 def MakeReleaseBuilderName(builder_base_name): 438 def MakeReleaseBuilderName(builder_base_name):
394 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_RELEASE) 439 return MakeBuilderName(builder_base_name, 'Release')
395 440
396 441
397 def MakeBenchBuilderName(builder_base_name): 442 def MakeBenchBuilderName(builder_base_name):
398 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_BENCH) 443 return MakeBuilderName(builder_base_name, 'Bench')
399 444
400 445
401 def MakeSchedulerName(builder_base_name): 446 def MakeSchedulerName(builder_base_name):
402 return MakeBuilderName(builder_base_name, 'Scheduler') 447 return MakeBuilderName(builder_base_name, 'Scheduler')
403 448
404 449
405 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, 450 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, factory_type,
406 perf_output_basedir=None, extra_branches=None, 451 perf_output_basedir=None, extra_branches=None,
407 factory_type=None, do_compile=True, do_debug=True, 452 do_compile=True, do_debug=True, do_release=True,
408 do_release=True, do_bench=True, try_schedulers=None, 453 do_bench=True, try_schedulers=None,
409 compile_bot_warnings_as_errors=True, 454 compile_bot_warnings_as_errors=True, **kwargs):
410 **kwargs):
411 """ Creates a trio of builders for a given platform: 455 """ Creates a trio of builders for a given platform:
412 1. Debug mode builder which runs all steps 456 1. Debug mode builder which runs all steps
413 2. Release mode builder which runs all steps EXCEPT benchmarks 457 2. Release mode builder which runs all steps EXCEPT benchmarks
414 3. Release mode builder which runs ONLY benchmarks. 458 3. Release mode builder which runs ONLY benchmarks.
415 """ 459 """
416 B = helper.Builder 460 B = helper.Builder
417 F = helper.Factory 461 F = helper.Factory
418 462
419 if not extra_branches: 463 if not extra_branches:
420 extra_branches = [] 464 extra_branches = []
(...skipping 13 matching lines...) Expand all
434 if do_compile: 478 if do_compile:
435 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name, 479 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name,
436 release=False) 480 release=False)
437 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, 481 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name,
438 # Do not add gatekeeper for trybots. 482 # Do not add gatekeeper for trybots.
439 gatekeeper='GateKeeper' if try_schedulers is None else None, 483 gatekeeper='GateKeeper' if try_schedulers is None else None,
440 scheduler=scheduler_name, override_category=CATEGORY_BUILD) 484 scheduler=scheduler_name, override_category=CATEGORY_BUILD)
441 F('f_%s' % compile_debug_builder_name, factory_type( 485 F('f_%s' % compile_debug_builder_name, factory_type(
442 builder_name=compile_debug_builder_name, 486 builder_name=compile_debug_builder_name,
443 other_subdirs=subdirs_to_checkout, 487 other_subdirs=subdirs_to_checkout,
444 configuration=skia_factory.CONFIG_DEBUG, 488 configuration='Debug',
445 gm_image_subdir=gm_image_subdir, 489 gm_image_subdir=gm_image_subdir,
446 do_patch_step=(try_schedulers is not None), 490 do_patch_step=(try_schedulers is not None),
447 perf_output_basedir=None, 491 perf_output_basedir=None,
448 compile_warnings_as_errors=compile_bot_warnings_as_errors, 492 compile_warnings_as_errors=compile_bot_warnings_as_errors,
449 **kwargs 493 **kwargs
450 ).BuildCompileOnly()) 494 ).BuildCompileOnly())
451 compile_release_builder_name = MakeCompileBuilderName(builder_base_name, 495 compile_release_builder_name = MakeCompileBuilderName(builder_base_name,
452 release=True) 496 release=True)
453 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, 497 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name,
454 # Do not add gatekeeper for trybots. 498 # Do not add gatekeeper for trybots.
455 gatekeeper='GateKeeper' if try_schedulers is None else None, 499 gatekeeper='GateKeeper' if try_schedulers is None else None,
456 scheduler=scheduler_name, override_category=CATEGORY_BUILD) 500 scheduler=scheduler_name, override_category=CATEGORY_BUILD)
457 F('f_%s' % compile_release_builder_name, factory_type( 501 F('f_%s' % compile_release_builder_name, factory_type(
458 builder_name=compile_release_builder_name, 502 builder_name=compile_release_builder_name,
459 other_subdirs=subdirs_to_checkout, 503 other_subdirs=subdirs_to_checkout,
460 configuration=skia_factory.CONFIG_RELEASE, 504 configuration='Release',
461 gm_image_subdir=gm_image_subdir, 505 gm_image_subdir=gm_image_subdir,
462 do_patch_step=(try_schedulers is not None), 506 do_patch_step=(try_schedulers is not None),
463 perf_output_basedir=None, 507 perf_output_basedir=None,
464 compile_warnings_as_errors=compile_bot_warnings_as_errors, 508 compile_warnings_as_errors=compile_bot_warnings_as_errors,
465 **kwargs 509 **kwargs
466 ).BuildCompileOnly()) 510 ).BuildCompileOnly())
467 511
468 if do_debug: 512 if do_debug:
469 debug_builder_name = MakeDebugBuilderName(builder_base_name) 513 debug_builder_name = MakeDebugBuilderName(builder_base_name)
470 B(debug_builder_name, 'f_%s' % debug_builder_name, 514 B(debug_builder_name, 'f_%s' % debug_builder_name,
471 scheduler=scheduler_name) 515 scheduler=scheduler_name)
472 F('f_%s' % debug_builder_name, factory_type( 516 F('f_%s' % debug_builder_name, factory_type(
473 builder_name=debug_builder_name, 517 builder_name=debug_builder_name,
474 other_subdirs=subdirs_to_checkout, 518 other_subdirs=subdirs_to_checkout,
475 configuration=skia_factory.CONFIG_DEBUG, 519 configuration='Debug',
476 gm_image_subdir=gm_image_subdir, 520 gm_image_subdir=gm_image_subdir,
477 do_patch_step=(try_schedulers is not None), 521 do_patch_step=(try_schedulers is not None),
478 perf_output_basedir=None, 522 perf_output_basedir=None,
479 compile_warnings_as_errors=False, 523 compile_warnings_as_errors=False,
480 **kwargs 524 **kwargs
481 ).Build()) 525 ).Build())
482 526
483 if do_release: 527 if do_release:
484 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name) 528 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name)
485 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, 529 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name,
486 scheduler=scheduler_name) 530 scheduler=scheduler_name)
487 F('f_%s' % no_perf_builder_name, factory_type( 531 F('f_%s' % no_perf_builder_name, factory_type(
488 builder_name=no_perf_builder_name, 532 builder_name=no_perf_builder_name,
489 other_subdirs=subdirs_to_checkout, 533 other_subdirs=subdirs_to_checkout,
490 configuration=skia_factory.CONFIG_RELEASE, 534 configuration='Release',
491 gm_image_subdir=gm_image_subdir, 535 gm_image_subdir=gm_image_subdir,
492 do_patch_step=(try_schedulers is not None), 536 do_patch_step=(try_schedulers is not None),
493 perf_output_basedir=None, 537 perf_output_basedir=None,
494 compile_warnings_as_errors=False, 538 compile_warnings_as_errors=False,
495 **kwargs 539 **kwargs
496 ).BuildNoPerf()) 540 ).BuildNoPerf())
497 541
498 if do_bench: 542 if do_bench:
499 perf_builder_name = MakeBenchBuilderName(builder_base_name) 543 perf_builder_name = MakeBenchBuilderName(builder_base_name)
500 B(perf_builder_name, 'f_%s' % perf_builder_name, 544 B(perf_builder_name, 'f_%s' % perf_builder_name,
501 scheduler=scheduler_name) 545 scheduler=scheduler_name)
502 F('f_%s' % perf_builder_name, factory_type( 546 F('f_%s' % perf_builder_name, factory_type(
503 builder_name=perf_builder_name, 547 builder_name=perf_builder_name,
504 other_subdirs=subdirs_to_checkout, 548 other_subdirs=subdirs_to_checkout,
505 configuration=skia_factory.CONFIG_RELEASE, 549 configuration='Release',
506 gm_image_subdir=gm_image_subdir, 550 gm_image_subdir=gm_image_subdir,
507 do_patch_step=(try_schedulers is not None), 551 do_patch_step=(try_schedulers is not None),
508 perf_output_basedir=perf_output_basedir, 552 perf_output_basedir=perf_output_basedir,
509 compile_warnings_as_errors=False, 553 compile_warnings_as_errors=False,
510 **kwargs 554 **kwargs
511 ).BuildPerfOnly()) 555 ).BuildPerfOnly())
512 556
513 557
514 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): 558 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs):
515 _MakeBuilderSet(try_schedulers=None, **kwargs) 559 _MakeBuilderSet(try_schedulers=None, **kwargs)
516 if do_trybots: 560 if do_trybots:
517 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) 561 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs)
518 562
519 563
520 def MakeBuilderSet(**kwargs): 564 def MakeBuilderSet(**kwargs):
521 _MakeBuilderAndMaybeTrybotSet(factory_type=skia_factory.SkiaFactory, **kwargs) 565 _MakeBuilderAndMaybeTrybotSet(**kwargs)
522 566
523 567
524 def MakeHousekeeperBuilderSet(helper, do_trybots, do_upload_results): 568 def MakeHousekeeperBuilderSet(helper, percommit_factory_type,
569 periodic_factory_type, do_trybots, **kwargs):
525 B = helper.Builder 570 B = helper.Builder
526 F = helper.Factory 571 F = helper.Factory
527 572
528 builder_factory_scheduler = [ 573 builder_factory_scheduler = [
529 # The Percommit housekeeper 574 # The Percommit housekeeper
530 ('Skia_PerCommit_House_Keeping', 575 ('Skia_PerCommit_House_Keeping',
531 housekeeping_percommit_factory.HouseKeepingPerCommitFactory, 576 percommit_factory_type,
532 'skia_rel'), 577 'skia_rel'),
533 # The Periodic housekeeper 578 # The Periodic housekeeper
534 ('Skia_Periodic_House_Keeping', 579 ('Skia_Periodic_House_Keeping',
535 housekeeping_periodic_factory.HouseKeepingPeriodicFactory, 580 periodic_factory_type,
536 'skia_periodic'), 581 'skia_periodic'),
537 ] 582 ]
538 if do_trybots: 583 if do_trybots:
539 # Add the corresponding trybot builders to the above list. 584 # Add the corresponding trybot builders to the above list.
540 builder_factory_scheduler.extend([ 585 builder_factory_scheduler.extend([
541 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR) 586 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR)
542 for (builder, factory, _scheduler) in builder_factory_scheduler]) 587 for (builder, factory, _scheduler) in builder_factory_scheduler])
543 588
544 for (builder_name, factory, scheduler) in builder_factory_scheduler: 589 for (builder_name, factory, scheduler) in builder_factory_scheduler:
545 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) 590 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler)
546 F('f_%s' % builder_name, 591 F('f_%s' % builder_name,
547 factory( 592 factory(
548 do_upload_results=do_upload_results,
549 target_platform=skia_factory.TARGET_PLATFORM_LINUX,
550 builder_name=builder_name, 593 builder_name=builder_name,
551 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), 594 do_patch_step=(scheduler == TRY_SCHEDULERS_STR),
552 ).Build()) 595 ).Build())
553 596
554
555 def MakeAndroidBuilderSet(extra_branches=None, **kwargs):
556 if not extra_branches:
557 extra_branches = []
558 extra_branches.append('android')
559 _MakeBuilderAndMaybeTrybotSet(factory_type=android_factory.AndroidFactory,
560 extra_branches=extra_branches,
561 **kwargs)
562
563
564 def MakeChromeOSBuilderSet(**kwargs):
565 _MakeBuilderAndMaybeTrybotSet(factory_type=chromeos_factory.ChromeOSFactory,
566 **kwargs)
567
568
569 def MakeIOSBuilderSet(**kwargs):
570 _MakeBuilderAndMaybeTrybotSet(factory_type=ios_factory.iOSFactory, **kwargs)
571
572
573 def MakeNaClBuilderSet(**kwargs):
574 _MakeBuilderAndMaybeTrybotSet(factory_type=nacl_factory.NaClFactory, **kwargs)
575
576
577 def CanMergeBuildRequests(req1, req2): 597 def CanMergeBuildRequests(req1, req2):
578 """ Determine whether or not two BuildRequests can be merged. Note that the 598 """ Determine whether or not two BuildRequests can be merged. Note that the
579 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously 599 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously
580 missing. This is because that method verifies that: 600 missing. This is because that method verifies that:
581 1. req1.source.repository == req2.source.repository 601 1. req1.source.repository == req2.source.repository
582 2. req1.source.project == req2.source.project 602 2. req1.source.project == req2.source.project
583 3. req1.source.branch == req2.source.branch 603 3. req1.source.branch == req2.source.branch
584 4. req1.patch == None and req2.patch = None 604 4. req1.patch == None and req2.patch = None
585 5. (req1.source.changes and req2.source.changes) or \ 605 5. (req1.source.changes and req2.source.changes) or \
586 (not req1.source.changes and not req2.source.changes and \ 606 (not req1.source.changes and not req2.source.changes and \
(...skipping 27 matching lines...) Expand all
614 # request is associated with a change but the revisions match (#5 above). 634 # request is associated with a change but the revisions match (#5 above).
615 if req1.source.changes and not req2.source.changes: 635 if req1.source.changes and not req2.source.changes:
616 return False 636 return False
617 if not req1.source.changes and req2.source.changes: 637 if not req1.source.changes and req2.source.changes:
618 return False 638 return False
619 if not (req1.source.changes and req2.source.changes): 639 if not (req1.source.changes and req2.source.changes):
620 if req1.source.revision != req2.source.revision: 640 if req1.source.revision != req2.source.revision:
621 return False 641 return False
622 642
623 return True 643 return True
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698