Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(361)

Side by Side Diff: master/skia_master_scripts/utils.py

Issue 14081044: Add Validation for BuildFactory Configuration (Closed) Base URL: http://skia.googlecode.com/svn/buildbot/
Patch Set: Created 7 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 5
6 """Miscellaneous utilities needed by the Skia buildbot master.""" 6 """Miscellaneous utilities needed by the Skia buildbot master."""
7 7
8 8
9 import httplib2 9 import httplib2
10 import re 10 import re
11 11
12 # requires Google APIs client library for Python; see 12 # requires Google APIs client library for Python; see
13 # https://code.google.com/p/google-api-python-client/wiki/Installation 13 # https://code.google.com/p/google-api-python-client/wiki/Installation
14 from apiclient.discovery import build 14 from apiclient.discovery import build
15 from buildbot.scheduler import AnyBranchScheduler 15 from buildbot.scheduler import AnyBranchScheduler
16 from buildbot.schedulers import timed 16 from buildbot.schedulers import timed
17 from buildbot.schedulers.filter import ChangeFilter 17 from buildbot.schedulers.filter import ChangeFilter
18 from buildbot.util import NotABranch 18 from buildbot.util import NotABranch
19 from config_private import TRY_SVN_BASEURL 19 from config_private import TRY_SVN_BASEURL
20 from master import master_config 20 from master import master_config
21 from master import try_job_svn 21 from master import try_job_svn
22 from master import try_job_rietveld 22 from master import try_job_rietveld
23 from master.builders_pools import BuildersPools 23 from master.builders_pools import BuildersPools
24 from oauth2client.client import SignedJwtAssertionCredentials 24 from oauth2client.client import SignedJwtAssertionCredentials
25 25
26 from skia_master_scripts import android_factory
27 from skia_master_scripts import chromeos_factory
28 from skia_master_scripts import factory as skia_factory
29 from skia_master_scripts import housekeeping_percommit_factory, \
30 housekeeping_periodic_factory
31 from skia_master_scripts import ios_factory
32 from skia_master_scripts import nacl_factory
33
34 import config_private 26 import config_private
35 27
36 28
37 CATEGORY_BUILD = ' Build' 29 CATEGORY_BUILD = ' Build'
38 TRYBOT_NAME_SUFFIX = '_Trybot' 30 TRYBOT_NAME_SUFFIX = '_Trybot'
39 TRY_SCHEDULER_SVN = 'skia_try_svn' 31 TRY_SCHEDULER_SVN = 'skia_try_svn'
40 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' 32 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld'
41 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] 33 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD]
42 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) 34 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS)
43 35
44 36
45 def IsTrybot(builder_name): 37 def IsTrybot(builder_name):
46 return builder_name.endswith(TRYBOT_NAME_SUFFIX) 38 return builder_name.endswith(TRYBOT_NAME_SUFFIX)
47 39
48 40
41 def _IndentStr(indent):
42 return ' ' * (indent + 1)
43
44
45 def ToString(obj, indent=0):
46 """ Returns a string representation of the given object. This differs from the
47 built-in string function in that it does not give memory locations.
48
49 obj: the object to print.
50 indent: integer; the current indent level.
51 """
52 if isinstance(obj, list):
53 return _ListToString(obj, indent)
54 elif isinstance(obj, dict):
55 return _DictToString(obj, indent)
56 elif isinstance(obj, tuple):
57 return _ListToString(obj, indent)
58 elif isinstance(obj, str):
59 return '\'%s\'' % obj
60 elif obj is None:
61 return 'None'
62 else:
63 return '<Object>'
64
65
66 def _ListToString(list_var, indent):
67 if not list_var:
68 return '[]'
69 indent_str = _IndentStr(indent)
70 val = '[\n'
71 indent += 1
72 val += ''.join(['%s%s,\n' % (indent_str, ToString(elem, indent)) \
73 for elem in list_var])
74 indent -= 1
75 indent_str = _IndentStr(indent - 1)
76 val += indent_str + ']'
77 return val
78
79
80 def _DictToString(d, indent):
81 if not d:
82 return '{}'
83 indent_str = _IndentStr(indent)
84 val = '{\n'
85 indent += 1
86 val += ''.join(['%s%s: %s,\n' % (indent_str, ToString(k, indent),
87 ToString(d[k], indent)) \
88 for k in sorted(d.keys())])
89 indent -= 1
90 indent_str = _IndentStr(indent - 1)
91 val += indent_str + '}'
92 return val
93
94
49 class SkiaChangeFilter(ChangeFilter): 95 class SkiaChangeFilter(ChangeFilter):
50 """Skia specific subclass of ChangeFilter.""" 96 """Skia specific subclass of ChangeFilter."""
51 97
52 def __init__(self, builders, **kwargs): 98 def __init__(self, builders, **kwargs):
53 self._builders = builders 99 self._builders = builders
54 ChangeFilter.__init__(self, **kwargs) 100 ChangeFilter.__init__(self, **kwargs)
55 101
56 def filter_change(self, change): 102 def filter_change(self, change):
57 """Overrides ChangeFilter.filter_change to pass builders to filter_fn. 103 """Overrides ChangeFilter.filter_change to pass builders to filter_fn.
58 104
(...skipping 321 matching lines...) Expand 10 before | Expand all | Expand 10 after
380 426
381 def MakeCompileBuilderName(builder_base_name, release=False): 427 def MakeCompileBuilderName(builder_base_name, release=False):
382 if release: 428 if release:
383 compile_name = 'Compile_Release' 429 compile_name = 'Compile_Release'
384 else: 430 else:
385 compile_name = 'Compile_Debug' 431 compile_name = 'Compile_Debug'
386 return MakeBuilderName(builder_base_name, compile_name) 432 return MakeBuilderName(builder_base_name, compile_name)
387 433
388 434
389 def MakeDebugBuilderName(builder_base_name): 435 def MakeDebugBuilderName(builder_base_name):
390 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_DEBUG) 436 return MakeBuilderName(builder_base_name, 'Debug')
391 437
392 438
393 def MakeReleaseBuilderName(builder_base_name): 439 def MakeReleaseBuilderName(builder_base_name):
394 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_RELEASE) 440 return MakeBuilderName(builder_base_name, 'Release')
395 441
396 442
397 def MakeBenchBuilderName(builder_base_name): 443 def MakeBenchBuilderName(builder_base_name):
398 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_BENCH) 444 return MakeBuilderName(builder_base_name, 'Bench')
399 445
400 446
401 def MakeSchedulerName(builder_base_name): 447 def MakeSchedulerName(builder_base_name):
402 return MakeBuilderName(builder_base_name, 'Scheduler') 448 return MakeBuilderName(builder_base_name, 'Scheduler')
403 449
404 450
405 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, 451 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, factory_type,
406 perf_output_basedir=None, extra_branches=None, 452 perf_output_basedir=None, extra_branches=None,
407 factory_type=None, do_compile=True, do_debug=True, 453 do_compile=True, do_debug=True, do_release=True,
408 do_release=True, do_bench=True, try_schedulers=None, 454 do_bench=True, try_schedulers=None,
409 compile_bot_warnings_as_errors=True, 455 compile_bot_warnings_as_errors=True, **kwargs):
410 **kwargs):
411 """ Creates a trio of builders for a given platform: 456 """ Creates a trio of builders for a given platform:
412 1. Debug mode builder which runs all steps 457 1. Debug mode builder which runs all steps
413 2. Release mode builder which runs all steps EXCEPT benchmarks 458 2. Release mode builder which runs all steps EXCEPT benchmarks
414 3. Release mode builder which runs ONLY benchmarks. 459 3. Release mode builder which runs ONLY benchmarks.
415 """ 460 """
416 B = helper.Builder 461 B = helper.Builder
417 F = helper.Factory 462 F = helper.Factory
418 463
419 if not extra_branches: 464 if not extra_branches:
420 extra_branches = [] 465 extra_branches = []
(...skipping 13 matching lines...) Expand all
434 if do_compile: 479 if do_compile:
435 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name, 480 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name,
436 release=False) 481 release=False)
437 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, 482 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name,
438 # Do not add gatekeeper for trybots. 483 # Do not add gatekeeper for trybots.
439 gatekeeper='GateKeeper' if try_schedulers is None else None, 484 gatekeeper='GateKeeper' if try_schedulers is None else None,
440 scheduler=scheduler_name, override_category=CATEGORY_BUILD) 485 scheduler=scheduler_name, override_category=CATEGORY_BUILD)
441 F('f_%s' % compile_debug_builder_name, factory_type( 486 F('f_%s' % compile_debug_builder_name, factory_type(
442 builder_name=compile_debug_builder_name, 487 builder_name=compile_debug_builder_name,
443 other_subdirs=subdirs_to_checkout, 488 other_subdirs=subdirs_to_checkout,
444 configuration=skia_factory.CONFIG_DEBUG, 489 configuration='Debug',
445 gm_image_subdir=gm_image_subdir, 490 gm_image_subdir=gm_image_subdir,
446 do_patch_step=(try_schedulers is not None), 491 do_patch_step=(try_schedulers is not None),
447 perf_output_basedir=None, 492 perf_output_basedir=None,
448 compile_warnings_as_errors=compile_bot_warnings_as_errors, 493 compile_warnings_as_errors=compile_bot_warnings_as_errors,
449 **kwargs 494 **kwargs
450 ).BuildCompileOnly()) 495 ).BuildCompileOnly())
451 compile_release_builder_name = MakeCompileBuilderName(builder_base_name, 496 compile_release_builder_name = MakeCompileBuilderName(builder_base_name,
452 release=True) 497 release=True)
453 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, 498 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name,
454 # Do not add gatekeeper for trybots. 499 # Do not add gatekeeper for trybots.
455 gatekeeper='GateKeeper' if try_schedulers is None else None, 500 gatekeeper='GateKeeper' if try_schedulers is None else None,
456 scheduler=scheduler_name, override_category=CATEGORY_BUILD) 501 scheduler=scheduler_name, override_category=CATEGORY_BUILD)
457 F('f_%s' % compile_release_builder_name, factory_type( 502 F('f_%s' % compile_release_builder_name, factory_type(
458 builder_name=compile_release_builder_name, 503 builder_name=compile_release_builder_name,
459 other_subdirs=subdirs_to_checkout, 504 other_subdirs=subdirs_to_checkout,
460 configuration=skia_factory.CONFIG_RELEASE, 505 configuration='Release',
461 gm_image_subdir=gm_image_subdir, 506 gm_image_subdir=gm_image_subdir,
462 do_patch_step=(try_schedulers is not None), 507 do_patch_step=(try_schedulers is not None),
463 perf_output_basedir=None, 508 perf_output_basedir=None,
464 compile_warnings_as_errors=compile_bot_warnings_as_errors, 509 compile_warnings_as_errors=compile_bot_warnings_as_errors,
465 **kwargs 510 **kwargs
466 ).BuildCompileOnly()) 511 ).BuildCompileOnly())
467 512
468 if do_debug: 513 if do_debug:
469 debug_builder_name = MakeDebugBuilderName(builder_base_name) 514 debug_builder_name = MakeDebugBuilderName(builder_base_name)
470 B(debug_builder_name, 'f_%s' % debug_builder_name, 515 B(debug_builder_name, 'f_%s' % debug_builder_name,
471 scheduler=scheduler_name) 516 scheduler=scheduler_name)
472 F('f_%s' % debug_builder_name, factory_type( 517 F('f_%s' % debug_builder_name, factory_type(
473 builder_name=debug_builder_name, 518 builder_name=debug_builder_name,
474 other_subdirs=subdirs_to_checkout, 519 other_subdirs=subdirs_to_checkout,
475 configuration=skia_factory.CONFIG_DEBUG, 520 configuration='Debug',
476 gm_image_subdir=gm_image_subdir, 521 gm_image_subdir=gm_image_subdir,
477 do_patch_step=(try_schedulers is not None), 522 do_patch_step=(try_schedulers is not None),
478 perf_output_basedir=None, 523 perf_output_basedir=None,
479 compile_warnings_as_errors=False, 524 compile_warnings_as_errors=False,
480 **kwargs 525 **kwargs
481 ).Build()) 526 ).Build())
482 527
483 if do_release: 528 if do_release:
484 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name) 529 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name)
485 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, 530 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name,
486 scheduler=scheduler_name) 531 scheduler=scheduler_name)
487 F('f_%s' % no_perf_builder_name, factory_type( 532 F('f_%s' % no_perf_builder_name, factory_type(
488 builder_name=no_perf_builder_name, 533 builder_name=no_perf_builder_name,
489 other_subdirs=subdirs_to_checkout, 534 other_subdirs=subdirs_to_checkout,
490 configuration=skia_factory.CONFIG_RELEASE, 535 configuration='Release',
491 gm_image_subdir=gm_image_subdir, 536 gm_image_subdir=gm_image_subdir,
492 do_patch_step=(try_schedulers is not None), 537 do_patch_step=(try_schedulers is not None),
493 perf_output_basedir=None, 538 perf_output_basedir=None,
494 compile_warnings_as_errors=False, 539 compile_warnings_as_errors=False,
495 **kwargs 540 **kwargs
496 ).BuildNoPerf()) 541 ).BuildNoPerf())
497 542
498 if do_bench: 543 if do_bench:
499 perf_builder_name = MakeBenchBuilderName(builder_base_name) 544 perf_builder_name = MakeBenchBuilderName(builder_base_name)
500 B(perf_builder_name, 'f_%s' % perf_builder_name, 545 B(perf_builder_name, 'f_%s' % perf_builder_name,
501 scheduler=scheduler_name) 546 scheduler=scheduler_name)
502 F('f_%s' % perf_builder_name, factory_type( 547 F('f_%s' % perf_builder_name, factory_type(
503 builder_name=perf_builder_name, 548 builder_name=perf_builder_name,
504 other_subdirs=subdirs_to_checkout, 549 other_subdirs=subdirs_to_checkout,
505 configuration=skia_factory.CONFIG_RELEASE, 550 configuration='Release',
506 gm_image_subdir=gm_image_subdir, 551 gm_image_subdir=gm_image_subdir,
507 do_patch_step=(try_schedulers is not None), 552 do_patch_step=(try_schedulers is not None),
508 perf_output_basedir=perf_output_basedir, 553 perf_output_basedir=perf_output_basedir,
509 compile_warnings_as_errors=False, 554 compile_warnings_as_errors=False,
510 **kwargs 555 **kwargs
511 ).BuildPerfOnly()) 556 ).BuildPerfOnly())
512 557
513 558
514 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): 559 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs):
515 _MakeBuilderSet(try_schedulers=None, **kwargs) 560 _MakeBuilderSet(try_schedulers=None, **kwargs)
516 if do_trybots: 561 if do_trybots:
517 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) 562 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs)
518 563
519 564
520 def MakeBuilderSet(**kwargs): 565 def MakeBuilderSet(**kwargs):
521 _MakeBuilderAndMaybeTrybotSet(factory_type=skia_factory.SkiaFactory, **kwargs) 566 _MakeBuilderAndMaybeTrybotSet(**kwargs)
522 567
523 568
524 def MakeHousekeeperBuilderSet(helper, do_trybots, do_upload_results): 569 def MakeHousekeeperBuilderSet(helper, percommit_factory_type,
570 periodic_factory_type, do_trybots, **kwargs):
525 B = helper.Builder 571 B = helper.Builder
526 F = helper.Factory 572 F = helper.Factory
527 573
528 builder_factory_scheduler = [ 574 builder_factory_scheduler = [
529 # The Percommit housekeeper 575 # The Percommit housekeeper
530 ('Skia_PerCommit_House_Keeping', 576 ('Skia_PerCommit_House_Keeping',
531 housekeeping_percommit_factory.HouseKeepingPerCommitFactory, 577 percommit_factory_type,
532 'skia_rel'), 578 'skia_rel'),
533 # The Periodic housekeeper 579 # The Periodic housekeeper
534 ('Skia_Periodic_House_Keeping', 580 ('Skia_Periodic_House_Keeping',
535 housekeeping_periodic_factory.HouseKeepingPeriodicFactory, 581 periodic_factory_type,
536 'skia_periodic'), 582 'skia_periodic'),
537 ] 583 ]
538 if do_trybots: 584 if do_trybots:
539 # Add the corresponding trybot builders to the above list. 585 # Add the corresponding trybot builders to the above list.
540 builder_factory_scheduler.extend([ 586 builder_factory_scheduler.extend([
541 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR) 587 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR)
542 for (builder, factory, _scheduler) in builder_factory_scheduler]) 588 for (builder, factory, _scheduler) in builder_factory_scheduler])
543 589
544 for (builder_name, factory, scheduler) in builder_factory_scheduler: 590 for (builder_name, factory, scheduler) in builder_factory_scheduler:
545 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) 591 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler)
546 F('f_%s' % builder_name, 592 F('f_%s' % builder_name,
547 factory( 593 factory(
548 do_upload_results=do_upload_results,
549 target_platform=skia_factory.TARGET_PLATFORM_LINUX,
550 builder_name=builder_name, 594 builder_name=builder_name,
551 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), 595 do_patch_step=(scheduler == TRY_SCHEDULERS_STR),
552 ).Build()) 596 ).Build())
553 597
554
555 def MakeAndroidBuilderSet(extra_branches=None, **kwargs):
556 if not extra_branches:
557 extra_branches = []
558 extra_branches.append('android')
559 _MakeBuilderAndMaybeTrybotSet(factory_type=android_factory.AndroidFactory,
560 extra_branches=extra_branches,
561 **kwargs)
562
563
564 def MakeChromeOSBuilderSet(**kwargs):
565 _MakeBuilderAndMaybeTrybotSet(factory_type=chromeos_factory.ChromeOSFactory,
566 **kwargs)
567
568
569 def MakeIOSBuilderSet(**kwargs):
570 _MakeBuilderAndMaybeTrybotSet(factory_type=ios_factory.iOSFactory, **kwargs)
571
572
573 def MakeNaClBuilderSet(**kwargs):
574 _MakeBuilderAndMaybeTrybotSet(factory_type=nacl_factory.NaClFactory, **kwargs)
575
576
577 def CanMergeBuildRequests(req1, req2): 598 def CanMergeBuildRequests(req1, req2):
578 """ Determine whether or not two BuildRequests can be merged. Note that the 599 """ Determine whether or not two BuildRequests can be merged. Note that the
579 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously 600 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously
580 missing. This is because that method verifies that: 601 missing. This is because that method verifies that:
581 1. req1.source.repository == req2.source.repository 602 1. req1.source.repository == req2.source.repository
582 2. req1.source.project == req2.source.project 603 2. req1.source.project == req2.source.project
583 3. req1.source.branch == req2.source.branch 604 3. req1.source.branch == req2.source.branch
584 4. req1.patch == None and req2.patch = None 605 4. req1.patch == None and req2.patch = None
585 5. (req1.source.changes and req2.source.changes) or \ 606 5. (req1.source.changes and req2.source.changes) or \
586 (not req1.source.changes and not req2.source.changes and \ 607 (not req1.source.changes and not req2.source.changes and \
(...skipping 27 matching lines...) Expand all
614 # request is associated with a change but the revisions match (#5 above). 635 # request is associated with a change but the revisions match (#5 above).
615 if req1.source.changes and not req2.source.changes: 636 if req1.source.changes and not req2.source.changes:
616 return False 637 return False
617 if not req1.source.changes and req2.source.changes: 638 if not req1.source.changes and req2.source.changes:
618 return False 639 return False
619 if not (req1.source.changes and req2.source.changes): 640 if not (req1.source.changes and req2.source.changes):
620 if req1.source.revision != req2.source.revision: 641 if req1.source.revision != req2.source.revision:
621 return False 642 return False
622 643
623 return True 644 return True
OLDNEW
« no previous file with comments | « master/skia_master_scripts/factory.py ('k') | tools/tests/factory_configuration/expected/Skia_ChromeOS_Alex_Bench_32 » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698