Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(149)

Side by Side Diff: master/skia_master_scripts/utils.py

Issue 14081044: Add Validation for BuildFactory Configuration (Closed) Base URL: http://skia.googlecode.com/svn/buildbot/
Patch Set: Created 7 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 5
6 """Miscellaneous utilities needed by the Skia buildbot master.""" 6 """Miscellaneous utilities needed by the Skia buildbot master."""
7 7
8 8
9 import httplib2 9 import httplib2
10 import re 10 import re
11 11
12 # requires Google APIs client library for Python; see 12 # requires Google APIs client library for Python; see
13 # https://code.google.com/p/google-api-python-client/wiki/Installation 13 # https://code.google.com/p/google-api-python-client/wiki/Installation
14 from apiclient.discovery import build 14 from apiclient.discovery import build
15 from buildbot.scheduler import AnyBranchScheduler 15 from buildbot.scheduler import AnyBranchScheduler
16 from buildbot.schedulers import timed 16 from buildbot.schedulers import timed
17 from buildbot.schedulers.filter import ChangeFilter 17 from buildbot.schedulers.filter import ChangeFilter
18 from buildbot.util import NotABranch 18 from buildbot.util import NotABranch
19 from config_private import TRY_SVN_BASEURL 19 from config_private import TRY_SVN_BASEURL
20 from master import master_config 20 from master import master_config
21 from master import try_job_svn 21 from master import try_job_svn
22 from master import try_job_rietveld 22 from master import try_job_rietveld
23 from master.builders_pools import BuildersPools 23 from master.builders_pools import BuildersPools
24 from oauth2client.client import SignedJwtAssertionCredentials 24 from oauth2client.client import SignedJwtAssertionCredentials
25 25
26 from skia_master_scripts import android_factory
27 from skia_master_scripts import chromeos_factory
28 from skia_master_scripts import factory as skia_factory
29 from skia_master_scripts import housekeeping_percommit_factory, \
30 housekeeping_periodic_factory
31 from skia_master_scripts import ios_factory
32 from skia_master_scripts import nacl_factory
33
34 import config_private 26 import config_private
35 27
36 28
37 CATEGORY_BUILD = ' Build' 29 CATEGORY_BUILD = ' Build'
38 TRYBOT_NAME_SUFFIX = '_Trybot' 30 TRYBOT_NAME_SUFFIX = '_Trybot'
39 TRY_SCHEDULER_SVN = 'skia_try_svn' 31 TRY_SCHEDULER_SVN = 'skia_try_svn'
40 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld' 32 TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld'
41 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD] 33 TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD]
42 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS) 34 TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS)
43 35
44 36
45 def IsTrybot(builder_name): 37 def IsTrybot(builder_name):
46 return builder_name.endswith(TRYBOT_NAME_SUFFIX) 38 return builder_name.endswith(TRYBOT_NAME_SUFFIX)
47 39
48 40
41 def _IndentStr(indent):
42 string = ''
epoger 2013/04/25 18:12:29 I think you can implement this as: return ' '
borenet 2013/04/25 18:28:09 Done.
43 for _ in range(indent + 1):
44 string += ' '
45 return string
46
47
48 def ToString(o, indent=0):
49 """ Returns a string representation of the given object. This differs from the
50 built-in string function in that it does not give memory locations.
51 z
epoger 2013/04/25 18:12:29 z ???
borenet 2013/04/25 18:28:09 Done.
52 o: the object to print.
epoger 2013/04/25 18:12:29 suggest o -> obj
borenet 2013/04/25 18:28:09 Done.
53 """
54 if isinstance(o, list):
55 return _ListToString(o, indent)
56 elif isinstance(o, dict):
57 return _DictToString(o, indent)
58 elif isinstance(o, tuple):
59 return _ListToString(o, indent)
60 elif isinstance(o, str):
61 return '\'%s\'' % o
62 elif o is None:
63 return 'None'
64 else:
65 return '<Object>'
66
67
68 def _ListToString(l, indent):
epoger 2013/04/25 18:12:29 suggest "list" or a variable name like that; "l" l
borenet 2013/04/25 18:28:09 Changed to list_var to avoid collision with built-
69 if not l:
70 return '[]'
71 indent_str = _IndentStr(indent)
72 val = '[\n'
73 indent += 1
74 val += ''.join(['%s%s,\n' % (indent_str, ToString(elem, indent)) \
75 for elem in l])
76 indent -= 1
77 indent_str = _IndentStr(indent - 1)
78 val += indent_str + ']'
79 return val
80
81
82 def _DictToString(d, indent):
83 if not d:
84 return '{}'
85 indent_str = _IndentStr(indent)
86 val = '{\n'
87 indent += 1
88 val += ''.join(['%s%s: %s,\n' % (indent_str, ToString(k, indent),
89 ToString(d[k], indent)) \
90 for k in sorted(d.keys())])
91 indent -= 1
92 indent_str = _IndentStr(indent - 1)
93 val += indent_str + '}'
94 return val
95
96
49 class SkiaChangeFilter(ChangeFilter): 97 class SkiaChangeFilter(ChangeFilter):
50 """Skia specific subclass of ChangeFilter.""" 98 """Skia specific subclass of ChangeFilter."""
51 99
52 def __init__(self, builders, **kwargs): 100 def __init__(self, builders, **kwargs):
53 self._builders = builders 101 self._builders = builders
54 ChangeFilter.__init__(self, **kwargs) 102 ChangeFilter.__init__(self, **kwargs)
55 103
56 def filter_change(self, change): 104 def filter_change(self, change):
57 """Overrides ChangeFilter.filter_change to pass builders to filter_fn. 105 """Overrides ChangeFilter.filter_change to pass builders to filter_fn.
58 106
(...skipping 321 matching lines...) Expand 10 before | Expand all | Expand 10 after
380 428
381 def MakeCompileBuilderName(builder_base_name, release=False): 429 def MakeCompileBuilderName(builder_base_name, release=False):
382 if release: 430 if release:
383 compile_name = 'Compile_Release' 431 compile_name = 'Compile_Release'
384 else: 432 else:
385 compile_name = 'Compile_Debug' 433 compile_name = 'Compile_Debug'
386 return MakeBuilderName(builder_base_name, compile_name) 434 return MakeBuilderName(builder_base_name, compile_name)
387 435
388 436
389 def MakeDebugBuilderName(builder_base_name): 437 def MakeDebugBuilderName(builder_base_name):
390 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_DEBUG) 438 return MakeBuilderName(builder_base_name, 'Debug')
391 439
392 440
393 def MakeReleaseBuilderName(builder_base_name): 441 def MakeReleaseBuilderName(builder_base_name):
394 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_RELEASE) 442 return MakeBuilderName(builder_base_name, 'Release')
395 443
396 444
397 def MakeBenchBuilderName(builder_base_name): 445 def MakeBenchBuilderName(builder_base_name):
398 return MakeBuilderName(builder_base_name, skia_factory.CONFIG_BENCH) 446 return MakeBuilderName(builder_base_name, 'Bench')
399 447
400 448
401 def MakeSchedulerName(builder_base_name): 449 def MakeSchedulerName(builder_base_name):
402 return MakeBuilderName(builder_base_name, 'Scheduler') 450 return MakeBuilderName(builder_base_name, 'Scheduler')
403 451
404 452
405 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, 453 def _MakeBuilderSet(helper, builder_base_name, gm_image_subdir, factory_type,
406 perf_output_basedir=None, extra_branches=None, 454 perf_output_basedir=None, extra_branches=None,
407 factory_type=None, do_compile=True, do_debug=True, 455 do_compile=True, do_debug=True, do_release=True,
408 do_release=True, do_bench=True, try_schedulers=None, 456 do_bench=True, try_schedulers=None,
409 compile_bot_warnings_as_errors=True, 457 compile_bot_warnings_as_errors=True, **kwargs):
410 **kwargs):
411 """ Creates a trio of builders for a given platform: 458 """ Creates a trio of builders for a given platform:
412 1. Debug mode builder which runs all steps 459 1. Debug mode builder which runs all steps
413 2. Release mode builder which runs all steps EXCEPT benchmarks 460 2. Release mode builder which runs all steps EXCEPT benchmarks
414 3. Release mode builder which runs ONLY benchmarks. 461 3. Release mode builder which runs ONLY benchmarks.
415 """ 462 """
416 B = helper.Builder 463 B = helper.Builder
417 F = helper.Factory 464 F = helper.Factory
418 465
419 if not extra_branches: 466 if not extra_branches:
420 extra_branches = [] 467 extra_branches = []
(...skipping 13 matching lines...) Expand all
434 if do_compile: 481 if do_compile:
435 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name, 482 compile_debug_builder_name = MakeCompileBuilderName(builder_base_name,
436 release=False) 483 release=False)
437 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name, 484 B(compile_debug_builder_name, 'f_%s' % compile_debug_builder_name,
438 # Do not add gatekeeper for trybots. 485 # Do not add gatekeeper for trybots.
439 gatekeeper='GateKeeper' if try_schedulers is None else None, 486 gatekeeper='GateKeeper' if try_schedulers is None else None,
440 scheduler=scheduler_name, override_category=CATEGORY_BUILD) 487 scheduler=scheduler_name, override_category=CATEGORY_BUILD)
441 F('f_%s' % compile_debug_builder_name, factory_type( 488 F('f_%s' % compile_debug_builder_name, factory_type(
442 builder_name=compile_debug_builder_name, 489 builder_name=compile_debug_builder_name,
443 other_subdirs=subdirs_to_checkout, 490 other_subdirs=subdirs_to_checkout,
444 configuration=skia_factory.CONFIG_DEBUG, 491 configuration='Debug',
epoger 2013/04/25 18:12:29 I think there's value in using constants to hold t
borenet 2013/04/25 18:28:09 Agreed. Unfortunately, I can't use the constants
445 gm_image_subdir=gm_image_subdir, 492 gm_image_subdir=gm_image_subdir,
446 do_patch_step=(try_schedulers is not None), 493 do_patch_step=(try_schedulers is not None),
447 perf_output_basedir=None, 494 perf_output_basedir=None,
448 compile_warnings_as_errors=compile_bot_warnings_as_errors, 495 compile_warnings_as_errors=compile_bot_warnings_as_errors,
449 **kwargs 496 **kwargs
450 ).BuildCompileOnly()) 497 ).BuildCompileOnly())
451 compile_release_builder_name = MakeCompileBuilderName(builder_base_name, 498 compile_release_builder_name = MakeCompileBuilderName(builder_base_name,
452 release=True) 499 release=True)
453 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name, 500 B(compile_release_builder_name, 'f_%s' % compile_release_builder_name,
454 # Do not add gatekeeper for trybots. 501 # Do not add gatekeeper for trybots.
455 gatekeeper='GateKeeper' if try_schedulers is None else None, 502 gatekeeper='GateKeeper' if try_schedulers is None else None,
456 scheduler=scheduler_name, override_category=CATEGORY_BUILD) 503 scheduler=scheduler_name, override_category=CATEGORY_BUILD)
457 F('f_%s' % compile_release_builder_name, factory_type( 504 F('f_%s' % compile_release_builder_name, factory_type(
458 builder_name=compile_release_builder_name, 505 builder_name=compile_release_builder_name,
459 other_subdirs=subdirs_to_checkout, 506 other_subdirs=subdirs_to_checkout,
460 configuration=skia_factory.CONFIG_RELEASE, 507 configuration='Release',
461 gm_image_subdir=gm_image_subdir, 508 gm_image_subdir=gm_image_subdir,
462 do_patch_step=(try_schedulers is not None), 509 do_patch_step=(try_schedulers is not None),
463 perf_output_basedir=None, 510 perf_output_basedir=None,
464 compile_warnings_as_errors=compile_bot_warnings_as_errors, 511 compile_warnings_as_errors=compile_bot_warnings_as_errors,
465 **kwargs 512 **kwargs
466 ).BuildCompileOnly()) 513 ).BuildCompileOnly())
467 514
468 if do_debug: 515 if do_debug:
469 debug_builder_name = MakeDebugBuilderName(builder_base_name) 516 debug_builder_name = MakeDebugBuilderName(builder_base_name)
470 B(debug_builder_name, 'f_%s' % debug_builder_name, 517 B(debug_builder_name, 'f_%s' % debug_builder_name,
471 scheduler=scheduler_name) 518 scheduler=scheduler_name)
472 F('f_%s' % debug_builder_name, factory_type( 519 F('f_%s' % debug_builder_name, factory_type(
473 builder_name=debug_builder_name, 520 builder_name=debug_builder_name,
474 other_subdirs=subdirs_to_checkout, 521 other_subdirs=subdirs_to_checkout,
475 configuration=skia_factory.CONFIG_DEBUG, 522 configuration='Debug',
476 gm_image_subdir=gm_image_subdir, 523 gm_image_subdir=gm_image_subdir,
477 do_patch_step=(try_schedulers is not None), 524 do_patch_step=(try_schedulers is not None),
478 perf_output_basedir=None, 525 perf_output_basedir=None,
479 compile_warnings_as_errors=False, 526 compile_warnings_as_errors=False,
480 **kwargs 527 **kwargs
481 ).Build()) 528 ).Build())
482 529
483 if do_release: 530 if do_release:
484 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name) 531 no_perf_builder_name = MakeReleaseBuilderName(builder_base_name)
485 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name, 532 B(no_perf_builder_name, 'f_%s' % no_perf_builder_name,
486 scheduler=scheduler_name) 533 scheduler=scheduler_name)
487 F('f_%s' % no_perf_builder_name, factory_type( 534 F('f_%s' % no_perf_builder_name, factory_type(
488 builder_name=no_perf_builder_name, 535 builder_name=no_perf_builder_name,
489 other_subdirs=subdirs_to_checkout, 536 other_subdirs=subdirs_to_checkout,
490 configuration=skia_factory.CONFIG_RELEASE, 537 configuration='Release',
491 gm_image_subdir=gm_image_subdir, 538 gm_image_subdir=gm_image_subdir,
492 do_patch_step=(try_schedulers is not None), 539 do_patch_step=(try_schedulers is not None),
493 perf_output_basedir=None, 540 perf_output_basedir=None,
494 compile_warnings_as_errors=False, 541 compile_warnings_as_errors=False,
495 **kwargs 542 **kwargs
496 ).BuildNoPerf()) 543 ).BuildNoPerf())
497 544
498 if do_bench: 545 if do_bench:
499 perf_builder_name = MakeBenchBuilderName(builder_base_name) 546 perf_builder_name = MakeBenchBuilderName(builder_base_name)
500 B(perf_builder_name, 'f_%s' % perf_builder_name, 547 B(perf_builder_name, 'f_%s' % perf_builder_name,
501 scheduler=scheduler_name) 548 scheduler=scheduler_name)
502 F('f_%s' % perf_builder_name, factory_type( 549 F('f_%s' % perf_builder_name, factory_type(
503 builder_name=perf_builder_name, 550 builder_name=perf_builder_name,
504 other_subdirs=subdirs_to_checkout, 551 other_subdirs=subdirs_to_checkout,
505 configuration=skia_factory.CONFIG_RELEASE, 552 configuration='Release',
506 gm_image_subdir=gm_image_subdir, 553 gm_image_subdir=gm_image_subdir,
507 do_patch_step=(try_schedulers is not None), 554 do_patch_step=(try_schedulers is not None),
508 perf_output_basedir=perf_output_basedir, 555 perf_output_basedir=perf_output_basedir,
509 compile_warnings_as_errors=False, 556 compile_warnings_as_errors=False,
510 **kwargs 557 **kwargs
511 ).BuildPerfOnly()) 558 ).BuildPerfOnly())
512 559
513 560
514 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs): 561 def _MakeBuilderAndMaybeTrybotSet(do_trybots=True, **kwargs):
515 _MakeBuilderSet(try_schedulers=None, **kwargs) 562 _MakeBuilderSet(try_schedulers=None, **kwargs)
516 if do_trybots: 563 if do_trybots:
517 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs) 564 _MakeBuilderSet(try_schedulers=TRY_SCHEDULERS, **kwargs)
518 565
519 566
520 def MakeBuilderSet(**kwargs): 567 def MakeBuilderSet(**kwargs):
521 _MakeBuilderAndMaybeTrybotSet(factory_type=skia_factory.SkiaFactory, **kwargs) 568 _MakeBuilderAndMaybeTrybotSet(**kwargs)
522 569
523 570
524 def MakeHousekeeperBuilderSet(helper, do_trybots, do_upload_results): 571 def MakeHousekeeperBuilderSet(helper, percommit_factory_type,
572 periodic_factory_type, do_trybots, **kwargs):
525 B = helper.Builder 573 B = helper.Builder
526 F = helper.Factory 574 F = helper.Factory
527 575
528 builder_factory_scheduler = [ 576 builder_factory_scheduler = [
529 # The Percommit housekeeper 577 # The Percommit housekeeper
530 ('Skia_PerCommit_House_Keeping', 578 ('Skia_PerCommit_House_Keeping',
531 housekeeping_percommit_factory.HouseKeepingPerCommitFactory, 579 percommit_factory_type,
532 'skia_rel'), 580 'skia_rel'),
533 # The Periodic housekeeper 581 # The Periodic housekeeper
534 ('Skia_Periodic_House_Keeping', 582 ('Skia_Periodic_House_Keeping',
535 housekeeping_periodic_factory.HouseKeepingPeriodicFactory, 583 periodic_factory_type,
536 'skia_periodic'), 584 'skia_periodic'),
537 ] 585 ]
538 if do_trybots: 586 if do_trybots:
539 # Add the corresponding trybot builders to the above list. 587 # Add the corresponding trybot builders to the above list.
540 builder_factory_scheduler.extend([ 588 builder_factory_scheduler.extend([
541 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR) 589 (builder + TRYBOT_NAME_SUFFIX, factory, TRY_SCHEDULERS_STR)
542 for (builder, factory, _scheduler) in builder_factory_scheduler]) 590 for (builder, factory, _scheduler) in builder_factory_scheduler])
543 591
544 for (builder_name, factory, scheduler) in builder_factory_scheduler: 592 for (builder_name, factory, scheduler) in builder_factory_scheduler:
545 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler) 593 B(builder_name, 'f_%s' % builder_name, scheduler=scheduler)
546 F('f_%s' % builder_name, 594 F('f_%s' % builder_name,
547 factory( 595 factory(
548 do_upload_results=do_upload_results,
549 target_platform=skia_factory.TARGET_PLATFORM_LINUX,
550 builder_name=builder_name, 596 builder_name=builder_name,
551 do_patch_step=(scheduler == TRY_SCHEDULERS_STR), 597 do_patch_step=(scheduler == TRY_SCHEDULERS_STR),
552 ).Build()) 598 ).Build())
553 599
554
555 def MakeAndroidBuilderSet(extra_branches=None, **kwargs):
556 if not extra_branches:
557 extra_branches = []
558 extra_branches.append('android')
559 _MakeBuilderAndMaybeTrybotSet(factory_type=android_factory.AndroidFactory,
560 extra_branches=extra_branches,
561 **kwargs)
562
563
564 def MakeChromeOSBuilderSet(**kwargs):
565 _MakeBuilderAndMaybeTrybotSet(factory_type=chromeos_factory.ChromeOSFactory,
566 **kwargs)
567
568
569 def MakeIOSBuilderSet(**kwargs):
570 _MakeBuilderAndMaybeTrybotSet(factory_type=ios_factory.iOSFactory, **kwargs)
571
572
573 def MakeNaClBuilderSet(**kwargs):
574 _MakeBuilderAndMaybeTrybotSet(factory_type=nacl_factory.NaClFactory, **kwargs)
575
576
577 def CanMergeBuildRequests(req1, req2): 600 def CanMergeBuildRequests(req1, req2):
578 """ Determine whether or not two BuildRequests can be merged. Note that the 601 """ Determine whether or not two BuildRequests can be merged. Note that the
579 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously 602 call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously
580 missing. This is because that method verifies that: 603 missing. This is because that method verifies that:
581 1. req1.source.repository == req2.source.repository 604 1. req1.source.repository == req2.source.repository
582 2. req1.source.project == req2.source.project 605 2. req1.source.project == req2.source.project
583 3. req1.source.branch == req2.source.branch 606 3. req1.source.branch == req2.source.branch
584 4. req1.patch == None and req2.patch = None 607 4. req1.patch == None and req2.patch = None
585 5. (req1.source.changes and req2.source.changes) or \ 608 5. (req1.source.changes and req2.source.changes) or \
586 (not req1.source.changes and not req2.source.changes and \ 609 (not req1.source.changes and not req2.source.changes and \
(...skipping 27 matching lines...) Expand all
614 # request is associated with a change but the revisions match (#5 above). 637 # request is associated with a change but the revisions match (#5 above).
615 if req1.source.changes and not req2.source.changes: 638 if req1.source.changes and not req2.source.changes:
616 return False 639 return False
617 if not req1.source.changes and req2.source.changes: 640 if not req1.source.changes and req2.source.changes:
618 return False 641 return False
619 if not (req1.source.changes and req2.source.changes): 642 if not (req1.source.changes and req2.source.changes):
620 if req1.source.revision != req2.source.revision: 643 if req1.source.revision != req2.source.revision:
621 return False 644 return False
622 645
623 return True 646 return True
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698