OLD | NEW |
---|---|
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 3 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """CBuildbot is wrapper around the build process used by the pre-flight queue""" | 7 """CBuildbot is wrapper around the build process used by the pre-flight queue""" |
8 | 8 |
9 import errno | 9 import errno |
10 import heapq | 10 import heapq |
11 import re | 11 import re |
12 import optparse | 12 import optparse |
13 import os | 13 import os |
14 import shutil | 14 import shutil |
15 import sys | 15 import sys |
16 | 16 |
17 import cbuildbot_comm | 17 import cbuildbot_comm |
18 from cbuildbot_config import config | 18 from cbuildbot_config import config |
19 | 19 |
20 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) | 20 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) |
21 from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand, | 21 from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand, |
22 Warning) | 22 Warning) |
23 | 23 |
24 _DEFAULT_RETRIES = 3 | 24 _DEFAULT_RETRIES = 3 |
25 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list' | 25 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list' |
dianders
2010/11/23 21:47:39
Please add comments about what these are: the name
davidjames
2010/11/23 22:53:14
Done.
| |
26 _FULL_BINHOST = 'PORTAGE_BINHOST' | |
27 _PREFLIGHT_BINHOST = 'PREFLIGHT_BINHOST' | |
26 ARCHIVE_BASE = '/var/www/archive' | 28 ARCHIVE_BASE = '/var/www/archive' |
27 ARCHIVE_COUNT = 10 | 29 ARCHIVE_COUNT = 10 |
28 | 30 |
29 # ======================== Utility functions ================================ | 31 # ======================== Utility functions ================================ |
30 | 32 |
31 def MakeDir(path, parents=False): | 33 def MakeDir(path, parents=False): |
32 """Basic wrapper around os.mkdirs. | 34 """Basic wrapper around os.mkdirs. |
33 | 35 |
34 Keyword arguments: | 36 Keyword arguments: |
35 path -- Path to create. | 37 path -- Path to create. |
(...skipping 255 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
291 """Performs a checkout without clobbering previous checkout.""" | 293 """Performs a checkout without clobbering previous checkout.""" |
292 RepoSync(buildroot, rw_checkout, retries) | 294 RepoSync(buildroot, rw_checkout, retries) |
293 | 295 |
294 | 296 |
295 def _MakeChroot(buildroot): | 297 def _MakeChroot(buildroot): |
296 """Wrapper around make_chroot.""" | 298 """Wrapper around make_chroot.""" |
297 cwd = os.path.join(buildroot, 'src', 'scripts') | 299 cwd = os.path.join(buildroot, 'src', 'scripts') |
298 RunCommand(['./make_chroot', '--fast'], cwd=cwd) | 300 RunCommand(['./make_chroot', '--fast'], cwd=cwd) |
299 | 301 |
300 | 302 |
303 def _GetPortageEnvVar(buildroot, board, envvar): | |
304 """Get the current portage binhost for the specified board, if any. | |
305 | |
dianders
2010/11/23 21:47:39
Describe Args and Returns according to standard.
davidjames
2010/11/23 22:53:14
Done.
| |
306 If no binhost can be found, return the empty string.""" | |
307 cwd = os.path.join(buildroot, 'src', 'scripts') | |
308 binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar], | |
309 cwd=cwd, redirect_stdout=True, enter_chroot=True, | |
310 error_ok=True) | |
311 return binhost.rstrip('\n') | |
312 | |
313 | |
301 def _SetupBoard(buildroot, board='x86-generic'): | 314 def _SetupBoard(buildroot, board='x86-generic'): |
302 """Wrapper around setup_board.""" | 315 """Wrapper around setup_board.""" |
303 cwd = os.path.join(buildroot, 'src', 'scripts') | 316 cwd = os.path.join(buildroot, 'src', 'scripts') |
304 RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board], | 317 RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board], |
305 cwd=cwd, enter_chroot=True) | 318 cwd=cwd, enter_chroot=True) |
306 | 319 |
307 | 320 |
308 def _Build(buildroot): | 321 def _Build(buildroot): |
309 """Wrapper around build_packages.""" | 322 """Wrapper around build_packages.""" |
310 cwd = os.path.join(buildroot, 'src', 'scripts') | 323 cwd = os.path.join(buildroot, 'src', 'scripts') |
(...skipping 178 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
489 paths = [private_overlay] | 502 paths = [private_overlay] |
490 elif overlays == 'public': | 503 elif overlays == 'public': |
491 paths = [public_overlay] | 504 paths = [public_overlay] |
492 elif overlays == 'both': | 505 elif overlays == 'both': |
493 paths = [public_overlay, private_overlay] | 506 paths = [public_overlay, private_overlay] |
494 else: | 507 else: |
495 Die('Incorrect overlay configuration: %s' % overlays) | 508 Die('Incorrect overlay configuration: %s' % overlays) |
496 return paths | 509 return paths |
497 | 510 |
498 | 511 |
512 def _UploadPrebuilts(buildroot, board, overlay_config, binhosts): | |
513 """Upload prebuilts. | |
514 | |
515 Args: | |
516 buildroot: The root directory where the build occurs. | |
517 board: Board type that was built on this machine | |
518 overlay_config: A string describing which overlays you want. | |
519 'private': Just the private overlay. | |
520 'public': Just the public overlay. | |
521 'both': Both the public and private overlays. | |
522 binhosts: The URLs of the current binhosts. Binaries that are already | |
523 present will not be uploaded twice. | |
dianders
2010/11/23 21:47:39
Add comment about binhosts:
Note: it's expected t
| |
524 """ | |
525 | |
526 cmd = ['%s/src/scripts/prebuilt.py' % buildroot, '--git-sync', | |
527 '--build-path', buildroot, '--board', board, | |
528 '--prepend-version', 'preflight', '--key', _PREFLIGHT_BINHOST] | |
dianders
2010/11/23 21:47:39
I'm curious why you don't just import a function f
davidjames
2010/11/23 22:53:14
That's a good point.
I like having a little separ
| |
529 for binhost in binhosts: | |
530 if binhost: | |
531 cmd.extend(['--previous-board-binhost-url', binhost]) | |
532 if overlay_config == 'public': | |
533 cmd.extend(['--upload', 'gs://chromeos-prebuilt']) | |
534 else: | |
535 assert overlay_config in ('private', 'both') | |
536 cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/', | |
537 '--binhost-base-url', 'http://chromeos-prebuilt']) | |
538 | |
539 RunCommand(cmd) | |
540 | |
541 | |
499 def main(): | 542 def main(): |
500 # Parse options | 543 # Parse options |
501 usage = "usage: %prog [options] cbuildbot_config" | 544 usage = "usage: %prog [options] cbuildbot_config" |
502 parser = optparse.OptionParser(usage=usage) | 545 parser = optparse.OptionParser(usage=usage) |
503 parser.add_option('-r', '--buildroot', | 546 parser.add_option('-r', '--buildroot', |
504 help='root directory where build occurs', default=".") | 547 help='root directory where build occurs', default=".") |
505 parser.add_option('-n', '--buildnumber', | 548 parser.add_option('-n', '--buildnumber', |
506 help='build number', type='int', default=0) | 549 help='build number', type='int', default=0) |
507 parser.add_option('-f', '--revisionfile', | 550 parser.add_option('-f', '--revisionfile', |
508 help='file where new revisions are stored') | 551 help='file where new revisions are stored') |
(...skipping 17 matching lines...) Expand all Loading... | |
526 | 569 |
527 if len(args) >= 1: | 570 if len(args) >= 1: |
528 buildconfig = _GetConfig(args[-1]) | 571 buildconfig = _GetConfig(args[-1]) |
529 else: | 572 else: |
530 Warning('Missing configuration description') | 573 Warning('Missing configuration description') |
531 parser.print_usage() | 574 parser.print_usage() |
532 sys.exit(1) | 575 sys.exit(1) |
533 | 576 |
534 # Calculate list of overlay directories. | 577 # Calculate list of overlay directories. |
535 overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) | 578 overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) |
579 board = buildconfig['board'] | |
536 | 580 |
537 try: | 581 try: |
538 _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) | 582 _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) |
583 chroot_path = os.path.join(buildroot, 'chroot') | |
584 boardpath = os.path.join(chroot_path, 'build', board) | |
539 if options.clobber or not os.path.isdir(buildroot): | 585 if options.clobber or not os.path.isdir(buildroot): |
540 _FullCheckout(buildroot, tracking_branch, url=options.url) | 586 _FullCheckout(buildroot, tracking_branch, url=options.url) |
541 else: | 587 else: |
588 old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) | |
542 _IncrementalCheckout(buildroot) | 589 _IncrementalCheckout(buildroot) |
590 new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) | |
591 if old_binhost != new_binhost: | |
592 RunCommand(['sudo', 'rm', '-rf', boardpath]) | |
543 | 593 |
544 # Check that all overlays can be found. | 594 # Check that all overlays can be found. |
545 for path in overlays: | 595 for path in overlays: |
546 assert ':' not in path, 'Overlay must not contain colons: %s' % path | 596 assert ':' not in path, 'Overlay must not contain colons: %s' % path |
547 if not os.path.isdir(path): | 597 if not os.path.isdir(path): |
548 Die('Missing overlay: %s' % path) | 598 Die('Missing overlay: %s' % path) |
549 | 599 |
550 chroot_path = os.path.join(buildroot, 'chroot') | |
551 if not os.path.isdir(chroot_path): | 600 if not os.path.isdir(chroot_path): |
552 _MakeChroot(buildroot) | 601 _MakeChroot(buildroot) |
553 | 602 |
554 boardpath = os.path.join(chroot_path, 'build', buildconfig['board']) | |
555 if not os.path.isdir(boardpath): | 603 if not os.path.isdir(boardpath): |
556 _SetupBoard(buildroot, board=buildconfig['board']) | 604 _SetupBoard(buildroot, board=buildconfig['board']) |
557 | 605 |
558 if buildconfig['uprev']: | 606 if buildconfig['uprev']: |
559 _UprevPackages(buildroot, tracking_branch, revisionfile, | 607 _UprevPackages(buildroot, tracking_branch, revisionfile, |
560 buildconfig['board'], overlays) | 608 buildconfig['board'], overlays) |
561 | 609 |
562 _EnableLocalAccount(buildroot) | 610 _EnableLocalAccount(buildroot) |
563 _Build(buildroot) | 611 _Build(buildroot) |
564 if buildconfig['unittests']: | 612 if buildconfig['unittests']: |
(...skipping 10 matching lines...) Expand all Loading... | |
575 _ArchiveTestResults(buildroot, buildconfig['board'], | 623 _ArchiveTestResults(buildroot, buildconfig['board'], |
576 archive_dir=options.buildnumber, | 624 archive_dir=options.buildnumber, |
577 test_results_dir=test_results_dir) | 625 test_results_dir=test_results_dir) |
578 | 626 |
579 if buildconfig['uprev']: | 627 if buildconfig['uprev']: |
580 # Don't push changes for developers. | 628 # Don't push changes for developers. |
581 if not options.debug: | 629 if not options.debug: |
582 if buildconfig['master']: | 630 if buildconfig['master']: |
583 # Master bot needs to check if the other slaves completed. | 631 # Master bot needs to check if the other slaves completed. |
584 if cbuildbot_comm.HaveSlavesCompleted(config): | 632 if cbuildbot_comm.HaveSlavesCompleted(config): |
633 binhosts = [ | |
634 _GetPortageEnvVar(buildroot, board, _PREFLIGHT_BINHOST), | |
635 _GetPortageEnvVar(buildroot, board, _FULL_BINHOST), | |
636 ] | |
585 _UprevPush(buildroot, tracking_branch, buildconfig['board'], | 637 _UprevPush(buildroot, tracking_branch, buildconfig['board'], |
586 overlays) | 638 overlays) |
639 _UploadPrebuilts(buildroot, board, buildconfig['overlays'], | |
640 binhosts) | |
587 else: | 641 else: |
588 Die('CBUILDBOT - One of the slaves has failed!!!') | 642 Die('CBUILDBOT - One of the slaves has failed!!!') |
589 | 643 |
590 else: | 644 else: |
591 # Publish my status to the master if its expecting it. | 645 # Publish my status to the master if its expecting it. |
592 if buildconfig['important']: | 646 if buildconfig['important']: |
593 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE) | 647 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE) |
594 | |
595 except: | 648 except: |
596 # Send failure to master bot. | 649 # Send failure to master bot. |
597 if not buildconfig['master'] and buildconfig['important']: | 650 if not buildconfig['master'] and buildconfig['important']: |
598 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED) | 651 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED) |
599 | 652 |
600 raise | 653 raise |
601 | 654 |
602 | 655 |
603 if __name__ == '__main__': | 656 if __name__ == '__main__': |
604 main() | 657 main() |
OLD | NEW |