Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(195)

Side by Side Diff: bin/cbuildbot.py

Issue 4969003: Update cbuildbot.py to upload prebuilts from preflight buildbot. (Closed) Base URL: ssh://git@gitrw.chromium.org:9222/crosutils.git@master
Patch Set: Add lots more unit tests. Created 10 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | bin/cbuildbot_unittest.py » ('j') | chromite/lib/binpkg.py » ('J')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 2
3 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. 3 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be 4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file. 5 # found in the LICENSE file.
6 6
7 """CBuildbot is wrapper around the build process used by the pre-flight queue""" 7 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
8 8
9 import errno 9 import errno
10 import heapq 10 import heapq
11 import re 11 import re
12 import optparse 12 import optparse
13 import os 13 import os
14 import shutil 14 import shutil
15 import sys 15 import sys
16 16
17 import cbuildbot_comm 17 import cbuildbot_comm
18 from cbuildbot_config import config 18 from cbuildbot_config import config
19 19
20 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) 20 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
21 from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand, 21 from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand,
22 Warning) 22 Warning)
23 23
24 _DEFAULT_RETRIES = 3 24 _DEFAULT_RETRIES = 3
25 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list' 25 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
dianders 2010/11/23 21:47:39 Please add comments about what these are: the name
davidjames 2010/11/23 22:53:14 Done.
26 _FULL_BINHOST = 'PORTAGE_BINHOST'
27 _PREFLIGHT_BINHOST = 'PREFLIGHT_BINHOST'
26 ARCHIVE_BASE = '/var/www/archive' 28 ARCHIVE_BASE = '/var/www/archive'
27 ARCHIVE_COUNT = 10 29 ARCHIVE_COUNT = 10
28 30
29 # ======================== Utility functions ================================ 31 # ======================== Utility functions ================================
30 32
31 def MakeDir(path, parents=False): 33 def MakeDir(path, parents=False):
32 """Basic wrapper around os.mkdirs. 34 """Basic wrapper around os.mkdirs.
33 35
34 Keyword arguments: 36 Keyword arguments:
35 path -- Path to create. 37 path -- Path to create.
(...skipping 255 matching lines...) Expand 10 before | Expand all | Expand 10 after
291 """Performs a checkout without clobbering previous checkout.""" 293 """Performs a checkout without clobbering previous checkout."""
292 RepoSync(buildroot, rw_checkout, retries) 294 RepoSync(buildroot, rw_checkout, retries)
293 295
294 296
295 def _MakeChroot(buildroot): 297 def _MakeChroot(buildroot):
296 """Wrapper around make_chroot.""" 298 """Wrapper around make_chroot."""
297 cwd = os.path.join(buildroot, 'src', 'scripts') 299 cwd = os.path.join(buildroot, 'src', 'scripts')
298 RunCommand(['./make_chroot', '--fast'], cwd=cwd) 300 RunCommand(['./make_chroot', '--fast'], cwd=cwd)
299 301
300 302
303 def _GetPortageEnvVar(buildroot, board, envvar):
304 """Get the current portage binhost for the specified board, if any.
305
dianders 2010/11/23 21:47:39 Describe Args and Returns according to standard.
davidjames 2010/11/23 22:53:14 Done.
306 If no binhost can be found, return the empty string."""
307 cwd = os.path.join(buildroot, 'src', 'scripts')
308 binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar],
309 cwd=cwd, redirect_stdout=True, enter_chroot=True,
310 error_ok=True)
311 return binhost.rstrip('\n')
312
313
301 def _SetupBoard(buildroot, board='x86-generic'): 314 def _SetupBoard(buildroot, board='x86-generic'):
302 """Wrapper around setup_board.""" 315 """Wrapper around setup_board."""
303 cwd = os.path.join(buildroot, 'src', 'scripts') 316 cwd = os.path.join(buildroot, 'src', 'scripts')
304 RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board], 317 RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
305 cwd=cwd, enter_chroot=True) 318 cwd=cwd, enter_chroot=True)
306 319
307 320
308 def _Build(buildroot): 321 def _Build(buildroot):
309 """Wrapper around build_packages.""" 322 """Wrapper around build_packages."""
310 cwd = os.path.join(buildroot, 'src', 'scripts') 323 cwd = os.path.join(buildroot, 'src', 'scripts')
(...skipping 178 matching lines...) Expand 10 before | Expand all | Expand 10 after
489 paths = [private_overlay] 502 paths = [private_overlay]
490 elif overlays == 'public': 503 elif overlays == 'public':
491 paths = [public_overlay] 504 paths = [public_overlay]
492 elif overlays == 'both': 505 elif overlays == 'both':
493 paths = [public_overlay, private_overlay] 506 paths = [public_overlay, private_overlay]
494 else: 507 else:
495 Die('Incorrect overlay configuration: %s' % overlays) 508 Die('Incorrect overlay configuration: %s' % overlays)
496 return paths 509 return paths
497 510
498 511
512 def _UploadPrebuilts(buildroot, board, overlay_config, binhosts):
513 """Upload prebuilts.
514
515 Args:
516 buildroot: The root directory where the build occurs.
517 board: Board type that was built on this machine
518 overlay_config: A string describing which overlays you want.
519 'private': Just the private overlay.
520 'public': Just the public overlay.
521 'both': Both the public and private overlays.
522 binhosts: The URLs of the current binhosts. Binaries that are already
523 present will not be uploaded twice.
dianders 2010/11/23 21:47:39 Add comment about binhosts: Note: it's expected t
524 """
525
526 cmd = ['%s/src/scripts/prebuilt.py' % buildroot, '--git-sync',
527 '--build-path', buildroot, '--board', board,
528 '--prepend-version', 'preflight', '--key', _PREFLIGHT_BINHOST]
dianders 2010/11/23 21:47:39 I'm curious why you don't just import a function f
davidjames 2010/11/23 22:53:14 That's a good point. I like having a little separ
529 for binhost in binhosts:
530 if binhost:
531 cmd.extend(['--previous-board-binhost-url', binhost])
532 if overlay_config == 'public':
533 cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
534 else:
535 assert overlay_config in ('private', 'both')
536 cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
537 '--binhost-base-url', 'http://chromeos-prebuilt'])
538
539 RunCommand(cmd)
540
541
499 def main(): 542 def main():
500 # Parse options 543 # Parse options
501 usage = "usage: %prog [options] cbuildbot_config" 544 usage = "usage: %prog [options] cbuildbot_config"
502 parser = optparse.OptionParser(usage=usage) 545 parser = optparse.OptionParser(usage=usage)
503 parser.add_option('-r', '--buildroot', 546 parser.add_option('-r', '--buildroot',
504 help='root directory where build occurs', default=".") 547 help='root directory where build occurs', default=".")
505 parser.add_option('-n', '--buildnumber', 548 parser.add_option('-n', '--buildnumber',
506 help='build number', type='int', default=0) 549 help='build number', type='int', default=0)
507 parser.add_option('-f', '--revisionfile', 550 parser.add_option('-f', '--revisionfile',
508 help='file where new revisions are stored') 551 help='file where new revisions are stored')
(...skipping 17 matching lines...) Expand all
526 569
527 if len(args) >= 1: 570 if len(args) >= 1:
528 buildconfig = _GetConfig(args[-1]) 571 buildconfig = _GetConfig(args[-1])
529 else: 572 else:
530 Warning('Missing configuration description') 573 Warning('Missing configuration description')
531 parser.print_usage() 574 parser.print_usage()
532 sys.exit(1) 575 sys.exit(1)
533 576
534 # Calculate list of overlay directories. 577 # Calculate list of overlay directories.
535 overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) 578 overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
579 board = buildconfig['board']
536 580
537 try: 581 try:
538 _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) 582 _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
583 chroot_path = os.path.join(buildroot, 'chroot')
584 boardpath = os.path.join(chroot_path, 'build', board)
539 if options.clobber or not os.path.isdir(buildroot): 585 if options.clobber or not os.path.isdir(buildroot):
540 _FullCheckout(buildroot, tracking_branch, url=options.url) 586 _FullCheckout(buildroot, tracking_branch, url=options.url)
541 else: 587 else:
588 old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
542 _IncrementalCheckout(buildroot) 589 _IncrementalCheckout(buildroot)
590 new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
591 if old_binhost != new_binhost:
592 RunCommand(['sudo', 'rm', '-rf', boardpath])
543 593
544 # Check that all overlays can be found. 594 # Check that all overlays can be found.
545 for path in overlays: 595 for path in overlays:
546 assert ':' not in path, 'Overlay must not contain colons: %s' % path 596 assert ':' not in path, 'Overlay must not contain colons: %s' % path
547 if not os.path.isdir(path): 597 if not os.path.isdir(path):
548 Die('Missing overlay: %s' % path) 598 Die('Missing overlay: %s' % path)
549 599
550 chroot_path = os.path.join(buildroot, 'chroot')
551 if not os.path.isdir(chroot_path): 600 if not os.path.isdir(chroot_path):
552 _MakeChroot(buildroot) 601 _MakeChroot(buildroot)
553 602
554 boardpath = os.path.join(chroot_path, 'build', buildconfig['board'])
555 if not os.path.isdir(boardpath): 603 if not os.path.isdir(boardpath):
556 _SetupBoard(buildroot, board=buildconfig['board']) 604 _SetupBoard(buildroot, board=buildconfig['board'])
557 605
558 if buildconfig['uprev']: 606 if buildconfig['uprev']:
559 _UprevPackages(buildroot, tracking_branch, revisionfile, 607 _UprevPackages(buildroot, tracking_branch, revisionfile,
560 buildconfig['board'], overlays) 608 buildconfig['board'], overlays)
561 609
562 _EnableLocalAccount(buildroot) 610 _EnableLocalAccount(buildroot)
563 _Build(buildroot) 611 _Build(buildroot)
564 if buildconfig['unittests']: 612 if buildconfig['unittests']:
(...skipping 10 matching lines...) Expand all
575 _ArchiveTestResults(buildroot, buildconfig['board'], 623 _ArchiveTestResults(buildroot, buildconfig['board'],
576 archive_dir=options.buildnumber, 624 archive_dir=options.buildnumber,
577 test_results_dir=test_results_dir) 625 test_results_dir=test_results_dir)
578 626
579 if buildconfig['uprev']: 627 if buildconfig['uprev']:
580 # Don't push changes for developers. 628 # Don't push changes for developers.
581 if not options.debug: 629 if not options.debug:
582 if buildconfig['master']: 630 if buildconfig['master']:
583 # Master bot needs to check if the other slaves completed. 631 # Master bot needs to check if the other slaves completed.
584 if cbuildbot_comm.HaveSlavesCompleted(config): 632 if cbuildbot_comm.HaveSlavesCompleted(config):
633 binhosts = [
634 _GetPortageEnvVar(buildroot, board, _PREFLIGHT_BINHOST),
635 _GetPortageEnvVar(buildroot, board, _FULL_BINHOST),
636 ]
585 _UprevPush(buildroot, tracking_branch, buildconfig['board'], 637 _UprevPush(buildroot, tracking_branch, buildconfig['board'],
586 overlays) 638 overlays)
639 _UploadPrebuilts(buildroot, board, buildconfig['overlays'],
640 binhosts)
587 else: 641 else:
588 Die('CBUILDBOT - One of the slaves has failed!!!') 642 Die('CBUILDBOT - One of the slaves has failed!!!')
589 643
590 else: 644 else:
591 # Publish my status to the master if its expecting it. 645 # Publish my status to the master if its expecting it.
592 if buildconfig['important']: 646 if buildconfig['important']:
593 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE) 647 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
594
595 except: 648 except:
596 # Send failure to master bot. 649 # Send failure to master bot.
597 if not buildconfig['master'] and buildconfig['important']: 650 if not buildconfig['master'] and buildconfig['important']:
598 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED) 651 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
599 652
600 raise 653 raise
601 654
602 655
603 if __name__ == '__main__': 656 if __name__ == '__main__':
604 main() 657 main()
OLDNEW
« no previous file with comments | « no previous file | bin/cbuildbot_unittest.py » ('j') | chromite/lib/binpkg.py » ('J')

Powered by Google App Engine
This is Rietveld 408576698