| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 3 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 """CBuildbot is wrapper around the build process used by the pre-flight queue""" | 7 """CBuildbot is wrapper around the build process used by the pre-flight queue""" |
| 8 | 8 |
| 9 import errno | 9 import errno |
| 10 import re | 10 import re |
| (...skipping 159 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 170 repo_name = revision_tuple[0].replace('.git', '') | 170 repo_name = revision_tuple[0].replace('.git', '') |
| 171 # Might not have entry if no matching ebuild. | 171 # Might not have entry if no matching ebuild. |
| 172 if repo_dictionary.has_key(repo_name): | 172 if repo_dictionary.has_key(repo_name): |
| 173 # May be many corresponding packages to a given git repo e.g. kernel). | 173 # May be many corresponding packages to a given git repo e.g. kernel). |
| 174 for package in repo_dictionary[repo_name]: | 174 for package in repo_dictionary[repo_name]: |
| 175 revisions[package] = revision_tuple[1] | 175 revisions[package] = revision_tuple[1] |
| 176 | 176 |
| 177 return revisions.items() | 177 return revisions.items() |
| 178 | 178 |
| 179 | 179 |
| 180 def _UprevFromRevisionList(buildroot, tracking_branch, revision_list): | 180 def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board): |
| 181 """Uprevs based on revision list.""" | 181 """Uprevs based on revision list.""" |
| 182 if not revision_list: | 182 if not revision_list: |
| 183 Info('No packages found to uprev') | 183 Info('No packages found to uprev') |
| 184 return | 184 return |
| 185 | 185 |
| 186 package_str = '' | 186 package_str = '' |
| 187 for package, revision in revision_list: | 187 for package, revision in revision_list: |
| 188 package_str += package + ' ' | 188 package_str += package + ' ' |
| 189 | 189 |
| 190 package_str = package_str.strip() | 190 package_str = package_str.strip() |
| 191 | 191 |
| 192 cwd = os.path.join(buildroot, 'src', 'scripts') | 192 cwd = os.path.join(buildroot, 'src', 'scripts') |
| 193 RunCommand(['./cros_mark_as_stable', | 193 RunCommand(['./cros_mark_as_stable', |
| 194 '--board=%s' % board, |
| 194 '--tracking_branch="%s"' % tracking_branch, | 195 '--tracking_branch="%s"' % tracking_branch, |
| 195 '--packages="%s"' % package_str, | 196 '--packages="%s"' % package_str, |
| 196 'commit'], | 197 'commit'], |
| 197 cwd=cwd, enter_chroot=True) | 198 cwd=cwd, enter_chroot=True) |
| 198 | 199 |
| 199 | 200 |
| 200 def _UprevAllPackages(buildroot, tracking_branch): | 201 def _UprevAllPackages(buildroot, tracking_branch, board): |
| 201 """Uprevs all packages that have been updated since last uprev.""" | 202 """Uprevs all packages that have been updated since last uprev.""" |
| 202 cwd = os.path.join(buildroot, 'src', 'scripts') | 203 cwd = os.path.join(buildroot, 'src', 'scripts') |
| 203 RunCommand(['./cros_mark_as_stable', '--all', | 204 RunCommand(['./cros_mark_as_stable', '--all', |
| 205 '--board=%s' % board, |
| 204 '--tracking_branch="%s"' % tracking_branch, 'commit'], | 206 '--tracking_branch="%s"' % tracking_branch, 'commit'], |
| 205 cwd=cwd, enter_chroot=True) | 207 cwd=cwd, enter_chroot=True) |
| 206 | 208 |
| 207 | 209 |
| 208 def _GetVMConstants(buildroot): | 210 def _GetVMConstants(buildroot): |
| 209 """Returns minimum (vdisk_size, statefulfs_size) recommended for VM's.""" | 211 """Returns minimum (vdisk_size, statefulfs_size) recommended for VM's.""" |
| 210 cwd = os.path.join(buildroot, 'src', 'scripts', 'lib') | 212 cwd = os.path.join(buildroot, 'src', 'scripts', 'lib') |
| 211 source_cmd = 'source %s/cros_vm_constants.sh' % cwd | 213 source_cmd = 'source %s/cros_vm_constants.sh' % cwd |
| 212 vdisk_size = RunCommand([ | 214 vdisk_size = RunCommand([ |
| 213 '/bin/bash', '-c', '%s && echo $MIN_VDISK_SIZE_FULL' % source_cmd], | 215 '/bin/bash', '-c', '%s && echo $MIN_VDISK_SIZE_FULL' % source_cmd], |
| 214 redirect_stdout=True) | 216 redirect_stdout=True) |
| 215 statefulfs_size = RunCommand([ | 217 statefulfs_size = RunCommand([ |
| 216 '/bin/bash', '-c', '%s && echo $MIN_STATEFUL_FS_SIZE_FULL' % source_cmd], | 218 '/bin/bash', '-c', '%s && echo $MIN_STATEFUL_FS_SIZE_FULL' % source_cmd], |
| 217 redirect_stdout=True) | 219 redirect_stdout=True) |
| 218 return (vdisk_size.strip(), statefulfs_size.strip()) | 220 return (vdisk_size.strip(), statefulfs_size.strip()) |
| 219 | 221 |
| 220 | 222 |
| 221 def _GitCleanup(buildroot): | 223 def _GitCleanup(buildroot, board): |
| 222 """Clean up git branch after previous uprev attempt.""" | 224 """Clean up git branch after previous uprev attempt.""" |
| 223 cwd = os.path.join(buildroot, 'src', 'scripts') | 225 cwd = os.path.join(buildroot, 'src', 'scripts') |
| 224 if os.path.exists(cwd): | 226 if os.path.exists(cwd): |
| 225 RunCommand(['./cros_mark_as_stable', '--srcroot=..', | 227 RunCommand(['./cros_mark_as_stable', '--srcroot=..', |
| 228 '--board=%s' % board, |
| 226 '--tracking_branch="cros/master"', 'clean'], | 229 '--tracking_branch="cros/master"', 'clean'], |
| 227 cwd=cwd, error_ok=True) | 230 cwd=cwd, error_ok=True) |
| 228 | 231 |
| 229 | 232 |
| 230 def _CleanUpMountPoints(buildroot): | 233 def _CleanUpMountPoints(buildroot): |
| 231 """Cleans up any stale mount points from previous runs.""" | 234 """Cleans up any stale mount points from previous runs.""" |
| 232 mount_output = RunCommand(['mount'], redirect_stdout=True) | 235 mount_output = RunCommand(['mount'], redirect_stdout=True) |
| 233 mount_pts_in_buildroot = RunCommand(['grep', buildroot], input=mount_output, | 236 mount_pts_in_buildroot = RunCommand(['grep', buildroot], input=mount_output, |
| 234 redirect_stdout=True, error_ok=True) | 237 redirect_stdout=True, error_ok=True) |
| 235 | 238 |
| 236 for mount_pt_str in mount_pts_in_buildroot.splitlines(): | 239 for mount_pt_str in mount_pts_in_buildroot.splitlines(): |
| 237 mount_pt = mount_pt_str.rpartition(' type ')[0].partition(' on ')[2] | 240 mount_pt = mount_pt_str.rpartition(' type ')[0].partition(' on ')[2] |
| 238 RunCommand(['sudo', 'umount', '-l', mount_pt], error_ok=True) | 241 RunCommand(['sudo', 'umount', '-l', mount_pt], error_ok=True) |
| 239 | 242 |
| 240 | 243 |
| 241 def _WipeOldOutput(buildroot): | 244 def _WipeOldOutput(buildroot): |
| 242 """Wipes out build output directories.""" | 245 """Wipes out build output directories.""" |
| 243 RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot) | 246 RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot) |
| 244 | 247 |
| 245 | 248 |
| 246 # =========================== Main Commands =================================== | 249 # =========================== Main Commands =================================== |
| 247 | 250 |
| 248 | 251 |
| 249 def _PreFlightRinse(buildroot): | 252 def _PreFlightRinse(buildroot, board): |
| 250 """Cleans up any leftover state from previous runs.""" | 253 """Cleans up any leftover state from previous runs.""" |
| 251 _GitCleanup(buildroot) | 254 _GitCleanup(buildroot, board) |
| 252 _CleanUpMountPoints(buildroot) | 255 _CleanUpMountPoints(buildroot) |
| 253 RunCommand(['sudo', 'killall', 'kvm'], error_ok=True) | 256 RunCommand(['sudo', 'killall', 'kvm'], error_ok=True) |
| 254 | 257 |
| 255 | 258 |
| 256 def _FullCheckout(buildroot, tracking_branch, rw_checkout=True, | 259 def _FullCheckout(buildroot, tracking_branch, rw_checkout=True, |
| 257 retries=_DEFAULT_RETRIES, | 260 retries=_DEFAULT_RETRIES, |
| 258 url='http://git.chromium.org/git/manifest'): | 261 url='http://git.chromium.org/git/manifest'): |
| 259 """Performs a full checkout and clobbers any previous checkouts.""" | 262 """Performs a full checkout and clobbers any previous checkouts.""" |
| 260 RunCommand(['sudo', 'rm', '-rf', buildroot]) | 263 RunCommand(['sudo', 'rm', '-rf', buildroot]) |
| 261 MakeDir(buildroot, parents=True) | 264 MakeDir(buildroot, parents=True) |
| (...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 352 revisions = 'None' | 355 revisions = 'None' |
| 353 | 356 |
| 354 revisions = revisions.strip() | 357 revisions = revisions.strip() |
| 355 | 358 |
| 356 # TODO(sosa): Un-comment once we close individual trees. | 359 # TODO(sosa): Un-comment once we close individual trees. |
| 357 # revisions == "None" indicates a Force Build. | 360 # revisions == "None" indicates a Force Build. |
| 358 #if revisions != 'None': | 361 #if revisions != 'None': |
| 359 # print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions | 362 # print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions |
| 360 # revision_list = _ParseRevisionString(revisions, | 363 # revision_list = _ParseRevisionString(revisions, |
| 361 # _CreateRepoDictionary(buildroot, board)) | 364 # _CreateRepoDictionary(buildroot, board)) |
| 362 # _UprevFromRevisionList(buildroot, tracking_branch, revision_list) | 365 # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board) |
| 363 #else: | 366 #else: |
| 364 Info('CBUILDBOT Revving all') | 367 Info('CBUILDBOT Revving all') |
| 365 _UprevAllPackages(buildroot, tracking_branch) | 368 _UprevAllPackages(buildroot, tracking_branch, board) |
| 366 | 369 |
| 367 | 370 |
| 368 def _UprevPush(buildroot, tracking_branch): | 371 def _UprevPush(buildroot, tracking_branch, board): |
| 369 """Pushes uprev changes to the main line.""" | 372 """Pushes uprev changes to the main line.""" |
| 370 cwd = os.path.join(buildroot, 'src', 'scripts') | 373 cwd = os.path.join(buildroot, 'src', 'scripts') |
| 371 RunCommand(['./cros_mark_as_stable', '--srcroot=..', | 374 RunCommand(['./cros_mark_as_stable', '--srcroot=..', |
| 375 '--board=%s' % board, |
| 372 '--tracking_branch="%s"' % tracking_branch, | 376 '--tracking_branch="%s"' % tracking_branch, |
| 373 '--push_options="--bypass-hooks -f"', 'push'], | 377 '--push_options="--bypass-hooks -f"', 'push'], |
| 374 cwd=cwd) | 378 cwd=cwd) |
| 375 | 379 |
| 376 | 380 |
| 377 def _GetConfig(config_name): | 381 def _GetConfig(config_name): |
| 378 """Gets the configuration for the build""" | 382 """Gets the configuration for the build""" |
| 379 default = config['default'] | 383 default = config['default'] |
| 380 buildconfig = {} | 384 buildconfig = {} |
| 381 if not config.has_key(config_name): | 385 if not config.has_key(config_name): |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 425 tracking_branch = options.tracking_branch | 429 tracking_branch = options.tracking_branch |
| 426 | 430 |
| 427 if len(args) >= 1: | 431 if len(args) >= 1: |
| 428 buildconfig = _GetConfig(args[-1]) | 432 buildconfig = _GetConfig(args[-1]) |
| 429 else: | 433 else: |
| 430 Warning('Missing configuration description') | 434 Warning('Missing configuration description') |
| 431 parser.print_usage() | 435 parser.print_usage() |
| 432 sys.exit(1) | 436 sys.exit(1) |
| 433 | 437 |
| 434 try: | 438 try: |
| 435 _PreFlightRinse(buildroot) | 439 _PreFlightRinse(buildroot, buildconfig['board']) |
| 436 if options.clobber or not os.path.isdir(buildroot): | 440 if options.clobber or not os.path.isdir(buildroot): |
| 437 _FullCheckout(buildroot, tracking_branch, url=options.url) | 441 _FullCheckout(buildroot, tracking_branch, url=options.url) |
| 438 else: | 442 else: |
| 439 _IncrementalCheckout(buildroot) | 443 _IncrementalCheckout(buildroot) |
| 440 | 444 |
| 441 chroot_path = os.path.join(buildroot, 'chroot') | 445 chroot_path = os.path.join(buildroot, 'chroot') |
| 442 if not os.path.isdir(chroot_path): | 446 if not os.path.isdir(chroot_path): |
| 443 _MakeChroot(buildroot) | 447 _MakeChroot(buildroot) |
| 444 | 448 |
| 445 boardpath = os.path.join(chroot_path, 'build', buildconfig['board']) | 449 boardpath = os.path.join(chroot_path, 'build', buildconfig['board']) |
| (...skipping 14 matching lines...) Expand all Loading... |
| 460 if buildconfig['smoke_bvt']: | 464 if buildconfig['smoke_bvt']: |
| 461 _BuildVMImageForTesting(buildroot) | 465 _BuildVMImageForTesting(buildroot) |
| 462 _RunSmokeSuite(buildroot) | 466 _RunSmokeSuite(buildroot) |
| 463 | 467 |
| 464 if buildconfig['uprev']: | 468 if buildconfig['uprev']: |
| 465 # Don't push changes for developers. | 469 # Don't push changes for developers. |
| 466 if not options.debug: | 470 if not options.debug: |
| 467 if buildconfig['master']: | 471 if buildconfig['master']: |
| 468 # Master bot needs to check if the other slaves completed. | 472 # Master bot needs to check if the other slaves completed. |
| 469 if cbuildbot_comm.HaveSlavesCompleted(config): | 473 if cbuildbot_comm.HaveSlavesCompleted(config): |
| 470 _UprevPush(buildroot, tracking_branch) | 474 _UprevPush(buildroot, tracking_branch, buildconfig['board']) |
| 471 else: | 475 else: |
| 472 Die('CBUILDBOT - One of the slaves has failed!!!') | 476 Die('CBUILDBOT - One of the slaves has failed!!!') |
| 473 | 477 |
| 474 else: | 478 else: |
| 475 # Publish my status to the master if its expecting it. | 479 # Publish my status to the master if its expecting it. |
| 476 if buildconfig['important']: | 480 if buildconfig['important']: |
| 477 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE) | 481 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE) |
| 478 | 482 |
| 479 except: | 483 except: |
| 480 # Send failure to master bot. | 484 # Send failure to master bot. |
| 481 if not buildconfig['master'] and buildconfig['important']: | 485 if not buildconfig['master'] and buildconfig['important']: |
| 482 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED) | 486 cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED) |
| 483 | 487 |
| 484 raise | 488 raise |
| 485 | 489 |
| 486 | 490 |
| 487 if __name__ == '__main__': | 491 if __name__ == '__main__': |
| 488 main() | 492 main() |
| OLD | NEW |