OLD | NEW |
---|---|
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 import datetime | 6 import datetime |
7 import multiprocessing | 7 import multiprocessing |
8 import optparse | 8 import optparse |
9 import os | 9 import os |
10 import re | 10 import re |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
44 """ | 44 """ |
45 | 45 |
46 # as per http://crosbug.com/5855 always filter the below packages | 46 # as per http://crosbug.com/5855 always filter the below packages |
47 _FILTER_PACKAGES = set() | 47 _FILTER_PACKAGES = set() |
48 _RETRIES = 3 | 48 _RETRIES = 3 |
49 _GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil' | 49 _GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil' |
50 _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs' | 50 _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs' |
51 _CATEGORIES_PATH = 'chroot/etc/portage/categories' | 51 _CATEGORIES_PATH = 'chroot/etc/portage/categories' |
52 _HOST_TARGET = 'amd64' | 52 _HOST_TARGET = 'amd64' |
53 _BOARD_PATH = 'chroot/build/%(board)s' | 53 _BOARD_PATH = 'chroot/build/%(board)s' |
54 _BOTO_CONFIG = '/home/chrome-bot/external-boto' | |
55 # board/board-target/version/packages/' | 54 # board/board-target/version/packages/' |
56 _REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages' | 55 _REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages' |
57 # host/host-target/version/packages/' | 56 # host/host-target/version/packages/' |
58 _REL_HOST_PATH = 'host/%(target)s/%(version)s/packages' | 57 _REL_HOST_PATH = 'host/%(target)s/%(version)s/packages' |
59 # Private overlays to look at for builds to filter | 58 # Private overlays to look at for builds to filter |
60 # relative to build path | 59 # relative to build path |
61 _PRIVATE_OVERLAY_DIR = 'src/private-overlays' | 60 _PRIVATE_OVERLAY_DIR = 'src/private-overlays' |
62 _BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt' | 61 _BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt' |
63 _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/' | 62 _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/' |
64 # Created in the event of new host targets becoming available | 63 # Created in the event of new host targets becoming available |
(...skipping 206 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
271 """Upload to GS bucket. | 270 """Upload to GS bucket. |
272 | 271 |
273 Args: | 272 Args: |
274 args: a tuple of three arguments that contains local_file, remote_file, and | 273 args: a tuple of three arguments that contains local_file, remote_file, and |
275 the acl used for uploading the file. | 274 the acl used for uploading the file. |
276 | 275 |
277 Returns: | 276 Returns: |
278 Return the arg tuple of two if the upload failed | 277 Return the arg tuple of two if the upload failed |
279 """ | 278 """ |
280 (local_file, remote_file, acl) = args | 279 (local_file, remote_file, acl) = args |
280 CANNED_ACLS = ['public-read', 'private', 'bucket-owner-read', | |
281 'authenticated-read', 'bucket-owner-full-control', | |
282 'public-read-write'] | |
283 acl_cmd = None | |
284 if acl in CANNED_ACLS: | |
285 cmd = '%s cp -a %s %s %s' % (_GSUTIL_BIN, acl, local_file, remote_file) | |
286 else: | |
287 # if acl is not a canned acl we upload everything private and check below if | |
288 # it is a file we can use to set acls | |
289 cmd = '%s cp -a private %s %s' % (_GSUTIL_BIN, local_file, remote_file) | |
290 if not os.path.exists(acl): | |
291 print ('You are specifying either a file that does not exist or an ' | |
292 'unknown canned acl. %s aborting upload') % acl | |
davidjames
2011/03/16 21:29:39
Could you line up the quotes here? Also, I don't u
scottz
2011/03/16 21:36:06
Done.
| |
293 # Since we are already a different process exiting seems to be the most | |
davidjames
2011/03/16 21:29:39
Could you add a comma between "process" and "exiti
scottz
2011/03/16 21:36:06
Done.
| |
294 # sane thing to do here. | |
295 sys.exit(1) | |
davidjames
2011/03/16 21:29:39
Did you test whether the parent process catch this
scottz
2011/03/16 21:36:06
Actually that is a good point I am moving this to
| |
281 | 296 |
282 cmd = '%s cp -a %s %s %s' % (_GSUTIL_BIN, acl, local_file, remote_file) | 297 acl_cmd = '%s setacl %s %s' % (_GSUTIL_BIN, acl, remote_file) |
298 | |
283 if not _RetryRun(cmd, print_cmd=False, shell=True): | 299 if not _RetryRun(cmd, print_cmd=False, shell=True): |
284 return (local_file, remote_file) | 300 return (local_file, remote_file) |
285 | 301 |
302 if acl_cmd: | |
303 # Apply the passed in ACL xml file to the uploaded object. | |
304 _RetryRun(acl_cmd, print_cmd=False, shell=True) | |
305 | |
306 | |
286 def RemoteUpload(acl, files, pool=10): | 307 def RemoteUpload(acl, files, pool=10): |
287 """Upload to google storage. | 308 """Upload to google storage. |
288 | 309 |
289 Create a pool of process and call _GsUpload with the proper arguments. | 310 Create a pool of process and call _GsUpload with the proper arguments. |
290 | 311 |
291 Args: | 312 Args: |
292 acl: The canned acl used for uploading. acl can be one of: "public-read", | 313 acl: The canned acl used for uploading. acl can be one of: "public-read", |
293 "public-read-write", "authenticated-read", "bucket-owner-read", | 314 "public-read-write", "authenticated-read", "bucket-owner-read", |
294 "bucket-owner-full-control", or "private". | 315 "bucket-owner-full-control", or "private". |
295 files: dictionary with keys to local files and values to remote path. | 316 files: dictionary with keys to local files and values to remote path. |
(...skipping 269 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
565 | 586 |
566 | 587 |
567 def usage(parser, msg): | 588 def usage(parser, msg): |
568 """Display usage message and parser help then exit with 1.""" | 589 """Display usage message and parser help then exit with 1.""" |
569 print >> sys.stderr, msg | 590 print >> sys.stderr, msg |
570 parser.print_help() | 591 parser.print_help() |
571 sys.exit(1) | 592 sys.exit(1) |
572 | 593 |
573 def ParseOptions(): | 594 def ParseOptions(): |
574 parser = optparse.OptionParser() | 595 parser = optparse.OptionParser() |
596 parser.add_option('-a', '--acl-file', dest='acl_file', default=None, | |
597 help='ACL File for Google Storage files') | |
575 parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url', | 598 parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url', |
576 default=_BINHOST_BASE_URL, | 599 default=_BINHOST_BASE_URL, |
577 help='Base URL to use for binhost in make.conf updates') | 600 help='Base URL to use for binhost in make.conf updates') |
578 parser.add_option('', '--previous-binhost-url', action='append', | 601 parser.add_option('', '--previous-binhost-url', action='append', |
579 default=[], dest='previous_binhost_url', | 602 default=[], dest='previous_binhost_url', |
580 help='Previous binhost URL') | 603 help='Previous binhost URL') |
581 parser.add_option('-b', '--board', dest='board', default=None, | 604 parser.add_option('-b', '--board', dest='board', default=None, |
582 help='Board type that was built on this machine') | 605 help='Board type that was built on this machine') |
583 parser.add_option('-p', '--build-path', dest='build_path', | 606 parser.add_option('-p', '--build-path', dest='build_path', |
584 help='Path to the directory containing the chroot') | 607 help='Path to the directory containing the chroot') |
(...skipping 19 matching lines...) Expand all Loading... | |
604 default=False, action='store_true', | 627 default=False, action='store_true', |
605 help='Update binhost.conf') | 628 help='Update binhost.conf') |
606 parser.add_option('-P', '--private', dest='private', action='store_true', | 629 parser.add_option('-P', '--private', dest='private', action='store_true', |
607 default=False, help='Mark gs:// uploads as private.') | 630 default=False, help='Mark gs:// uploads as private.') |
608 | 631 |
609 options, args = parser.parse_args() | 632 options, args = parser.parse_args() |
610 if not options.build_path: | 633 if not options.build_path: |
611 usage(parser, 'Error: you need provide a chroot path') | 634 usage(parser, 'Error: you need provide a chroot path') |
612 if not options.upload: | 635 if not options.upload: |
613 usage(parser, 'Error: you need to provide an upload location using -u') | 636 usage(parser, 'Error: you need to provide an upload location using -u') |
614 if options.private and not (options.binhost_base_url.startswith('gs://') and | 637 |
615 options.upload.startswith('gs://')): | 638 if options.acl_file and not os.path.isfile(options.acl_file): |
616 usage(parser, 'Error: --private is only valid for gs:// URLs.\n' | 639 usage(parser, 'Error: ACL file provided is not a file') |
617 'Both --binhost-base-url and --upload must be gs:// URLs.') | 640 |
641 if options.private: | |
642 if not options.upload.startswith('gs://'): | |
643 usage(parser, 'Error: --private is only valid for gs:// URLs.\n' | |
644 '--upload must be a gs:// URL.') | |
645 if options.binhost_base_url != _BINHOST_BASE_URL: | |
646 usage(parser, 'Error: when using --private the --binhost-base-url ' | |
647 'is automatically derived.') | |
618 return options | 648 return options |
619 | 649 |
620 def main(): | 650 def main(): |
621 options = ParseOptions() | 651 options = ParseOptions() |
622 | 652 |
623 # Setup boto environment for gsutil to use | |
624 os.environ['BOTO_CONFIG'] = _BOTO_CONFIG | |
625 | |
626 if options.filters: | 653 if options.filters: |
627 LoadPrivateFilters(options.build_path) | 654 LoadPrivateFilters(options.build_path) |
628 | 655 |
629 acl = 'public-read' | |
630 if options.private: | |
631 acl = 'private' | |
632 | 656 |
633 # Calculate a list of Packages index files to compare against. Whenever we | 657 # Calculate a list of Packages index files to compare against. Whenever we |
634 # upload a package, we check to make sure it's not already stored in one of | 658 # upload a package, we check to make sure it's not already stored in one of |
635 # the packages files we uploaded. This list of packages files might contain | 659 # the packages files we uploaded. This list of packages files might contain |
636 # both board and host packages. | 660 # both board and host packages. |
637 pkg_indexes = _GrabAllRemotePackageIndexes(options.previous_binhost_url) | 661 pkg_indexes = _GrabAllRemotePackageIndexes(options.previous_binhost_url) |
638 | 662 |
639 version = GetVersion() | 663 version = GetVersion() |
640 if options.prepend_version: | 664 if options.prepend_version: |
641 version = '%s-%s' % (options.prepend_version, version) | 665 version = '%s-%s' % (options.prepend_version, version) |
642 | 666 |
643 uploader = PrebuiltUploader(options.upload, acl, options.binhost_base_url, | 667 acl = 'public-read' |
668 binhost_base_url = options.binhost_base_url | |
669 | |
670 if options.private: | |
671 acl = 'private' | |
672 binhost_base_url = options.upload | |
673 | |
674 if options.acl_file: | |
675 acl = options.acl_file | |
676 | |
677 uploader = PrebuiltUploader(options.upload, acl, binhost_base_url, | |
644 pkg_indexes) | 678 pkg_indexes) |
645 | 679 |
646 if options.sync_host: | 680 if options.sync_host: |
647 uploader._SyncHostPrebuilts(options.build_path, version, options.key, | 681 uploader._SyncHostPrebuilts(options.build_path, version, options.key, |
648 options.git_sync, options.sync_binhost_conf) | 682 options.git_sync, options.sync_binhost_conf) |
649 | 683 |
650 if options.board: | 684 if options.board: |
651 uploader._SyncBoardPrebuilts(options.board, options.build_path, version, | 685 uploader._SyncBoardPrebuilts(options.board, options.build_path, version, |
652 options.key, options.git_sync, | 686 options.key, options.git_sync, |
653 options.sync_binhost_conf) | 687 options.sync_binhost_conf) |
654 | 688 |
655 if __name__ == '__main__': | 689 if __name__ == '__main__': |
656 main() | 690 main() |
OLD | NEW |