OLD | NEW |
---|---|
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 import datetime | 6 import datetime |
7 import multiprocessing | 7 import multiprocessing |
8 import optparse | 8 import optparse |
9 import os | 9 import os |
10 import re | 10 import re |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
44 """ | 44 """ |
45 | 45 |
46 # as per http://crosbug.com/5855 always filter the below packages | 46 # as per http://crosbug.com/5855 always filter the below packages |
47 _FILTER_PACKAGES = set() | 47 _FILTER_PACKAGES = set() |
48 _RETRIES = 3 | 48 _RETRIES = 3 |
49 _GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil' | 49 _GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil' |
50 _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs' | 50 _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs' |
51 _CATEGORIES_PATH = 'chroot/etc/portage/categories' | 51 _CATEGORIES_PATH = 'chroot/etc/portage/categories' |
52 _HOST_TARGET = 'amd64' | 52 _HOST_TARGET = 'amd64' |
53 _BOARD_PATH = 'chroot/build/%(board)s' | 53 _BOARD_PATH = 'chroot/build/%(board)s' |
54 _BOTO_CONFIG = '/home/chrome-bot/external-boto' | |
55 # board/board-target/version/packages/' | 54 # board/board-target/version/packages/' |
56 _REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages' | 55 _REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages' |
57 # host/host-target/version/packages/' | 56 # host/host-target/version/packages/' |
58 _REL_HOST_PATH = 'host/%(target)s/%(version)s/packages' | 57 _REL_HOST_PATH = 'host/%(target)s/%(version)s/packages' |
59 # Private overlays to look at for builds to filter | 58 # Private overlays to look at for builds to filter |
60 # relative to build path | 59 # relative to build path |
61 _PRIVATE_OVERLAY_DIR = 'src/private-overlays' | 60 _PRIVATE_OVERLAY_DIR = 'src/private-overlays' |
62 _BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt' | 61 _BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt' |
63 _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/' | 62 _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/' |
64 # Created in the event of new host targets becoming available | 63 # Created in the event of new host targets becoming available |
(...skipping 206 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
271 """Upload to GS bucket. | 270 """Upload to GS bucket. |
272 | 271 |
273 Args: | 272 Args: |
274 args: a tuple of three arguments that contains local_file, remote_file, and | 273 args: a tuple of three arguments that contains local_file, remote_file, and |
275 the acl used for uploading the file. | 274 the acl used for uploading the file. |
276 | 275 |
277 Returns: | 276 Returns: |
278 Return the arg tuple of two if the upload failed | 277 Return the arg tuple of two if the upload failed |
279 """ | 278 """ |
280 (local_file, remote_file, acl) = args | 279 (local_file, remote_file, acl) = args |
281 | 280 CANNED_ACLS = ['public-read', 'private', 'bucket-owner-read', |
282 cmd = '%s cp -a %s %s %s' % (_GSUTIL_BIN, acl, local_file, remote_file) | 281 'authenticated-read', 'bucket-owner-full-control', |
282 'public-read-write'] | |
283 if acl in CANNED_ACLS: | |
284 cmd = '%s cp -a %s %s %s' % (_GSUTIL_BIN, acl, local_file, remote_file) | |
285 else: | |
286 # if acl is not a canned acl we upload everything private and check below if | |
287 # it is a file we can use to set acls | |
288 cmd = '%s cp -a private %s %s' % (_GSUTIL_BIN, local_file, remote_file) | |
davidjames
2011/03/14 22:31:44
Can we check that the acl file exists in this case
scottz
2011/03/14 22:57:45
I actually had that around here and I moved it dow
davidjames
2011/03/16 18:48:32
I'm not worried about it disappearing during the r
| |
283 if not _RetryRun(cmd, print_cmd=False, shell=True): | 289 if not _RetryRun(cmd, print_cmd=False, shell=True): |
284 return (local_file, remote_file) | 290 return (local_file, remote_file) |
285 | 291 |
292 if os.path.exists(acl): | |
293 # We have an acl xml file we want to apply | |
294 acl_cmd = '%s setacl %s %s' % (_GSUTIL_BIN, acl, remote_file) | |
295 _RetryRun(acl_cmd, print_cmd=False, shell=True) | |
296 | |
297 | |
286 def RemoteUpload(acl, files, pool=10): | 298 def RemoteUpload(acl, files, pool=10): |
287 """Upload to google storage. | 299 """Upload to google storage. |
288 | 300 |
289 Create a pool of process and call _GsUpload with the proper arguments. | 301 Create a pool of process and call _GsUpload with the proper arguments. |
290 | 302 |
291 Args: | 303 Args: |
292 acl: The canned acl used for uploading. acl can be one of: "public-read", | 304 acl: The canned acl used for uploading. acl can be one of: "public-read", |
293 "public-read-write", "authenticated-read", "bucket-owner-read", | 305 "public-read-write", "authenticated-read", "bucket-owner-read", |
294 "bucket-owner-full-control", or "private". | 306 "bucket-owner-full-control", or "private". |
295 files: dictionary with keys to local files and values to remote path. | 307 files: dictionary with keys to local files and values to remote path. |
(...skipping 269 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
565 | 577 |
566 | 578 |
567 def usage(parser, msg): | 579 def usage(parser, msg): |
568 """Display usage message and parser help then exit with 1.""" | 580 """Display usage message and parser help then exit with 1.""" |
569 print >> sys.stderr, msg | 581 print >> sys.stderr, msg |
570 parser.print_help() | 582 parser.print_help() |
571 sys.exit(1) | 583 sys.exit(1) |
572 | 584 |
573 def ParseOptions(): | 585 def ParseOptions(): |
574 parser = optparse.OptionParser() | 586 parser = optparse.OptionParser() |
587 parser.add_option('-a', '--acl-file', dest='acl_file', default=None, | |
588 help='ACL File for Google Storage files') | |
575 parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url', | 589 parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url', |
576 default=_BINHOST_BASE_URL, | 590 default=_BINHOST_BASE_URL, |
577 help='Base URL to use for binhost in make.conf updates') | 591 help='Base URL to use for binhost in make.conf updates') |
578 parser.add_option('', '--previous-binhost-url', action='append', | 592 parser.add_option('', '--previous-binhost-url', action='append', |
579 default=[], dest='previous_binhost_url', | 593 default=[], dest='previous_binhost_url', |
580 help='Previous binhost URL') | 594 help='Previous binhost URL') |
581 parser.add_option('-b', '--board', dest='board', default=None, | 595 parser.add_option('-b', '--board', dest='board', default=None, |
582 help='Board type that was built on this machine') | 596 help='Board type that was built on this machine') |
583 parser.add_option('-p', '--build-path', dest='build_path', | 597 parser.add_option('-p', '--build-path', dest='build_path', |
584 help='Path to the directory containing the chroot') | 598 help='Path to the directory containing the chroot') |
(...skipping 19 matching lines...) Expand all Loading... | |
604 default=False, action='store_true', | 618 default=False, action='store_true', |
605 help='Update binhost.conf') | 619 help='Update binhost.conf') |
606 parser.add_option('-P', '--private', dest='private', action='store_true', | 620 parser.add_option('-P', '--private', dest='private', action='store_true', |
607 default=False, help='Mark gs:// uploads as private.') | 621 default=False, help='Mark gs:// uploads as private.') |
608 | 622 |
609 options, args = parser.parse_args() | 623 options, args = parser.parse_args() |
610 if not options.build_path: | 624 if not options.build_path: |
611 usage(parser, 'Error: you need provide a chroot path') | 625 usage(parser, 'Error: you need provide a chroot path') |
612 if not options.upload: | 626 if not options.upload: |
613 usage(parser, 'Error: you need to provide an upload location using -u') | 627 usage(parser, 'Error: you need to provide an upload location using -u') |
614 if options.private and not (options.binhost_base_url.startswith('gs://') and | 628 |
615 options.upload.startswith('gs://')): | 629 if options.acl_file and not os.path.isfile(options.acl_file): |
616 usage(parser, 'Error: --private is only valid for gs:// URLs.\n' | 630 usage(parser, 'Error: ACL file provided is not a file') |
617 'Both --binhost-base-url and --upload must be gs:// URLs.') | 631 |
632 if options.private: | |
633 if not options.upload.startswith('gs://'): | |
634 usage(parser, 'Error: --private is only valid for gs:// URLs.\n' | |
635 '--upload must be a gs:// URL.') | |
636 if (options.binhost_base_url != _BINHOST_BASE_URL): | |
davidjames
2011/03/14 22:31:44
No parens needed here.
scottz
2011/03/14 22:57:45
Done.
| |
637 usage(parser, 'Error: when using --private the --binhost-base-url ' | |
638 'is automatically derived.') | |
618 return options | 639 return options |
619 | 640 |
620 def main(): | 641 def main(): |
621 options = ParseOptions() | 642 options = ParseOptions() |
622 | 643 |
623 # Setup boto environment for gsutil to use | |
624 os.environ['BOTO_CONFIG'] = _BOTO_CONFIG | |
625 | |
626 if options.filters: | 644 if options.filters: |
627 LoadPrivateFilters(options.build_path) | 645 LoadPrivateFilters(options.build_path) |
628 | 646 |
629 acl = 'public-read' | |
630 if options.private: | |
631 acl = 'private' | |
632 | 647 |
633 # Calculate a list of Packages index files to compare against. Whenever we | 648 # Calculate a list of Packages index files to compare against. Whenever we |
634 # upload a package, we check to make sure it's not already stored in one of | 649 # upload a package, we check to make sure it's not already stored in one of |
635 # the packages files we uploaded. This list of packages files might contain | 650 # the packages files we uploaded. This list of packages files might contain |
636 # both board and host packages. | 651 # both board and host packages. |
637 pkg_indexes = _GrabAllRemotePackageIndexes(options.previous_binhost_url) | 652 pkg_indexes = _GrabAllRemotePackageIndexes(options.previous_binhost_url) |
638 | 653 |
639 version = GetVersion() | 654 version = GetVersion() |
640 if options.prepend_version: | 655 if options.prepend_version: |
641 version = '%s-%s' % (options.prepend_version, version) | 656 version = '%s-%s' % (options.prepend_version, version) |
642 | 657 |
643 uploader = PrebuiltUploader(options.upload, acl, options.binhost_base_url, | 658 acl = 'public-read' |
659 binhost_base_url = options.binhost_base_url | |
660 | |
661 if options.private: | |
662 acl = 'private' | |
663 binhost_base_url = options.upload | |
664 | |
665 if options.acl_file: | |
666 acl = options.acl_file | |
667 | |
668 uploader = PrebuiltUploader(options.upload, acl, binhost_base_url, | |
644 pkg_indexes) | 669 pkg_indexes) |
645 | 670 |
646 if options.sync_host: | 671 if options.sync_host: |
647 uploader._SyncHostPrebuilts(options.build_path, version, options.key, | 672 uploader._SyncHostPrebuilts(options.build_path, version, options.key, |
648 options.git_sync, options.sync_binhost_conf) | 673 options.git_sync, options.sync_binhost_conf) |
649 | 674 |
650 if options.board: | 675 if options.board: |
651 uploader._SyncBoardPrebuilts(options.board, options.build_path, version, | 676 uploader._SyncBoardPrebuilts(options.board, options.build_path, version, |
652 options.key, options.git_sync, | 677 options.key, options.git_sync, |
653 options.sync_binhost_conf) | 678 options.sync_binhost_conf) |
654 | 679 |
655 if __name__ == '__main__': | 680 if __name__ == '__main__': |
656 main() | 681 main() |
OLD | NEW |