Update prebuilt.py to handle private google storage uploads.

buildbot/prebuilt.py

Index: buildbot/prebuilt.py
diff --git a/buildbot/prebuilt.py b/buildbot/prebuilt.py
index 7b2e6eac3e329e9e869226cc601235107c65e1a3..df2a08e7e33a0de8ed223e9db13e5f0de4568c45 100755
--- a/buildbot/prebuilt.py
+++ b/buildbot/prebuilt.py
@@ -51,7 +51,6 @@ _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
 _CATEGORIES_PATH = 'chroot/etc/portage/categories'
 _HOST_TARGET = 'amd64'
 _BOARD_PATH = 'chroot/build/%(board)s'
-_BOTO_CONFIG = '/home/chrome-bot/external-boto'
 # board/board-target/version/packages/'
 _REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages'
 # host/host-target/version/packages/'
@@ -278,11 +277,33 @@ def _GsUpload(args):
     Return the arg tuple of two if the upload failed
   """
   (local_file, remote_file, acl) = args
+  CANNED_ACLS = ['public-read', 'private', 'bucket-owner-read',
+                 'authenticated-read', 'bucket-owner-full-control',
+                 'public-read-write']
+  acl_cmd = None
+  if acl in CANNED_ACLS:
+    cmd = '%s cp -a %s %s %s' % (_GSUTIL_BIN, acl, local_file, remote_file)
+  else:
+    # if acl is not a canned acl we upload everything private and check below if
+    # it is a file we can use to set acls
+    cmd = '%s cp -a private %s %s' % (_GSUTIL_BIN, local_file, remote_file)
+    if not os.path.exists(acl):
+      print ('You are specifying either a file that does not exist or an '
+            'unknown canned acl. %s aborting upload') % acl

[davidjames, 2011/03/16 21:29:39]

Could you line up the quotes here? Also, I don't understand what "%s aborting
upload" means. Maybe this instead?

 You are specifying either a file that does not exist or an
 unknown canned acl: %s. Aborting upload.

[scottz, 2011/03/16 21:36:06]

Done.

+      # Since we are already a different process  exiting seems to be the most

[davidjames, 2011/03/16 21:29:39]

Could you add a comma between "process" and "exiting" and get rid of the double
space?

[scottz, 2011/03/16 21:36:06]

Done.

+      # sane thing to do here.
+      sys.exit(1)

[davidjames, 2011/03/16 21:29:39]

Did you test whether the parent process catch this error and exit appropriately?

[scottz, 2011/03/16 21:36:06]

Actually that is a good point I am moving this to just return as if the upload
failed for each object since that is basically what is happening

On 2011/03/16 21:29:39, davidjames wrote:

+
+    acl_cmd = '%s setacl %s %s' % (_GSUTIL_BIN, acl, remote_file)
 
-  cmd = '%s cp -a %s %s %s' % (_GSUTIL_BIN, acl, local_file, remote_file)
   if not _RetryRun(cmd, print_cmd=False, shell=True):
     return (local_file, remote_file)
 
+  if acl_cmd:
+    # Apply the passed in ACL xml file to the uploaded object.
+    _RetryRun(acl_cmd, print_cmd=False, shell=True)
+
+
 def RemoteUpload(acl, files, pool=10):
   """Upload to google storage.
 
@@ -572,6 +593,8 @@ def usage(parser, msg):
 
 def ParseOptions():
   parser = optparse.OptionParser()
+  parser.add_option('-a', '--acl-file', dest='acl_file', default=None,
+                    help='ACL File for Google Storage files')
   parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url',
                     default=_BINHOST_BASE_URL,
                     help='Base URL to use for binhost in make.conf updates')
@@ -611,24 +634,25 @@ def ParseOptions():
     usage(parser, 'Error: you need provide a chroot path')
   if not options.upload:
     usage(parser, 'Error: you need to provide an upload location using -u')
-  if options.private and not (options.binhost_base_url.startswith('gs://') and
-                              options.upload.startswith('gs://')):
-    usage(parser, 'Error: --private is only valid for gs:// URLs.\n'
-                  'Both --binhost-base-url and --upload must be gs:// URLs.')
+
+  if options.acl_file and not os.path.isfile(options.acl_file):
+    usage(parser, 'Error: ACL file provided is not a file')
+
+  if options.private:
+    if not options.upload.startswith('gs://'):
+      usage(parser, 'Error: --private is only valid for gs:// URLs.\n'
+                    '--upload must be a gs:// URL.')
+    if options.binhost_base_url != _BINHOST_BASE_URL:
+        usage(parser, 'Error: when using --private the --binhost-base-url '
+                      'is automatically derived.')
   return options
 
 def main():
   options = ParseOptions()
 
-  # Setup boto environment for gsutil to use
-  os.environ['BOTO_CONFIG'] = _BOTO_CONFIG
-
   if options.filters:
     LoadPrivateFilters(options.build_path)
 
-  acl = 'public-read'
-  if options.private:
-    acl = 'private'
 
   # Calculate a list of Packages index files to compare against. Whenever we
   # upload a package, we check to make sure it's not already stored in one of
@@ -640,7 +664,17 @@ def main():
   if options.prepend_version:
     version = '%s-%s' % (options.prepend_version, version)
 
-  uploader = PrebuiltUploader(options.upload, acl, options.binhost_base_url,
+  acl = 'public-read'
+  binhost_base_url = options.binhost_base_url
+
+  if options.private:
+    acl = 'private'
+    binhost_base_url = options.upload
+
+  if options.acl_file:
+    acl = options.acl_file
+
+  uploader = PrebuiltUploader(options.upload, acl, binhost_base_url,
                               pkg_indexes)
 
   if options.sync_host: