Update cbuildbot.py to upload prebuilts from preflight buildbot.

prebuilt.py

 #!/usr/bin/python
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import multiprocessing
 import optparse
 import os
 import re
@@ skipping 41 matching lines @@
 # Private overlays to look at for builds to filter
 # relative to build path
 _PRIVATE_OVERLAY_DIR = 'src/private-overlays'
 _BINHOST_BASE_DIR = 'src/overlays'
 #_BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt'
 _BINHOST_BASE_URL = 'http://gsdview.appspot.com/chromeos-prebuilt'
 _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
 # Created in the event of new host targets becoming available
 _PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
                                              'make.conf.amd64-host')}
+_BINHOST_CONF_DIR = 'src/third_party/chromiumos-overlay/chromeos/binhost'

[sosa, 2010/11/24 00:40:02]

As an FYI this list is sorta getting out of hand.  We should probably consider
storing it somewhere else besides a list of constants.  

[scottz, 2010/11/29 17:56:39]

+1
On 2010/11/24 00:40:02, sosa wrote:

 
 
 class FiltersEmpty(Exception):
   """Raised when filters are used but none are found."""
   pass
 
 
 class UploadFailed(Exception):
   """Raised when one of the files uploaded failed."""
   pass
@@ skipping 13 matching lines @@
   Note quotes are added automatically
 
   Args:
     filename: Name of file to modify.
     value: Value to write with the key.
     key: The variable key to update. (Default: PORTAGE_BINHOST)
   """
   file_fh = open(filename)
   file_lines = []
   found = False
+  keyval_str = '%(key)s=%(value)s'
   for line in file_fh:
     # Strip newlines from end of line. We already add newlines below.
     line = line.rstrip("\n")
 
     if len(line.split('=')) != 2:
       # Skip any line that doesn't fit key=val.
       file_lines.append(line)
       continue
 
     file_var, file_val = line.split('=')
-    keyval_str = '%(key)s=%(value)s'
     if file_var == key:
       found = True
       print 'Updating %s=%s to %s="%s"' % (file_var, file_val, key, value)
       value = '"%s"' % value
       file_lines.append(keyval_str % {'key': key, 'value': value})
     else:
       file_lines.append(keyval_str % {'key': file_var, 'value': file_val})
 
   if not found:
     file_lines.append(keyval_str % {'key': key, 'value': value})
 
   file_fh.close()
   # write out new file
   new_file_fh = open(filename, 'w')
-  new_file_fh.write('\n'.join(file_lines))
+  new_file_fh.write('\n'.join(file_lines) + '\n')
   new_file_fh.close()
 
 
 def RevGitPushWithRetry(retries=5):
   """Repo sync and then push git changes in flight.
 
     Args:
       retries: The number of times to retry before giving up, default: 5
 
     Raises:
       GitPushFailed if push was unsuccessful after retries
   """
   for retry in range(1, retries+1):
     try:
       cros_build_lib.RunCommand('repo sync .', shell=True)
       cros_build_lib.RunCommand('git push', shell=True)
       break
     except cros_build_lib.RunCommandError:
       if retry < retries:
         print 'Error pushing changes trying again (%s/%s)' % (retry, retries)
         time.sleep(5*retry)
   else:
     raise GitPushFailed('Failed to push change after %s retries' % retries)
 
 
-def RevGitFile(filename, value, retries=5):
+def RevGitFile(filename, value, retries=5, key='PORTAGE_BINHOST'):
   """Update and push the git file.
 
     Args:
       filename: file to modify that is in a git repo already
       value: string representing the version of the prebuilt that has been
         uploaded.
       retries: The number of times to retry before giving up, default: 5
+      key: The variable key to update in the git file.
+        (Default: PORTAGE_BINHOST)
   """
   prebuilt_branch = 'prebuilt_branch'
   old_cwd = os.getcwd()
   os.chdir(os.path.dirname(filename))
 
   cros_build_lib.RunCommand('repo sync .', shell=True)
   cros_build_lib.RunCommand('repo start %s .' % prebuilt_branch, shell=True)
   git_ssh_config_cmd = (
     'git config url.ssh://git@gitrw.chromium.org:9222.pushinsteadof '
     'http://git.chromium.org/git')
   cros_build_lib.RunCommand(git_ssh_config_cmd, shell=True)
-  description = 'Update PORTAGE_BINHOST="%s" in %s' % (value, filename)
+  description = 'Update %s="%s" in %s' % (key, value, filename)
   print description
   try:
-    UpdateLocalFile(filename, value)
+    UpdateLocalFile(filename, value, key)
     cros_build_lib.RunCommand('git config push.default tracking', shell=True)
     cros_build_lib.RunCommand('git commit -am "%s"' % description, shell=True)
     RevGitPushWithRetry(retries)
   finally:
     cros_build_lib.RunCommand('repo abandon %s .' % prebuilt_branch, shell=True)
     os.chdir(old_cwd)
 
 
 def GetVersion():
   """Get the version to put in LATEST and update the git version with."""
@@ skipping 220 matching lines @@
   elif re.match('.*?-\w+', target):
     overlay_str = 'overlay-%s' % target
     make_path = os.path.join(_BINHOST_BASE_DIR, overlay_str, 'make.conf')
   else:
     raise UnknownBoardFormat('Unknown format: %s' % target)
 
   return os.path.join(make_path)
 
 
 def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
-                   board=None, git_sync=False, git_sync_retries=5):
+                   board=None, git_sync=False, git_sync_retries=5,
+                   key='PORTAGE_BINHOST', sync_binhost_conf=False):
   """Upload Host prebuilt files to Google Storage space.
 
   Args:
     build_path: The path to the root of the chroot.
     upload_location: The upload location.
     board: The board to upload to Google Storage, if this is None upload
       host packages.
     git_sync: If set, update make.conf of target to reference the latest
-      prebuilt packages genereated here.
+      prebuilt packages generated here.
     git_sync_retries: How many times to retry pushing when updating git files.
       This helps avoid failures when multiple bots are modifying the same Repo.
       default: 5
+    key: The variable key to update in the git file. (Default: PORTAGE_BINHOST)
+    sync_binhost_conf: If set, update binhost config file in chromiumos-overlay
+      for the current board or host.
   """
 
   if not board:
     # We are uploading host packages
     # TODO(scottz): eventually add support for different host_targets
     package_path = os.path.join(build_path, _HOST_PACKAGES_PATH)
     url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
     package_string = _HOST_TARGET
     git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET])
+    binhost_conf = os.path.join(build_path,
+        '%s/host/%s.conf' % (_BINHOST_CONF_DIR, _HOST_TARGET))

[diandersAtChromium, 2010/11/24 01:30:31]

Just pass 4 params to os.path.join:

os.path.join(build_path, _BINHOST_CONF_DIR, 'host', HOST_TARGET)

[scottz, 2010/11/29 17:56:39]

Agreed but the last one will still need the .conf added. 


On 2010/11/24 01:30:31, diandersAtChromium wrote:

[davidjames, 2010/11/29 21:18:55]

Done.

   else:
     board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
     package_path = os.path.join(board_path, 'packages')
     package_string = board
     url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
     git_file = os.path.join(build_path, DetermineMakeConfFile(board))
+    binhost_conf = os.path.join(build_path,
+        '%s/target/%s.conf' % (_BINHOST_CONF_DIR, board))

[diandersAtChromium, 2010/11/24 01:30:31]

Same: 4 params to os.path.join.

[davidjames, 2010/11/29 21:18:55]

Done.

   remote_location = os.path.join(upload_location, url_suffix)
 
   if upload_location.startswith('gs://'):
     upload_files = GenerateUploadDict(package_path, remote_location)
 
     print 'Uploading %s' % package_string
     failed_uploads = RemoteUpload(upload_files)
     if len(failed_uploads) > 1 or (None not in failed_uploads):
       error_msg = ['%s -> %s\n' % args for args in failed_uploads]
       raise UploadFailed('Error uploading:\n%s' % error_msg)
   else:
     ssh_server, remote_path = remote_location.split(':', 1)
     cmds = ['ssh %s mkdir -p %s' % (ssh_server, remote_path),
             'rsync -av %s/ %s/' % (package_path, remote_location)]
     for cmd in cmds:
       if not _RetryRun(cmd, shell=True):
         raise UploadFailed('Could not run %s' % cmd)
 
+  url_value = '%s/%s/' % (binhost_base_url, url_suffix)
+
   if git_sync:
-    url_value = '%s/%s/' % (binhost_base_url, url_suffix)
-    RevGitFile(git_file, url_value, retries=git_sync_retries)
+    RevGitFile(git_file, url_value, retries=git_sync_retries, key=key)
+
+  if sync_binhost_conf:

[scottz, 2010/11/29 17:56:39]

Move this into a function. 

[davidjames, 2010/11/29 21:18:55]

Done.

+    binhost_dir = os.path.dirname(os.path.abspath(binhost_conf))
+    binhost_filename = os.path.basename(binhost_conf)
+    if not os.path.isdir(binhost_dir):
+      os.makedirs(binhost_dir)
+    if not os.path.isfile(binhost_conf):
+      f = file(binhost_conf, 'w')

[scottz, 2010/11/29 17:56:39]

Nit, use something better than f please. I know you use it for three lines but I
still prefer to keep our variable names coherent throughout. 

[davidjames, 2010/11/29 21:18:55]

Done.

+      f.write('FULL_BINHOST="$PORTAGE_BINHOST"\n')
+      f.close()
+    UpdateLocalFile(binhost_conf, url_value, key)
+    cros_build_lib.RunCommand('git add %s' % binhost_filename, cwd=binhost_dir,
+                              shell=True)
+    description = 'Update %s=%s in %s' % (key, url_value, binhost_filename)
+    cros_build_lib.RunCommand('git commit -m "%s"' % description,
+                              cwd=binhost_dir, shell=True)
 
 
 def usage(parser, msg):
   """Display usage message and parser help then exit with 1."""
   print >> sys.stderr, msg
   parser.print_help()
   sys.exit(1)
 
 
 def main():
@@ skipping 13 matching lines @@
                     help='Enable git version sync (This commits to a repo)')
   parser.add_option('-u', '--upload', dest='upload',
                     default=None,
                     help='Upload location')
   parser.add_option('-V', '--prepend-version', dest='prepend_version',
                     default=None,
                     help='Add an identifier to the front of the version')
   parser.add_option('-f', '--filters', dest='filters', action='store_true',
                     default=False,
                     help='Turn on filtering of private ebuild packages')
+  parser.add_option('-k', '--key', dest='key',
+                    default='PORTAGE_BINHOST',
+                    help='Key to update in make.conf / binhost.conf')
+  parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf',
+                    default=False, action='store_true',
+                    help='Update binhost.conf')
 
   options, args = parser.parse_args()
   # Setup boto environment for gsutil to use
   os.environ['BOTO_CONFIG'] = _BOTO_CONFIG
   if not options.build_path:
     usage(parser, 'Error: you need provide a chroot path')
 
   if not options.upload:
     usage(parser, 'Error: you need to provide an upload location using -u')
 
   if options.filters:
     # TODO(davidjames): It might be nice to be able to filter private ebuilds
     # from rsync uploads as well, some day. But for now it's not needed.
     if not options.upload.startswith("gs://"):
       usage(parser, 'Error: filtering only works with gs:// paths')
     LoadPrivateFilters(options.build_path)
 
   version = GetVersion()
   if options.prepend_version:
     version = '%s-%s' % (options.prepend_version, version)
 
   if options.sync_host:
     UploadPrebuilt(options.build_path, options.upload, version,
-                   options.binhost_base_url, git_sync=options.git_sync)
+                   options.binhost_base_url, git_sync=options.git_sync,
+                   key=options.key,
+                   sync_binhost_conf=options.sync_binhost_conf)
 
   if options.board:
     UploadPrebuilt(options.build_path, options.upload, version,
                    options.binhost_base_url, board=options.board,
-                   git_sync=options.git_sync)
+                   git_sync=options.git_sync, key=options.key,
+                   sync_binhost_conf=options.sync_binhost_conf)
 
 
 if __name__ == '__main__':
   main()