Change _ArchiveTestResults to upload to Google Storage

bin/cbuildbot.py

 #!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
 
 import errno
 import heapq
 import re
 import optparse
 import os
 import shutil
 import sys
 
 import cbuildbot_comm
 from cbuildbot_config import config
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
-from cros_build_lib import Die, Info, RunCommand, Warning
+from cros_build_lib import Die, Info, RunCommand, RunCommandWithRetries, Warning
 
 _DEFAULT_RETRIES = 3
 ARCHIVE_BASE = '/var/www/archive'
 ARCHIVE_COUNT = 10
 
 # ======================== Utility functions ================================
 
 def MakeDir(path, parents=False):
   """Basic wrapper around os.mkdirs.
 
@@ skipping 362 matching lines @@
   else:
     overlays = [public_overlay, private_overlay]
   RunCommand(['./cros_mark_as_stable', '--srcroot=..',
               '--board=%s' % board,
               '--overlays=%s' % " ".join(overlays),
               '--tracking_branch=%s' % tracking_branch,
               '--push_options=--bypass-hooks -f', 'push'],
              cwd=cwd)
 
 
-def _ArchiveTestResults(buildroot, board, archive_dir, test_results_dir):
-  """Archives the test results into the www dir for later use.
+def _ArchiveTestResults(buildroot, board, test_results_dir,
+                        gsutil, archive_dir, acl):
+  """Archives the test results into Google Storage
 
-  Takes the results from the test_results_dir and dumps them into the archive
-  dir specified.  This also archives the last qemu image.
+  Takes the results from the test_results_dir and the last qemu image and
+  uploads them to Google Storage.
 
-  board:  Board to find the qemu image.
-  archive_dir:  Path from ARCHIVE_BASE to store image.
-  test_results_dir: Path from buildroot/chroot to find test results.  This must
-    a subdir of /tmp.
+  Arguments:

[sosa, 2010/11/12 21:41:45]

buildroot?

+    board: Board to find the qemu image.
+    test_results_dir: Path from buildroot/chroot to find test results.
+      This must a subdir of /tmp.
+    gsutil: Location of gsutil
+    archive_dir: Google Storage path to store the archive
+    acl: ACL to set on archive in Google Storage
   """
+  num_gsutil_retries = 5
   test_results_dir = test_results_dir.lstrip('/')
-  if not os.path.exists(ARCHIVE_BASE):
-    os.makedirs(ARCHIVE_BASE)
-  else:
-    dir_entries = os.listdir(ARCHIVE_BASE)
-    if len(dir_entries) >= ARCHIVE_COUNT:
-      oldest_dirs = heapq.nsmallest((len(dir_entries) - ARCHIVE_COUNT) + 1,
-          [os.path.join(ARCHIVE_BASE, filename) for filename in dir_entries],
-          key=lambda fn: os.stat(fn).st_mtime)
-      Info('Removing archive dirs %s' % oldest_dirs)
-      for oldest_dir in oldest_dirs:
-        shutil.rmtree(os.path.join(ARCHIVE_BASE, oldest_dir))
-
-  archive_target = os.path.join(ARCHIVE_BASE, str(archive_dir))
-  if os.path.exists(archive_target):
-    shutil.rmtree(archive_target)
-
   results_path = os.path.join(buildroot, 'chroot', test_results_dir)
   RunCommand(['sudo', 'chmod', '-R', '+r', results_path])
   try:
-    shutil.copytree(results_path, archive_target)
-  except:
-    Warning('Some files could not be copied')
+    # gsutil has the ability to resume an upload when the command is retried
+    RunCommandWithRetries([gsutil, 'cp', '-R', results_path, archive_dir],
+                          num_gsutil_retries)
+    RunCommand([gsutil, 'setacl', acl, archive_dir])
 
-  image_name = 'chromiumos_qemu_image.bin'
-  image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
-                            'latest', image_name)
-  RunCommand(['gzip', '-f', '--fast', image_path])
-  shutil.copyfile(image_path + '.gz', os.path.join(archive_target,
-                                                   image_name + '.gz'))
-
+    image_name = 'chromiumos_qemu_image.bin'
+    image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
+                              'latest', image_name)
+    RunCommand(['gzip', '-f', '--fast', image_path])
+    RunCommandWithRetries([gsutil, 'cp', image_path + '.gz', archive_dir],
+                          num_gsutil_retries)
+  except Exception, e:
+    Warning('Could not archive test results (error=%s)' % str(e))
 
 
 def _GetConfig(config_name):
   """Gets the configuration for the build"""
   default = config['default']
   buildconfig = {}
   if not config.has_key(config_name):
     Warning('Non-existent configuration specified.')
     Warning('Please specify one of:')
     config_names = config.keys()
@@ skipping 25 matching lines @@
                     default=False,
                     help='Clobbers an old checkout before syncing')
   parser.add_option('--debug', action='store_true', dest='debug',
                     default=False,
                     help='Override some options to run as a developer.')
   parser.add_option('-t', '--tracking-branch', dest='tracking_branch',
                     default='cros/master', help='Run the buildbot on a branch')
   parser.add_option('-u', '--url', dest='url',
                     default='http://git.chromium.org/git/manifest',
                     help='Run the buildbot on internal manifest')
+  parser.add_option('-g', '--gsutil', default='', help='Location of gsutil')
+  parser.add_option('-c', '--gsutil_archive', default='',
+                    help='Datastore archive location')
+  parser.add_option('-a', '--acl', default='private',
+                    help='ACL to set on GSD archives')
 
   (options, args) = parser.parse_args()
 
   buildroot = options.buildroot
   revisionfile = options.revisionfile
   tracking_branch = options.tracking_branch
 
   if len(args) >= 1:
     buildconfig = _GetConfig(args[-1])
   else:
@@ skipping 26 matching lines @@
       _RunUnitTests(buildroot)
 
     _BuildImage(buildroot)
 
     if buildconfig['smoke_bvt']:
       _BuildVMImageForTesting(buildroot)
       test_results_dir = '/tmp/run_remote_tests.%s' % options.buildnumber
       try:
         _RunSmokeSuite(buildroot, test_results_dir)
       finally:
+        archive_full_path=os.path.join(options.gsutil_archive,
+                                       str(options.buildnumber))
         _ArchiveTestResults(buildroot, buildconfig['board'],
-                            archive_dir=options.buildnumber,
-                            test_results_dir=test_results_dir)
+                            test_results_dir=test_results_dir,
+                            gsutil=options.gsutil,
+                            archive_dir=archive_full_path,

[sosa, 2010/11/12 21:41:45]

You have tabs here.  This is bad.

+                            acl=options.acl)
 
     if buildconfig['uprev']:
       # Don't push changes for developers.
       if not options.debug:
         if buildconfig['master']:
           # Master bot needs to check if the other slaves completed.
           if cbuildbot_comm.HaveSlavesCompleted(config):
             _UprevPush(buildroot, tracking_branch, buildconfig['board'],
                        buildconfig['overlays'])
           else:
             Die('CBUILDBOT - One of the slaves has failed!!!')
 
         else:
           # Publish my status to the master if its expecting it.
           if buildconfig['important']:
             cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
 
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']:
       cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
 
     raise
 
 
 if __name__ == '__main__':
     main()