Refactor a copy of dartium/buildbot_annotated_steps.py

tools/dartium/upload_steps.py

+#!/usr/bin/python
+
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Steps to archive dartium, content_shell, and chromedriver from buildbots.
+
+Imported by buildbot_annotated_steps.py and multivm_archive.py
+"""
+
+import imp
+import os
+import platform
+import re
+import subprocess
+import sys
+
+import archive
+
+BUILDER_NAME = 'BUILDBOT_BUILDERNAME'
+REVISION = 'BUILDBOT_REVISION'
+BUILDER_PATTERN = (r'^(dartium|multivm)-(mac|lucid64|lucid32|win)'
+   r'-(full|inc|debug|build)(-ninja)?(-(be|dev|stable|integration))?$')
+
+if platform.system() == 'Windows':
+  GSUTIL = 'e:/b/build/scripts/slave/gsutil.bat'
+else:
+  GSUTIL = '/b/build/scripts/slave/gsutil'
+ACL = 'public-read'
+GS_SITE = 'gs://'
+GS_URL = 'https://sandbox.google.com/storage/'
+GS_DIR = 'dartium-archive'
+LATEST = 'latest'
+CONTINUOUS = 'continuous'
+
+SRC_PATH = os.path.dirname(

[ricow1, 2014/05/14 17:28:18]

as mentioned elsewhere, this should be in utils I think

+           os.path.dirname(
+           os.path.dirname(
+           os.path.dirname(os.path.abspath(__file__)))))
+DART_PATH = os.path.join(SRC_PATH, 'dart')
+
+bot_utils = imp.load_source('bot_utils',
+    os.path.join(DART_PATH, 'tools', 'bots', 'bot_utils.py'))
+
+class BuildInfo(object):
+  """
+    name: A name for the build - the buildbot host if a buildbot.
+    mode: 'Debug' or 'Release'
+    arch: target architecture
+    channel: the channel this build is happening on
+    is_full: True if this is a full build.
+    is_incremental: True if this is an incremental build.
+    is_build: True if this is a builder for the performance testers.
+    is_win_ninja: True if this is a ninja build on Windows.
+
+  """
+  def __init__(self, revision, version):
+
+    self.revision = revision
+    self.version = version
+    # Populate via builder environment variables.
+    self.name = os.environ[BUILDER_NAME]
+    # Temporary hack, until we rename the FYI bots.
+    # We should eventually rename all to linux32 and linux64.
+    self.name = self.name.replace('-linux-', '-lucid64-')
+
+    self.is_incremental = '-inc' in self.name
+    self.is_win_ninja = 'win-inc-ninja' in self.name
+    pattern = re.match(BUILDER_PATTERN, self.name)
+    assert pattern
+    self.arch = 'x64' if pattern.group(2) == 'lucid64' else 'ia32'
+    self.mode = 'Debug' if pattern.group(3) == 'debug' else 'Release'
+    self.is_full = pattern.group(3) == 'full'
+    self.is_build = pattern.group(3) == 'build'
+    self.channel = pattern.group(6) if pattern.group(6) else 'be'
+
+
+def ArchiveAndUpload(info, archive_latest=False):
+  print '@@@BUILD_STEP dartium_generate_archive@@@'
+  cwd = os.getcwd()
+
+  dartium_bucket = info.name
+  dartium_bucket = dartium_bucket.replace('multivm', 'multivm-dartium')
+  drt_bucket = dartium_bucket.replace('dartium', 'drt')
+  chromedriver_bucket = dartium_bucket.replace('dartium', 'chromedriver')
+  dartium_archive = dartium_bucket + '-' + info.version
+  drt_archive = drt_bucket + '-' + info.version
+  chromedriver_archive = chromedriver_bucket + '-' + info.version
+  dartium_zip, drt_zip, chromedriver_zip = archive.Archive(
+      SRC_PATH,
+      info.mode,
+      dartium_archive,
+      drt_archive,
+      chromedriver_archive,
+      is_win_ninja=info.is_win_ninja)
+
+  status = 0
+  # Upload bleeding-edge builds to old dartium-archive bucket
+  if info.channel == 'be':

[ricow1, 2014/05/14 17:28:18]

that is true, we upload bleeding edge to the old buckets, and dev/stable to the
new ones, I forgot about this when we discussed this earlier today

[Bill Hesse, 2014/05/15 10:46:37]

We also upload bleeding-edge to the new buckets.

+    status = (OldUpload('dartium', dartium_bucket,
+                        os.path.abspath(dartium_zip),
+                        archive_latest=archive_latest)
+              or OldUpload('drt', drt_bucket,
+                           os.path.abspath(drt_zip),
+                           archive_latest=archive_latest)
+              or OldUpload('chromedriver', chromedriver_bucket,
+                           os.path.abspath(chromedriver_zip),
+                           archive_latest=archive_latest))
+
+  # Upload to new dart-archive bucket using GCSNamer, but not incremental
+  # or perf builder builds.
+  if not info.is_incremental and not info.is_build:
+    Upload('dartium', os.path.abspath(dartium_zip),
+           info, archive_latest=archive_latest)
+    Upload('drt', os.path.abspath(drt_zip),
+           info, archive_latest=archive_latest)
+    Upload('chromedriver', os.path.abspath(chromedriver_zip),
+           info, archive_latest=archive_latest)
+
+  os.chdir(cwd)
+  if status != 0:
+    print '@@@STEP_FAILURE@@@'
+  return status
+
+
+def OldUpload(module, bucket, zip_file, archive_latest=False):
+  """Upload a zip file to the old bucket gs://dartium-archive/
+  """
+  # TODO(whesse): Remove the old archiving code (OldUpload, OldUploadFile,
+  # and constants they use) once everything points to the new location.
+  status = 0
+  _, filename = os.path.split(zip_file)
+  if not archive_latest:
+    target = '/'.join([GS_DIR, bucket, filename])
+    print '@@@BUILD_STEP %s_upload_archive_old@@@' % module
+    status = OldUploadFile(zip_file, GS_SITE + target)
+    print '@@@STEP_LINK@download@' + GS_URL + target + '@@@'
+  else:
+    print '@@@BUILD_STEP %s_upload_latest_old@@@' % module
+    # Clear latest for this build type.
+    old = '/'.join([GS_DIR, LATEST, bucket + '-*'])
+    old_archives = ListArchives(GS_SITE + old)
+
+    # Upload the new latest and remove unnecessary old ones.
+    target = GS_SITE + '/'.join([GS_DIR, LATEST, filename])
+    status = OldUploadFile(zip_file, target)
+    if status == 0:
+      RemoveArchives(
+          [iarch for iarch in old_archives if iarch != target])
+    else:
+      print 'Upload failed'
+
+    # Upload unversioned name to continuous site for incremental
+    # builds.
+    if '-inc' in bucket:
+      continuous_name = bucket[:bucket.find('-inc')]
+      target = GS_SITE + '/'.join([GS_DIR, CONTINUOUS,
+                                   continuous_name + '.zip'])
+      status = OldUploadFile(zip_file, target)
+
+  print ('@@@BUILD_STEP %s_upload_archive is over (status = %s)@@@' %
+      (module, status))
+  return status
+
+
+def Upload(module, zip_file, info, archive_latest=False):
+  """Upload a zip file to cloud storage bucket gs://dart-archive/
+  """
+  print '@@@BUILD_STEP %s_upload_archive @@@' % module
+  revision = 'latest' if archive_latest else info.revision
+  name = module.replace('drt', 'content_shell')
+  namer = bot_utils.GCSNamer(info.channel, bot_utils.ReleaseType.RAW)
+  remote_path = namer.dartium_variant_zipfilepath(revision,
+                                                  name,
+                                                  sys.platform,
+                                                  info.arch,
+                                                  info.mode.lower())
+  UploadFile(zip_file, remote_path, create_md5sum=True)
+
+  print '@@@STEP_LINK@download@' + remote_path + '@@@'
+
+
+def OldUploadFile(source, target):
+  """Upload an archive zip file to Google storage.
+  """
+
+  # Upload file.
+  cmd = [GSUTIL, 'cp', source, target]
+  (status, output) = ExecuteCommand(cmd)
+  if status != 0:
+    return status
+  print 'Uploaded: ' + output
+
+  # Set ACL.
+  if ACL is not None:
+    cmd = [GSUTIL, 'setacl', ACL, target]
+    (status, output) = ExecuteCommand(cmd)
+  return status
+
+
+def UploadFile(local_path, remote_path, create_md5sum=False):
+  # Copy it to the new unified gs://dart-archive bucket
+  gsutil = bot_utils.GSUtil()
+  gsutil.upload(local_path, remote_path, public=True)
+  if create_md5sum:
+    # 'local_path' may have a different filename than 'remote_path'. So we need
+    # to make sure the *.md5sum file contains the correct name.
+    assert '/' in remote_path and not remote_path.endswith('/')
+    mangled_filename = remote_path[remote_path.rfind('/') + 1:]
+    local_md5sum = bot_utils.CreateChecksumFile(local_path, mangled_filename)
+    gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True)
+
+
+def ExecuteCommand(cmd):
+  """Execute a command in a subprocess.
+  """
+  print 'Executing: ' + ' '.join(cmd)
+  try:
+    pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    (output, error) = pipe.communicate()
+    if pipe.returncode != 0:
+      print 'Execution failed: ' + str(error)
+    return (pipe.returncode, output)
+  except:
+    import traceback
+    print 'Execution raised exception:', traceback.format_exc()
+    return (-1, '')
+
+
+def UploadDartTestsResults(layout_test_results_dir, name, version,
+                           component, checked):
+  """Uploads test results to google storage.
+  """
+  print ('@@@BUILD_STEP archive %s_layout_%s_tests results@@@' %
+         (component, checked))
+  dir_name = os.path.dirname(layout_test_results_dir)
+  base_name = os.path.basename(layout_test_results_dir)
+  cwd = os.getcwd()
+  os.chdir(dir_name)
+
+  archive_name = 'layout_test_results.zip'
+  archive.ZipDir(archive_name, base_name)
+
+  target = '/'.join([GS_DIR, 'layout-test-results', name, component + '-' +
+                     checked + '-' + version + '.zip'])
+  status = OldUploadFile(os.path.abspath(archive_name), GS_SITE + target)
+  os.remove(archive_name)
+  if status == 0:
+    print ('@@@STEP_LINK@download@' + GS_URL + target + '@@@')
+  else:
+    print '@@@STEP_FAILURE@@@'
+  os.chdir(cwd)
+
+
+def ListArchives(pattern):
+  """List the contents in Google storage matching the file pattern.
+  """
+  cmd = [GSUTIL, 'ls', pattern]
+  (status, output) = ExecuteCommand(cmd)
+  if status != 0:
+    return []
+  return output.split(os.linesep)
+
+
+def RemoveArchives(archives):
+  """Remove the list of archives in Google storage.
+  """
+  for archive in archives:
+    if archive.find(GS_SITE) == 0:
+      cmd = [GSUTIL, 'rm', archive.rstrip()]
+      (status, _) = ExecuteCommand(cmd)
+      if status != 0:
+        return status
+  return 0