Make editor builders archive to gs://dart-archive/ in addition to the other GCS buckets

dart/tools/bots/bot_utils.py

+#!/usr/bin/env python
+#
+# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import hashlib
+import imp
+import os
+import subprocess
+import sys
+
+DART_DIR = os.path.abspath(
+    os.path.normpath(os.path.join(__file__, '..', '..', '..')))
+
+def GetUtils():
+  '''Dynamically load the tools/utils.py python module.'''
+  return imp.load_source('utils', os.path.join(DART_DIR, 'tools', 'utils.py'))
+
+utils = GetUtils()
+
+SYSTEM_RENAMES = {
+  'win32': 'windows',
+  'windows': 'windows',
+  'win': 'windows',
+
+  'linux': 'linux',
+  'linux2': 'linux',
+  'lucid32': 'linux',
+  'lucid64': 'linux',
+
+  'darwin': 'macos',
+  'mac': 'macos',
+  'macos': 'macos',
+}
+
+ARCH_RENAMES = {
+  '32': 'ia32',
+  'ia32': 'ia32',
+
+  '64': 'x64',
+  'x64': 'x64',
+}
+
+class Channel(object):
+  BLEEDING_EDGE = 'be'
+  DEV = 'dev'
+  STABLE = 'stable'
+  ALL_CHANNELS = [BLEEDING_EDGE, DEV, STABLE]
+
+class ReleaseType(object):

[ricow1, 2013/09/16 11:51:30]

To be consistent, should we make a Mode class as well?
I know it will just be for asserting, but anyway

[kustermann, 2013/09/16 14:15:54]

Done.

+  RAW = 'raw'
+  SIGNED = 'signed'
+  RELEASE = 'release'
+  ALL_TYPES = [RAW, SIGNED, RELEASE]
+
+class GCSNamer(object):
+  def __init__(self, channel=Channel.BLEEDING_EDGE,
+      release_type=ReleaseType.RAW):
+    assert channel in Channel.ALL_CHANNELS 
+    assert release_type in ReleaseType.ALL_TYPES
+
+    self.channel = channel
+    self.release_type = release_type
+    self.bucket = 'gs://dart-archive'
+
+  def dartium_directory(self, revision):
+    return self._variant_directory('dartium', revision)
+
+  def sdk_directory(self, revision):
+    return self._variant_directory('sdk', revision)
+
+  def editor_directory(self, revision):
+    return self._variant_directory('editor', revision)
+
+  def editor_eclipse_update_directory(self, revision):
+    return self._variant_directory('editor-eclipse-update', revision)
+
+  def apidocs_directory(self, revision):
+    return self._variant_directory('api-docs', revision)
+
+  def misc_directory(self, revision):
+    return self._variant_directory('misc', revision)
+
+  def version_file(self, revision):
+    return '%s/channels/%s/%s/%s/VERSION' % (self.bucket, self.channel, self.release_type,
+        revision)
+
+  def _variant_directory(self, name, revision):
+    return '%s/channels/%s/%s/%s/%s' % (self.bucket, self.channel, self.release_type,
+        revision, name)
+
+  def apidocs_zipfile(self):

[ricow1, 2013/09/16 11:51:30]

we use zipfilename in other functions

[kustermann, 2013/09/16 14:15:54]

Done.

+    return 'dart-api-docs.zip'
+
+  def editor_zipfilename(self, system, arch):
+    return 'darteditor-%s-%s.zip' % (
+        SYSTEM_RENAMES[system], ARCH_RENAMES[arch])
+
+  def sdk_zipfilename(self, system, arch, mode):
+    assert mode in ['release', 'debug']

[ricow1, 2013/09/16 11:51:30]

if you create a Mode class, then use it here for the assert

[kustermann, 2013/09/16 14:15:54]

Done.

+    return 'dartsdk-%s-%s-%s.zip' % (
+        SYSTEM_RENAMES[system], ARCH_RENAMES[arch], mode)
+
+  def dartium_zipfilename(self, system, arch, mode):
+    return self._dartium_variant_zipfilename('dartium', system, arch, mode)
+
+  def content_shell_zipfilename(self, system, arch, mode):
+    return self._dartium_variant_zipfilename(
+        'content_shell', system, arch, mode)
+
+  def cromedriver_zipfilename(self, system, arch, mode):
+    return self.dartium_variant_zipfilename(
+        'chromedriver', system, arch, mode)
+
+  def dartium_variant_zipfilename(self, name, system, arch, mode):

[ricow1, 2013/09/16 11:51:30]

do we need the above functions if we have this?

[kustermann, 2013/09/16 14:15:54]

Probably not. I started with this function to be private, but I figured it would
be beneficial to expose it.

I'll remove the old specific functions.

+    assert name in ['chromedriver', 'dartium', 'content_shell']
+    assert mode in ['release', 'debug']
+    return '%s-%s-%s-%s.zip' % (
+        name, SYSTEM_RENAMES[system], ARCH_RENAMES[arch], mode)
+
+def run(command, env=None):
+  print "Running command: ", command
+  p = subprocess.Popen(command, stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, env=env)
+  (stdout, stderr) = p.communicate()
+  if p.returncode:

[ricow1, 2013/09/16 11:51:30]

I would actually be specific here
if p.returncode != 0:
 ....

[kustermann, 2013/09/16 14:15:54]

I changed it -- but that's not the "pythonic way"!

+    print >> sys.stderr, "Failed to execute '%s'. Exit code: %s." % (
+        command, p.returncode)
+    print >> sys.stderr, "stdout: ", stdout
+    print >> sys.stderr, "stderr: ", stderr
+    raise Exception("Failed to execute %s." % command)
+
+class GSUtil(object):
+  GSUTIL_PATH = os.path.join(DART_DIR, 'third_party', 'gsutil', 'gsutil')

[ricow1, 2013/09/16 11:51:30]

this will probably not work on dartium, since we don't use the normal DEPS file
there (i.e., we don't have the gsutil tool in this location)

[kustermann, 2013/09/16 14:15:54]

Good point. I've added more logic now to search for gsutil inside all PATH
directories in addition to the dart-gsutil and buildbot gsutil.

+  assert os.path.exists(GSUTIL_PATH)
+
+  def execute(self, gsutil_args):
+    env = dict(os.environ)
+    # If we're on the buildbot, we use a specific boto file.
+    if utils.GetUserName() == 'chrome-bot':
+      boto_config = {
+        'linux': '/mnt/data/b/build/site_config/.boto',
+        'macos': '/Volumes/data/b/build/site_config/.boto',
+        'win32': r'e:\b\build\site_config\.boto',
+      }[utils.GuessOS()]
+      env['AWS_CREDENTIAL_FILE'] = boto_config

[ricow1, 2013/09/16 11:51:30]

do we need to specify this at this point?
This will only be valid for the env used to execute this specific command right
(which is gsutil)

[kustermann, 2013/09/16 14:15:54]

We probably don't need it. But BB contains it in it's default environment and
there is no reason not to include it here (maybe a future version of gsutil will
use it ....).

+      env['BOTO_CONFIG'] = boto_config
+    run([sys.executable, GSUtil.GSUTIL_PATH] + gsutil_args,
+        env=env)
+
+def CalculateChecksum(filename):
+  """Calculate the MD5 checksum for filename."""
+
+  md5 = hashlib.md5()
+
+  with open(filename, 'rb') as f:
+    data = f.read(65536)
+    while len(data) > 0:
+      md5.update(data)
+      data = f.read(65536)
+
+  return md5.hexdigest()
+
+def CreateChecksumFile(filename, mangled_filename=None):
+  """Create and upload an MD5 checksum file for filename."""
+  if not mangled_filename:
+    mangled_filename = os.path.basename(filename)
+
+  checksum = CalculateChecksum(filename)
+  checksum_filename = '%s.md5sum' % filename
+
+  with open(checksum_filename, 'w') as f:
+    f.write('%s *%s' % (checksum, mangled_filename))
+
+  return checksum_filename
+