Make editor builders archive to gs://dart-archive/ in addition to the other GCS buckets

dart/editor/build/build.py

 #!/usr/bin/env python
 #
 # Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import glob
 import gsutil
 import hashlib
 import imp
@@ skipping 10 matching lines @@
 
 BUILD_OS = None
 DART_PATH = None
 TOOLS_PATH = None
 
 GSU_PATH_REV = None
 GSU_PATH_LATEST = None
 GSU_API_DOCS_PATH = None
 GSU_API_DOCS_BUCKET = 'gs://dartlang-api-docs'
 
+CHANNEL = None
+PLUGINS_BUILD = None
 REVISION = None
+SYSTEM = None
 TRUNK_BUILD = None
-PLUGINS_BUILD = None
 
 NO_UPLOAD = None
 
-def GetUtils():
-  '''Dynamically load the tools/utils.py python module.'''
-  dart_dir = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
-  return imp.load_source('utils', os.path.join(dart_dir, 'tools', 'utils.py'))
+DART_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
+utils = imp.load_source('utils', os.path.join(DART_DIR, 'tools', 'utils.py'))
+bot_utils = imp.load_source('bot_utils',
+    os.path.join(DART_DIR, 'tools', 'bots', 'bot_utils.py'))
 
-utils = GetUtils()
+def DartArchiveFile(local_path, remote_path, create_md5sum=False):
+  # Copy it to the new unified gs://dart-archive bucket
+  # TODO(kustermann/ricow): Remove all the old archiving code, once everything
+  # points to the new location
+  gsutil = bot_utils.GSUtil()
+  gsutil.upload(local_path, remote_path, public=True)
+  if create_md5sum:
+    # 'local_path' may have a different filename than 'remote_path'. So we need
+    # to make sure the *.md5sum file contains the correct name.
+    assert '/' in remote_path and not remote_path.endswith('/')
+    mangled_filename = remote_path[remote_path.rfind('/') + 1:]
+    local_md5sum = bot_utils.CreateChecksumFile(local_path, mangled_filename)
+    gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True)
+
+def DartArchiveUploadEditorZipFile(zipfile):
+  # TODO(kustermann): We don't archive trunk builds to gs://dart-archive/
+  # Remove this once the channel transition is done.
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  gsutil = bot_utils.GSUtil()
+
+  basename = os.path.basename(zipfile)
+  system = None
+  arch = None
+  if basename.startswith('darteditor-linux'):
+    system = 'linux'
+  elif basename.startswith('darteditor-mac'):
+    system = 'macos'
+  elif basename.startswith('darteditor-win'):
+    system = 'windows'
+
+  if basename.endswith('-32.zip'):
+    arch = 'ia32'
+  elif basename.endswith('-64.zip'):
+    arch = 'x64'
+
+  assert system and arch
+
+  for revision in [REVISION, 'latest']:
+    DartArchiveFile(zipfile, namer.editor_zipfilepath(revision, system, arch),
+        create_md5sum=True)
+
+def DartArchiveUploadUpdateSite(local_path):
+  # TODO(kustermann): We don't archive trunk builds to gs://dart-archive/
+  # Remove this once the channel transition is done.
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  gsutil = bot_utils.GSUtil()
+  for revision in [REVISION, 'latest']:
+    update_site_dir = namer.editor_eclipse_update_directory(revision)
+    try:
+      gsutil.remove(update_site_dir, recursive=True)
+    except:
+      # Ignore this, in the general case there is nothing.
+      pass
+    gsutil.upload(local_path, update_site_dir, recursive=True, public=True)
+
+def DartArchiveUploadSDKs(system, sdk32_zip, sdk64_zip):
+  # TODO(kustermann): We don't archive trunk builds to gs://dart-archive/
+  # Remove this once the channel transition is done.
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  for revision in [REVISION, 'latest']:
+    path32 = namer.sdk_zipfilepath(revision, system, 'ia32', 'release')
+    path64 = namer.sdk_zipfilepath(revision, system, 'x64', 'release')
+    DartArchiveFile(sdk32_zip, path32, create_md5sum=True)
+    DartArchiveFile(sdk64_zip, path64, create_md5sum=True)
+
+def DartArchiveUploadAPIDocs(api_zip):
+  # TODO(kustermann): We don't archive trunk builds to gs://dart-archive/
+  # Remove this once the channel transition is done.
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  for revision in [REVISION, 'latest']:
+    destination = (namer.apidocs_directory(revision) + '/' +
+        namer.apidocs_zipfilename())
+    DartArchiveFile(api_zip, destination, create_md5sum=False)
+
+def DartArchiveUploadVersionFile(version_file):
+  # TODO(kustermann): We don't archive trunk builds to gs://dart-archive/.
+  # Remove this once the channel transition is done.
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  for revision in [REVISION, 'latest']:
+    DartArchiveFile(version_file, namer.version_filepath(revision),
+        create_md5sum=False)
 
 class AntWrapper(object):
   """A wrapper for ant build invocations"""
 
   _antpath = None
   _bzippath = None
   _propertyfile = None
 
   def __init__(self, propertyfile, antpath='/usr/bin', bzippath=None):
     """Initialize the ant path.
@@ skipping 119 matching lines @@
                     help='Output Directory.',
                     action='store')
   result.add_option('--dest',
                     help='Output Directory.',
                     action='store')
   return result
 
 def main():
   """Main entry point for the build program."""
   global BUILD_OS
+  global CHANNEL
   global DART_PATH
-  global TOOLS_PATH
-  global GSU_PATH_REV
   global GSU_API_DOCS_PATH
   global GSU_PATH_LATEST
+  global GSU_PATH_REV
+  global NO_UPLOAD
+  global PLUGINS_BUILD
   global REVISION
+  global SYSTEM
+  global TOOLS_PATH
   global TRUNK_BUILD
-  global PLUGINS_BUILD
-  global NO_UPLOAD
 
   if not sys.argv:
     print 'Script pathname not known, giving up.'
     return 1
 
   scriptdir = os.path.abspath(os.path.dirname(sys.argv[0]))
   editorpath = os.path.abspath(os.path.join(scriptdir, '..'))
   thirdpartypath = os.path.abspath(os.path.join(scriptdir, '..', '..',
                                                 'third_party'))
   toolspath = os.path.abspath(os.path.join(scriptdir, '..', '..',
@@ skipping 90 matching lines @@
     # Get user name. If it does not start with chrome then deploy to the test
     # bucket; otherwise deploy to the continuous bucket.
     username = os.environ.get('USER')
     if username is None:
       username = os.environ.get('USERNAME')
 
     if username is None:
       print 'Could not find username'
       return 6
 
-    # dart-editor[-trunk], dart-editor-(win/mac/linux)[-trunk]
+    # dart-editor[-trunk], dart-editor-(win/mac/linux)[-trunk/be/dev/stable]
     builder_name = str(options.name)
 
-    TRUNK_BUILD = builder_name.endswith("-trunk")
-    PLUGINS_BUILD = (builder_name == 'dart-editor' or
-                     builder_name == 'dart-editor-trunk')
+    EDITOR_REGEXP = (r'^dart-editor(-(?P<system>(win|mac|linux)))?' +
+                      '(-(?P<channel>(trunk|be|dev|stable)))?$')
+    match = re.match(EDITOR_REGEXP, builder_name)
+    if not match:
+      raise Exception("Buildername '%s' does not match pattern '%s'."
+                      % (builder_name, EDITOR_REGEXP))
+
+    CHANNEL = match.groupdict()['channel'] or 'be'
+    SYSTEM = match.groupdict()['system']
+
+    TRUNK_BUILD = CHANNEL == 'trunk'
+    PLUGINS_BUILD = SYSTEM is None
+    REVISION = revision
 
     build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
     sdk_environment = os.environ
     if username.startswith('chrome'):
       if TRUNK_BUILD:
         to_bucket = 'gs://dart-editor-archive-trunk'
       else:
         to_bucket = 'gs://dart-editor-archive-continuous'
       running_on_buildbot = True
     else:
       to_bucket = 'gs://dart-editor-archive-testing'
       running_on_buildbot = False
       sdk_environment['DART_LOCAL_BUILD'] = 'dart-editor-archive-testing'
 
-    REVISION = revision
     GSU_PATH_REV = '%s/%s' % (to_bucket, REVISION)
     GSU_PATH_LATEST = '%s/%s' % (to_bucket, 'latest')
     GSU_API_DOCS_PATH = '%s/%s' % (GSU_API_DOCS_BUCKET, REVISION)
 
     homegsutil = join(DART_PATH, 'third_party', 'gsutil', 'gsutil')
     gsu = gsutil.GsUtil(False, homegsutil,
       running_on_buildbot=running_on_buildbot)
     InstallDartium(buildroot, buildout, buildos, gsu)
     if sdk_environment.has_key('JAVA_HOME'):
       print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME']))
@@ skipping 68 matching lines @@
 
         # dart-editor-linux.gtk.x86.zip --> darteditor-linux-32.zip
         RenameRcpZipFiles(buildout)
 
         PostProcessEditorBuilds(buildout)
 
         if running_on_buildbot:
           version_file = _FindVersionFile(buildout)
           if version_file:
             UploadFile(version_file, False)
+            DartArchiveUploadVersionFile(version_file)
 
           found_zips = _FindRcpZipFiles(buildout)
           for zipfile in found_zips:
             UploadFile(zipfile)
+            DartArchiveUploadEditorZipFile(zipfile)
 
       return 0
   finally:
     if ant_property_file is not None:
       print 'cleaning up temp file {0}'.format(ant_property_file.name)
       os.remove(ant_property_file.name)
     print 'cleaning up {0}'.format(buildroot)
     shutil.rmtree(buildroot, True)
     print 'Build Done'
 
@@ skipping 182 matching lines @@
         unzip_dir = os.path.join(tmp_dir,
           os.path.splitext(os.path.basename(dartiumFile))[0])
         if not os.path.exists(unzip_dir):
           os.makedirs(unzip_dir)
         # Always download as searchString.zip
         basename = "%s.zip" % searchString
         tmp_zip_file = os.path.join(tmp_dir, basename)
 
         if not os.path.exists(tmp_zip_file):
           gsu.Copy(dartiumFile, tmp_zip_file, False)
-   
+
           # Upload dartium zip to make sure we have consistent dartium downloads
           UploadFile(tmp_zip_file)
 
           # Dartium is unzipped into ~ unzip_dir/dartium-win-full-7665.7665
           dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
           dartium_zip.UnZip(unzip_dir)
         else:
           dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
 
         dart_zip_path = join(buildout, rcpZipFile)
@@ skipping 72 matching lines @@
       infofile = join('dart', 'DartEditor.app', 'Contents', 'Info.plist')
       subprocess.call(['unzip', zipFile, infofile], env=os.environ)
       ReplaceInFiles(
           [infofile],
           [('<dict>',
             '<dict>\n\t<key>NSHighResolutionCapable</key>\n\t\t<true/>')])
       subprocess.call(['zip', '-q', zipFile, infofile], env=os.environ)
       os.remove(infofile)
 
 
-def CalculateChecksum(filename):
-  """Calculate the MD5 checksum for filename."""
-
-  md5 = hashlib.md5()
-
-  with open(filename, 'rb') as f:
-    data = f.read(65536)
-    while len(data) > 0:
-      md5.update(data)
-      data = f.read(65536)
-
-  return md5.hexdigest()
-
-
-def CreateChecksumFile(filename):
-  """Create and upload an MD5 checksum file for filename."""
-
-  checksum = CalculateChecksum(filename)
-  checksum_filename = '%s.md5sum' % filename
-
-  with open(checksum_filename, 'w') as f:
-    f.write('%s *%s' % (checksum, os.path.basename(filename)))
-
-  return checksum_filename
-
-
 def RunEditorTests(buildout, buildos):
   StartBuildStep('run_tests')
 
   for editorArchive in _GetTestableRcpArchives(buildout):
     with utils.TempDir('editor_') as tempDir:
       print 'Running tests for %s...' % editorArchive
 
       zipper = ziputils.ZipUtil(join(buildout, editorArchive), buildos)
       zipper.UnZip(tempDir)
 
@@ skipping 87 matching lines @@
   try:
     Gsutil(['rm', '-R', join(gsPath, '*')])
   except:
     # Ignore this, in the general case there is nothing.
     pass
   # create eclipse-update/index.html first to ensure eclipse-update prefix
   # exists (needed for recursive copy to follow)
   Gsutil(['cp', '-a', 'public-read',
           r'file://' + join(buildout, 'buildRepo', 'index.html'),
           join(gsPath,'index.html')])
+
   # recursively copy update site contents
   UploadDirectory(glob.glob(join(buildout, 'buildRepo', '*')), gsPath)
-
+  DartArchiveUploadUpdateSite(join(buildout, 'buildRepo'))
 
 def CreateApiDocs(buildLocation):
   """Zip up api_docs, upload it, and upload the raw tree of docs"""
 
   apidir = join(DART_PATH,
                 utils.GetBuildRoot(BUILD_OS, 'release', 'ia32'),
                 'api_docs')
 
   shutil.rmtree(apidir, ignore_errors = True)
 
   CallBuildScript('release', 'ia32', 'api_docs')
 
   UploadApiDocs(apidir)
 
   api_zip = join(buildLocation, 'dart-api-docs.zip')
 
   CreateZip(apidir, api_zip)
 
   # upload to continuous/svn_rev and to continuous/latest
   UploadFile(api_zip, False)
 
+  DartArchiveUploadAPIDocs(api_zip)
+
 
 def CreateSDK(sdkpath):
   """Create the dart-sdk's for the current OS"""
 
   if BUILD_OS == 'linux':
     return CreateLinuxSDK(sdkpath)
   if BUILD_OS == 'macos':
     return CreateMacosSDK(sdkpath)
   if BUILD_OS == 'win32':
     return CreateWin32SDK(sdkpath)
 
-
 def CreateLinuxSDK(sdkpath):
   sdkdir32 = join(DART_PATH, utils.GetBuildRoot('linux', 'release', 'ia32'),
                   'dart-sdk')
   sdkdir64 = join(DART_PATH, utils.GetBuildRoot('linux', 'release', 'x64'),
                   'dart-sdk')
 
   # Build the SDK
   CallBuildScript('release', 'ia32,x64', 'create_sdk')
 
   sdk32_zip = join(sdkpath, 'dartsdk-linux-32.zip')
   sdk32_tgz = join(sdkpath, 'dartsdk-linux-32.tar.gz')
   sdk64_zip = join(sdkpath, 'dartsdk-linux-64.zip')
   sdk64_tgz = join(sdkpath, 'dartsdk-linux-64.tar.gz')
 
   CreateZip(sdkdir32, sdk32_zip)
   CreateTgz(sdkdir32, sdk32_tgz)
   CreateZip(sdkdir64, sdk64_zip)
   CreateTgz(sdkdir64, sdk64_tgz)
 
   UploadFile(sdk32_zip)
   UploadFile(sdk32_tgz)
   UploadFile(sdk64_zip)
   UploadFile(sdk64_tgz)
 
+  DartArchiveUploadSDKs('linux', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
 def CreateMacosSDK(sdkpath):
   # Build the SDK
   CallBuildScript('release', 'ia32,x64', 'create_sdk')
 
   sdk32_zip = join(sdkpath, 'dartsdk-macos-32.zip')
   sdk64_zip = join(sdkpath, 'dartsdk-macos-64.zip')
   sdk32_tgz = join(sdkpath, 'dartsdk-macos-32.tar.gz')
   sdk64_tgz = join(sdkpath, 'dartsdk-macos-64.tar.gz')
 
   CreateZip(join(DART_PATH, utils.GetBuildRoot('macos', 'release', 'ia32'),
                  'dart-sdk'), sdk32_zip)
   CreateZip(join(DART_PATH, utils.GetBuildRoot('macos', 'release', 'x64'),
                  'dart-sdk'), sdk64_zip)
   CreateTgz(join(DART_PATH, utils.GetBuildRoot('macos', 'release', 'ia32'),
                  'dart-sdk'), sdk32_tgz)
   CreateTgz(join(DART_PATH, utils.GetBuildRoot('macos', 'release', 'x64'),
                  'dart-sdk'), sdk64_tgz)
 
   UploadFile(sdk32_zip)
   UploadFile(sdk64_zip)
   UploadFile(sdk32_tgz)
   UploadFile(sdk64_tgz)
 
+  DartArchiveUploadSDKs('macos', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
 def CreateWin32SDK(sdkpath):
   # Build the SDK
   CallBuildScript('release', 'ia32,x64', 'create_sdk')
 
   sdk32_zip = join(sdkpath, 'dartsdk-win32-32.zip')
   sdk64_zip = join(sdkpath, 'dartsdk-win32-64.zip')
 
   CreateZipWindows(join(DART_PATH,
                         utils.GetBuildRoot('win32', 'release', 'ia32'),
                         'dart-sdk'), sdk32_zip)
   CreateZipWindows(join(DART_PATH,
                         utils.GetBuildRoot('win32', 'release', 'x64'),
                         'dart-sdk'), sdk64_zip)
 
   UploadFile(sdk32_zip)
   UploadFile(sdk64_zip)
 
+  DartArchiveUploadSDKs('win32', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
 def CallBuildScript(mode, arch, target):
   """invoke tools/build.py"""
   buildScript = join(TOOLS_PATH, 'build.py')
   cmd = [sys.executable, buildScript, '--mode=%s' % mode, '--arch=%s' % arch,
          target]
   try:
     ExecuteCommand(cmd, DART_PATH)
@@ skipping 27 matching lines @@
   FileDelete(targetFile)
   ExecuteCommand(['tar', 'czf', targetFile, os.path.basename(directory)],
                  os.path.dirname(directory))
 
 
 def UploadFile(targetFile, createChecksum=True):
   """Upload the given file to google storage."""
 
   if (NO_UPLOAD):
     return
-    
+
   filePathRev = "%s/%s" % (GSU_PATH_REV, os.path.basename(targetFile))
   filePathLatest = "%s/%s" % (GSU_PATH_LATEST, os.path.basename(targetFile))
 
-  if (createChecksum):
-    checksum = CreateChecksumFile(targetFile)
+  if createChecksum:
+    checksum = bot_utils.CreateChecksumFile(targetFile)
 
     checksumRev = "%s/%s" % (GSU_PATH_REV, os.path.basename(checksum))
     checksumLatest = "%s/%s" % (GSU_PATH_LATEST, os.path.basename(checksum))
 
   Gsutil(['cp', '-a', 'public-read', r'file://' + targetFile, filePathRev])
 
   if (createChecksum):
     Gsutil(['cp', '-a', 'public-read', r'file://' + checksum, checksumRev])
 
   Gsutil(['cp', '-a', 'public-read', filePathRev, filePathLatest])
@@ skipping 55 matching lines @@
   """delete the given file - do not re-throw any exceptions that occur"""
   if os.path.exists(f):
     try:
       os.remove(f)
     except OSError:
       print 'error deleting %s' % f
 
 
 if __name__ == '__main__':
   sys.exit(main())