Make editor builders archive to gs://dart-archive/ in addition to the other GCS buckets

dart/editor/build/build.py

 #!/usr/bin/env python
 #
 # Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import glob
 import gsutil
 import hashlib
 import imp
@@ skipping 10 matching lines @@
 
 BUILD_OS = None
 DART_PATH = None
 TOOLS_PATH = None
 
 GSU_PATH_REV = None
 GSU_PATH_LATEST = None
 GSU_API_DOCS_PATH = None
 GSU_API_DOCS_BUCKET = 'gs://dartlang-api-docs'
 
+CHANNEL = None
+PLUGINS_BUILD = None
 REVISION = None
+SYSTEM = None
 TRUNK_BUILD = None
-PLUGINS_BUILD = None
 
 NO_UPLOAD = None
 
-def GetUtils():
-  '''Dynamically load the tools/utils.py python module.'''
-  dart_dir = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
-  return imp.load_source('utils', os.path.join(dart_dir, 'tools', 'utils.py'))
+DART_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
+utils = imp.load_source('utils', os.path.join(DART_DIR, 'tools', 'utils.py'))
+bot_utils = imp.load_source('bot_utils',
+    os.path.join(DART_DIR, 'tools', 'bots', 'bot_utils.py'))
 
-utils = GetUtils()
+def DartArchiveFile(local_path, remote_path, create_md5sum=False):
+  # Copy it to the new unified gs://dart-archive bucket
+  # TODO(kustermann/ricow): Remove all the old archiving code, once everything
+  # points to the new location
+  gsutil = bot_utils.GSUtil()
+  gsutil.execute(['cp', '-a', 'public-read', local_path, remote_path])

[ricow1, 2013/09/16 11:51:30]

how about a copyPublic method in GSUtil class?

[kustermann, 2013/09/16 14:15:54]

Added upload/remove with named parameters for public/recursive.

+  if create_md5sum:
+    # 'local_path' may have a different filename than 'remote_path'. So we need
+    # to make sure the *.md5sum file contains the correct name.
+    assert '/' in remote_path and not remote_path.endswith('/')
+    mangled_filename = remote_path[remote_path.rfind('/') + 1:]
+    local_md5sum = bot_utils.CreateChecksumFile(local_path, mangled_filename)
+    gsutil.execute(['cp', '-a', 'public-read', local_md5sum,
+        remote_path + '.md5sum'])
+
+def DartArchiveUploadEditorZipFile(zipfile):
+  if CHANNEL == 'trunk': return

[ricow1, 2013/09/16 11:51:30]

add a todo here saying why we do this

[kustermann, 2013/09/16 14:15:54]

Done.

+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  gsutil = bot_utils.GSUtil()
+
+  basename = os.path.basename(zipfile)
+  system = None
+  arch = None
+  if basename.startswith('darteditor-linux'):

[ricow1, 2013/09/16 11:51:30]

shouldn't we pass this info along instead of extracting it from the filename?
If the reason is that we want to make as little addition to existing code as
possible we should have a todo to remove this

[kustermann, 2013/09/16 14:15:54]

I don't know what you mean by passing it along.

bot_utils expect to get an "system"/"arch" parameter -- and we extract these two
parts here.

I don't want to change the name of the zip file, since we want to keep doing
trunk builds/archives for a while (and I don't want to touch these pieces)

+    system = 'linux'
+  elif basename.startswith('darteditor-mac'):
+    system = 'macos'
+  elif basename.startswith('darteditor-win'):
+    system = 'windows'
+
+  if basename.endswith('-32.zip'):
+    arch = 'ia32'
+  elif basename.endswith('-64.zip'):
+    arch = 'x64'
+
+  if system and arch:
+    for revision in [REVISION, 'latest']:
+      editor_dir = namer.editor_directory(revision)
+      destination_zipfile = namer.editor_zipfilename(system, arch)
+      DartArchiveFile(zipfile, editor_dir + '/' + destination_zipfile,

[ricow1, 2013/09/16 11:51:30]

how about a namer.editor_zipfilepath instead, that just returns the full gs://
path?

[kustermann, 2013/09/16 14:15:54]

Added filepath functions in addition to filename/directory functions. 

+          create_md5sum=True)

[ricow1, 2013/09/16 11:51:30]

else:
  throw an error?

[kustermann, 2013/09/16 14:15:54]

I added an assert (I thought we could call this function with 'tar.gz' as well
-- now I think we don't).

+
+def DartArchiveUploadUpdateSite(local_path):
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  gsutil = bot_utils.GSUtil()
+  for revision in [REVISION, 'latest']:
+    update_site_dir = namer.editor_eclipse_update_directory(revision)
+    try:
+      gsutil.execute(['rm', '-R', update_site_dir])

[ricow1, 2013/09/16 11:51:30]

isn't this asking for a race condition?

[kustermann, 2013/09/16 14:15:54]

Not AFAIK. Please explain.

[ricow1, 2013/09/16 15:11:28]

If people are using the web interface to browse the docs?

+    except:
+      # Ignore this, in the general case there is nothing.
+      pass
+    gsutil.execute(['cp', '-a', 'public-read', '-R',
+        local_path, update_site_dir])
+
+def DartArchiveUploadSDKs(system, sdk32_zip, sdk64_zip):
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  for revision in [REVISION, 'latest']:
+    sdk_dir = namer.sdk_directory(revision)
+    path32 = sdk_dir + '/' + namer.sdk_zipfilename(system, 'ia32', 'release')

[ricow1, 2013/09/16 11:51:30]

the latest one will already be there right, will this not throw an error?

[kustermann, 2013/09/16 14:15:54]

Not AFAIK. Please explain.

[ricow1, 2013/09/16 15:11:28]

latest will already be there

+    path64 = sdk_dir + '/' + namer.sdk_zipfilename(system, 'x64', 'release')
+    DartArchiveFile(sdk32_zip, path32, create_md5sum=True)
+    DartArchiveFile(sdk64_zip, path64, create_md5sum=True)
+
 
 class AntWrapper(object):
   """A wrapper for ant build invocations"""
 
   _antpath = None
   _bzippath = None
   _propertyfile = None
 
   def __init__(self, propertyfile, antpath='/usr/bin', bzippath=None):
     """Initialize the ant path.
@@ skipping 119 matching lines @@
                     help='Output Directory.',
                     action='store')
   result.add_option('--dest',
                     help='Output Directory.',
                     action='store')
   return result
 
 def main():
   """Main entry point for the build program."""
   global BUILD_OS
+  global CHANNEL
   global DART_PATH
-  global TOOLS_PATH
-  global GSU_PATH_REV
   global GSU_API_DOCS_PATH
   global GSU_PATH_LATEST
+  global GSU_PATH_REV
+  global NO_UPLOAD
+  global PLUGINS_BUILD
   global REVISION
+  global SYSTEM
+  global TOOLS_PATH
   global TRUNK_BUILD
-  global PLUGINS_BUILD
-  global NO_UPLOAD
 
   if not sys.argv:
     print 'Script pathname not known, giving up.'
     return 1
 
   scriptdir = os.path.abspath(os.path.dirname(sys.argv[0]))
   editorpath = os.path.abspath(os.path.join(scriptdir, '..'))
   thirdpartypath = os.path.abspath(os.path.join(scriptdir, '..', '..',
                                                 'third_party'))
   toolspath = os.path.abspath(os.path.join(scriptdir, '..', '..',
@@ skipping 90 matching lines @@
     # Get user name. If it does not start with chrome then deploy to the test
     # bucket; otherwise deploy to the continuous bucket.
     username = os.environ.get('USER')
     if username is None:
       username = os.environ.get('USERNAME')
 
     if username is None:
       print 'Could not find username'
       return 6
 
-    # dart-editor[-trunk], dart-editor-(win/mac/linux)[-trunk]
+    # dart-editor[-trunk], dart-editor-(win/mac/linux)[-trunk/be/dev/stable]
     builder_name = str(options.name)
 
-    TRUNK_BUILD = builder_name.endswith("-trunk")
-    PLUGINS_BUILD = (builder_name == 'dart-editor' or
-                     builder_name == 'dart-editor-trunk')
+    EDITOR_REGEXP = (r'^dart-editor(-(?P<system>(win|mac|linux)))?' +
+                      '(-(?P<channel>(trunk|be|dev|stable)))?$')
+    match = re.match(EDITOR_REGEXP, builder_name)
+    if not match:
+      raise Exception("Buildername '%s' does not match pattern '%s'."
+                      % (builder_name, EDITOR_REGEXP))
+
+    CHANNEL = match.groupdict()['channel'] or 'be'
+    SYSTEM = match.groupdict()['system']
+
+    TRUNK_BUILD = CHANNEL == 'trunk'
+    PLUGINS_BUILD = SYSTEM is None
+    REVISION = revision
 
     build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
     sdk_environment = os.environ
     if username.startswith('chrome'):
       if TRUNK_BUILD:
         to_bucket = 'gs://dart-editor-archive-trunk'
       else:
         to_bucket = 'gs://dart-editor-archive-continuous'
       running_on_buildbot = True
     else:
       to_bucket = 'gs://dart-editor-archive-testing'
       running_on_buildbot = False
       sdk_environment['DART_LOCAL_BUILD'] = 'dart-editor-archive-testing'
 
-    REVISION = revision
     GSU_PATH_REV = '%s/%s' % (to_bucket, REVISION)
     GSU_PATH_LATEST = '%s/%s' % (to_bucket, 'latest')
     GSU_API_DOCS_PATH = '%s/%s' % (GSU_API_DOCS_BUCKET, REVISION)
 
     homegsutil = join(DART_PATH, 'third_party', 'gsutil', 'gsutil')
     gsu = gsutil.GsUtil(False, homegsutil,
       running_on_buildbot=running_on_buildbot)
     InstallDartium(buildroot, buildout, buildos, gsu)
     if sdk_environment.has_key('JAVA_HOME'):
       print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME']))
@@ skipping 68 matching lines @@
 
         # dart-editor-linux.gtk.x86.zip --> darteditor-linux-32.zip
         RenameRcpZipFiles(buildout)
 
         PostProcessEditorBuilds(buildout)
 
         if running_on_buildbot:
           version_file = _FindVersionFile(buildout)
           if version_file:
             UploadFile(version_file, False)
+            if CHANNEL != 'trunk':
+              namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)

[ricow1, 2013/09/16 11:51:30]

Extract this to a method as well to be consistent

[kustermann, 2013/09/16 14:15:54]

Extracted this and another one. Refactored bot_utils a bit.

+              for revision in [REVISION, 'latest']:
+                DartArchiveFile(version_file, namer.version_file(revision),
+                    create_md5sum=False)
 
           found_zips = _FindRcpZipFiles(buildout)
           for zipfile in found_zips:
             UploadFile(zipfile)
+            DartArchiveUploadEditorZipFile(zipfile)
 
       return 0
   finally:
     if ant_property_file is not None:
       print 'cleaning up temp file {0}'.format(ant_property_file.name)
       os.remove(ant_property_file.name)
     print 'cleaning up {0}'.format(buildroot)
     shutil.rmtree(buildroot, True)
     print 'Build Done'
 
@@ skipping 182 matching lines @@
         unzip_dir = os.path.join(tmp_dir,
           os.path.splitext(os.path.basename(dartiumFile))[0])
         if not os.path.exists(unzip_dir):
           os.makedirs(unzip_dir)
         # Always download as searchString.zip
         basename = "%s.zip" % searchString
         tmp_zip_file = os.path.join(tmp_dir, basename)
 
         if not os.path.exists(tmp_zip_file):
           gsu.Copy(dartiumFile, tmp_zip_file, False)
-   
+
           # Upload dartium zip to make sure we have consistent dartium downloads
           UploadFile(tmp_zip_file)
 
           # Dartium is unzipped into ~ unzip_dir/dartium-win-full-7665.7665
           dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
           dartium_zip.UnZip(unzip_dir)
         else:
           dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
 
         dart_zip_path = join(buildout, rcpZipFile)
@@ skipping 72 matching lines @@
       infofile = join('dart', 'DartEditor.app', 'Contents', 'Info.plist')
       subprocess.call(['unzip', zipFile, infofile], env=os.environ)
       ReplaceInFiles(
           [infofile],
           [('<dict>',
             '<dict>\n\t<key>NSHighResolutionCapable</key>\n\t\t<true/>')])
       subprocess.call(['zip', '-q', zipFile, infofile], env=os.environ)
       os.remove(infofile)
 
 
-def CalculateChecksum(filename):
-  """Calculate the MD5 checksum for filename."""
-
-  md5 = hashlib.md5()
-
-  with open(filename, 'rb') as f:
-    data = f.read(65536)
-    while len(data) > 0:
-      md5.update(data)
-      data = f.read(65536)
-
-  return md5.hexdigest()
-
-
-def CreateChecksumFile(filename):
-  """Create and upload an MD5 checksum file for filename."""
-
-  checksum = CalculateChecksum(filename)
-  checksum_filename = '%s.md5sum' % filename
-
-  with open(checksum_filename, 'w') as f:
-    f.write('%s *%s' % (checksum, os.path.basename(filename)))
-
-  return checksum_filename
-
-
 def RunEditorTests(buildout, buildos):
   StartBuildStep('run_tests')
 
   for editorArchive in _GetTestableRcpArchives(buildout):
     with utils.TempDir('editor_') as tempDir:
       print 'Running tests for %s...' % editorArchive
 
       zipper = ziputils.ZipUtil(join(buildout, editorArchive), buildos)
       zipper.UnZip(tempDir)
 
@@ skipping 87 matching lines @@
   try:
     Gsutil(['rm', '-R', join(gsPath, '*')])
   except:
     # Ignore this, in the general case there is nothing.
     pass
   # create eclipse-update/index.html first to ensure eclipse-update prefix
   # exists (needed for recursive copy to follow)
   Gsutil(['cp', '-a', 'public-read',
           r'file://' + join(buildout, 'buildRepo', 'index.html'),
           join(gsPath,'index.html')])
+
   # recursively copy update site contents
   UploadDirectory(glob.glob(join(buildout, 'buildRepo', '*')), gsPath)
-
+  DartArchiveUploadUpdateSite(join(buildout, 'buildRepo'))
 
 def CreateApiDocs(buildLocation):
   """Zip up api_docs, upload it, and upload the raw tree of docs"""
 
   apidir = join(DART_PATH,
                 utils.GetBuildRoot(BUILD_OS, 'release', 'ia32'),
                 'api_docs')
 
   shutil.rmtree(apidir, ignore_errors = True)
 
   CallBuildScript('release', 'ia32', 'api_docs')
 
   UploadApiDocs(apidir)
 
   api_zip = join(buildLocation, 'dart-api-docs.zip')
 
   CreateZip(apidir, api_zip)
 
   # upload to continuous/svn_rev and to continuous/latest
   UploadFile(api_zip, False)
 
+  if CHANNEL != 'trunk':
+    namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)

[ricow1, 2013/09/16 11:51:30]

also put this into a method

[kustermann, 2013/09/16 14:15:54]

Done.

+    for revision in [REVISION, 'latest']:
+      destination = (namer.apidocs_directory(revision) + '/' +
+          namer.apidocs_zipfile())
+      DartArchiveFile(api_zip, destination, create_md5sum=False)
+
 
 def CreateSDK(sdkpath):
   """Create the dart-sdk's for the current OS"""
 
   if BUILD_OS == 'linux':
     return CreateLinuxSDK(sdkpath)
   if BUILD_OS == 'macos':
     return CreateMacosSDK(sdkpath)
   if BUILD_OS == 'win32':
     return CreateWin32SDK(sdkpath)
 
-
 def CreateLinuxSDK(sdkpath):
   sdkdir32 = join(DART_PATH, utils.GetBuildRoot('linux', 'release', 'ia32'),
                   'dart-sdk')
   sdkdir64 = join(DART_PATH, utils.GetBuildRoot('linux', 'release', 'x64'),
                   'dart-sdk')
 
   # Build the SDK
   CallBuildScript('release', 'ia32,x64', 'create_sdk')
 
   sdk32_zip = join(sdkpath, 'dartsdk-linux-32.zip')
   sdk32_tgz = join(sdkpath, 'dartsdk-linux-32.tar.gz')
   sdk64_zip = join(sdkpath, 'dartsdk-linux-64.zip')
   sdk64_tgz = join(sdkpath, 'dartsdk-linux-64.tar.gz')
 
   CreateZip(sdkdir32, sdk32_zip)
   CreateTgz(sdkdir32, sdk32_tgz)
   CreateZip(sdkdir64, sdk64_zip)
   CreateTgz(sdkdir64, sdk64_tgz)
 
   UploadFile(sdk32_zip)
   UploadFile(sdk32_tgz)
   UploadFile(sdk64_zip)
   UploadFile(sdk64_tgz)
 
+  DartArchiveUploadSDKs('linux', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
 def CreateMacosSDK(sdkpath):
   # Build the SDK
   CallBuildScript('release', 'ia32,x64', 'create_sdk')
 
   sdk32_zip = join(sdkpath, 'dartsdk-macos-32.zip')
   sdk64_zip = join(sdkpath, 'dartsdk-macos-64.zip')
   sdk32_tgz = join(sdkpath, 'dartsdk-macos-32.tar.gz')
   sdk64_tgz = join(sdkpath, 'dartsdk-macos-64.tar.gz')
 
   CreateZip(join(DART_PATH, utils.GetBuildRoot('macos', 'release', 'ia32'),
                  'dart-sdk'), sdk32_zip)
   CreateZip(join(DART_PATH, utils.GetBuildRoot('macos', 'release', 'x64'),
                  'dart-sdk'), sdk64_zip)
   CreateTgz(join(DART_PATH, utils.GetBuildRoot('macos', 'release', 'ia32'),
                  'dart-sdk'), sdk32_tgz)
   CreateTgz(join(DART_PATH, utils.GetBuildRoot('macos', 'release', 'x64'),
                  'dart-sdk'), sdk64_tgz)
 
   UploadFile(sdk32_zip)
   UploadFile(sdk64_zip)
   UploadFile(sdk32_tgz)
   UploadFile(sdk64_tgz)
 
+  DartArchiveUploadSDKs('macos', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
 def CreateWin32SDK(sdkpath):
   # Build the SDK
   CallBuildScript('release', 'ia32,x64', 'create_sdk')
 
   sdk32_zip = join(sdkpath, 'dartsdk-win32-32.zip')
   sdk64_zip = join(sdkpath, 'dartsdk-win32-64.zip')
 
   CreateZipWindows(join(DART_PATH,
                         utils.GetBuildRoot('win32', 'release', 'ia32'),
                         'dart-sdk'), sdk32_zip)
   CreateZipWindows(join(DART_PATH,
                         utils.GetBuildRoot('win32', 'release', 'x64'),
                         'dart-sdk'), sdk64_zip)
 
   UploadFile(sdk32_zip)
   UploadFile(sdk64_zip)
 
+  DartArchiveUploadSDKs('win32', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
 def CallBuildScript(mode, arch, target):
   """invoke tools/build.py"""
   buildScript = join(TOOLS_PATH, 'build.py')
   cmd = [sys.executable, buildScript, '--mode=%s' % mode, '--arch=%s' % arch,
          target]
   try:
     ExecuteCommand(cmd, DART_PATH)
@@ skipping 27 matching lines @@
   FileDelete(targetFile)
   ExecuteCommand(['tar', 'czf', targetFile, os.path.basename(directory)],
                  os.path.dirname(directory))
 
 
 def UploadFile(targetFile, createChecksum=True):
   """Upload the given file to google storage."""
 
   if (NO_UPLOAD):
     return
-    
+
   filePathRev = "%s/%s" % (GSU_PATH_REV, os.path.basename(targetFile))
   filePathLatest = "%s/%s" % (GSU_PATH_LATEST, os.path.basename(targetFile))
 
-  if (createChecksum):
-    checksum = CreateChecksumFile(targetFile)
+  if createChecksum:
+    checksum = bot_utils.CreateChecksumFile(targetFile)
 
     checksumRev = "%s/%s" % (GSU_PATH_REV, os.path.basename(checksum))
     checksumLatest = "%s/%s" % (GSU_PATH_LATEST, os.path.basename(checksum))
 
   Gsutil(['cp', '-a', 'public-read', r'file://' + targetFile, filePathRev])
 
   if (createChecksum):
     Gsutil(['cp', '-a', 'public-read', r'file://' + checksum, checksumRev])
 
   Gsutil(['cp', '-a', 'public-read', filePathRev, filePathLatest])
@@ skipping 55 matching lines @@
   """delete the given file - do not re-throw any exceptions that occur"""
   if os.path.exists(f):
     try:
       os.remove(f)
     except OSError:
       print 'error deleting %s' % f
 
 
 if __name__ == '__main__':
   sys.exit(main())