Changes to annotated step scripts: dart-editor-installer-* builders will build installer

editor/build/build.py

 #!/usr/bin/env python
 #
-# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+# Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import glob
 import gsutil
 import imp
 import optparse
 import os
 import re
 import shutil
@@ skipping 107 matching lines @@
 
 def DartArchiveUploadVersionFile(version_file):
   # TODO(kustermann): We don't archive trunk builds to gs://dart-archive/.
   # Remove this once the channel transition is done.
   if CHANNEL == 'trunk': return
   namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
   for revision in [REVISION, 'latest']:
     DartArchiveFile(version_file, namer.version_filepath(revision),
         create_md5sum=False)
 
+def DartArchiveUploadInstaller(
+      arch, installer_file, extension, release_type=bot_utils.ReleaseType.RAW):
+  namer = bot_utils.GCSNamer(CHANNEL, release_type)
+  for revision in [REVISION, 'latest']:
+    gsu_path = namer.editor_installer_zipfilepath(
+        revision, SYSTEM, arch, extension)
+    DartArchiveFile(installer_file, gsu_path, create_md5sum=False)
+
 class AntWrapper(object):
   """A wrapper for ant build invocations"""
 
   _antpath = None
   _bzippath = None
   _propertyfile = None
 
   def __init__(self, propertyfile, antpath='/usr/bin', bzippath=None):
     """Initialize the ant path.
 
@@ skipping 113 matching lines @@
                     action='store')
   result.add_option('-n', '--name',
                     help='builder name.',
                     action='store')
   result.add_option('-o', '--out',
                     help='Output Directory.',
                     action='store')
   result.add_option('--dest',
                     help='Output Directory.',
                     action='store')
+  result.add_option('--build-installer',
+                    help='Fetch editor, build installer and archive it.',
+                    action='store_true', default=False)
   return result
 
 def main():
   """Main entry point for the build program."""
   global BUILD_OS
   global CHANNEL
   global DART_PATH
   global GSU_API_DOCS_PATH
   global GSU_PATH_LATEST
   global GSU_PATH_REV
@@ skipping 30 matching lines @@
 
   # TODO(devoncarew): remove this hardcoded e:\ path
   buildroot_parent = {'linux': dartpath, 'macos': dartpath, 'win32': r'e:\tmp'}
   buildroot = os.path.join(buildroot_parent[buildos], 'build_root')
 
   os.chdir(buildpath)
   ant_property_file = None
   sdk_zip = None
 
   try:
-    ant_property_file = tempfile.NamedTemporaryFile(suffix='.property',
-                                                    prefix='AntProperties',
-                                                    delete=False)
-    ant_property_file.close()
-    ant = AntWrapper(ant_property_file.name, os.path.join(antpath, 'bin'),
-                     bzip2libpath)
-
-    ant.RunAnt(os.getcwd(), '', '', '', '',
-               '', '', buildos, ['-diagnostics'])
-
     parser = BuildOptions()
     (options, args) = parser.parse_args()
     # Determine which targets to build. By default we build the "all" target.
     if args:
       print 'only options should be passed to this script'
       parser.print_help()
       return 2
 
     if str(options.revision) == 'None':
       print 'missing revision option'
@@ skipping 28 matching lines @@
     lastc = revision[-1]
     if lastc.isalpha():
       revision = revision[0:-1]
     index = revision.find(':')
     if index > -1:
       revision = revision[0:index]
     print 'revision       = |{0}|'.format(revision)
     buildout = os.path.abspath(options.out)
     print 'buildout       = {0}'.format(buildout)
 
-    if not os.path.exists(buildout):
-      os.makedirs(buildout)
-
-    # clean out old build artifacts
-    for f in os.listdir(buildout):
-      if ('dartsdk-' in f) or ('darteditor-' in f) or ('dart-editor-' in f):
-        os.remove(join(buildout, f))
-
     # Get user name. If it does not start with chrome then deploy to the test
     # bucket; otherwise deploy to the continuous bucket.
     username = os.environ.get('USER')
     if username is None:
       username = os.environ.get('USERNAME')
 
     if username is None:
       print 'Could not find username'
       return 6
 
+    build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
+    sdk_environment = os.environ
+    if sdk_environment.has_key('JAVA_HOME'):
+      print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME']))
+
     # dart-editor[-trunk], dart-editor-(win/mac/linux)[-trunk/be/dev/stable]
     builder_name = str(options.name)
 
-    EDITOR_REGEXP = (r'^dart-editor(-(?P<system>(win|mac|linux)))?' +
+    EDITOR_REGEXP = (r'^dart-editor(-(?P<installer>(installer)))?(-(?P<system>(win|mac|linux)))?' +

[ricow1, 2013/10/18 11:18:08]

long line

[kustermann, 2013/10/18 14:12:13]

Done.

                       '(-(?P<channel>(trunk|be|dev|stable)))?$')
     match = re.match(EDITOR_REGEXP, builder_name)
     if not match:
       raise Exception("Buildername '%s' does not match pattern '%s'."
                       % (builder_name, EDITOR_REGEXP))
 
     CHANNEL = match.groupdict()['channel'] or 'be'
     SYSTEM = match.groupdict()['system']
+    BUILD_INSTALLER = bool(match.groupdict()['installer'])
 
     TRUNK_BUILD = CHANNEL == 'trunk'
     PLUGINS_BUILD = SYSTEM is None
     REVISION = revision
 
-    build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
-    sdk_environment = os.environ
+    # Make sure the buildername and the options agree
+    assert BUILD_INSTALLER == options.build_installer
+
     if username.startswith('chrome'):
       if TRUNK_BUILD:
-        to_bucket = 'gs://dart-editor-archive-trunk'
+        bucket = 'gs://dart-editor-archive-trunk'
       else:
-        to_bucket = 'gs://dart-editor-archive-continuous'
+        bucket = 'gs://dart-editor-archive-continuous'
       running_on_buildbot = True
     else:
-      to_bucket = 'gs://dart-editor-archive-testing'
+      bucket = 'gs://dart-editor-archive-testing'
       running_on_buildbot = False
       sdk_environment['DART_LOCAL_BUILD'] = 'dart-editor-archive-testing'
 
-    GSU_PATH_REV = '%s/%s' % (to_bucket, REVISION)
-    GSU_PATH_LATEST = '%s/%s' % (to_bucket, 'latest')
+    GSU_PATH_REV = '%s/%s' % (bucket, REVISION)
+    GSU_PATH_LATEST = '%s/%s' % (bucket, 'latest')
     GSU_API_DOCS_PATH = '%s/%s' % (GSU_API_DOCS_BUCKET, REVISION)
 
     homegsutil = join(DART_PATH, 'third_party', 'gsutil', 'gsutil')
     gsu = gsutil.GsUtil(False, homegsutil,
       running_on_buildbot=running_on_buildbot)
-    InstallDartium(buildroot, buildout, buildos, gsu)
-    if sdk_environment.has_key('JAVA_HOME'):
-      print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME']))
 
-    if not PLUGINS_BUILD:
-      StartBuildStep('create_sdk')
-      EnsureDirectoryExists(buildout)
-      try:
-        sdk_zip = CreateSDK(buildout)
-      except:
-        BuildStepFailure()
+    def build_installer():
+      def old_location_pair(arch, extension):
+        """Returns a tuple (zip_file, installer_file) of google cloud storage
+           locations."""
+        os_rename = {'win': 'win32', 'mac': 'macos', 'linux': 'linux'}

[ricow1, 2013/10/18 11:18:08]

we have a bunch of dictionaries for renames, we could extract that - don't do
that  in this cl though

[kustermann, 2013/10/18 14:12:13]

Well, we can remove all this special casing once we no longer use the old bucket
locations.

+        system = os_rename[SYSTEM]
+        return (
+            ("%s/darteditor-%s-%s.zip" % (GSU_PATH_REV, system, arch)),
+            ("%s/darteditor-installer-%s-%s.%s"
+             % (GSU_PATH_REV, system, arch, extension)))
+      if SYSTEM == 'mac':
+        for arch in ['32', '64']:

[ricow1, 2013/10/18 11:18:08]

I would extract this into create_mac_installer()

[kustermann, 2013/10/18 14:12:13]

Done.

+          with utils.TempDir('build_editor_installer') as temp_dir:
+            with utils.ChangedWorkingDirectory(temp_dir):
+              (gsu_editor_zip, gsu_editor_dmg) = old_location_pair(arch, 'dmg')
+              # Fetch the editor zip file from the old location.
+              if gsu.Copy(gsu_editor_zip, 'dart.zip', False):

[ricow1, 2013/10/18 11:18:08]

dart.zip -> editor.zip (+ you could put it into a variable, you use it twice

[kustermann, 2013/10/18 14:12:13]

Partly done (I think the additional variable would make the code harder to
read).

+                raise Exception("gsutil command failed, aborting.")
+
+              # Unzip the editor (which contains a directory named 'dart').
+              bot_utils.run(['unzip', 'dart.zip'])
+              assert os.path.exists('dart') and os.path.isdir('dart')
+
+              # Build the dmg installer
+              dmg_installer = os.path.join(temp_dir,'darteditor-installer.dmg')
+              dart_folder_icon = os.path.join(DART_PATH,
+                  'editor/tools/plugins/com.google.dart.tools.ui/' +
+                  'icons/dart_about_140_160.png')
+              dmg_builder = os.path.join(DART_PATH, 'tools',
+                  'mac_build_editor_dmg.sh')
+              bot_utils.run([dmg_builder, dmg_installer, 'dart',
+                  dart_folder_icon, "Dart Distribution"])
+              assert os.path.exists(dmg_installer)

[ricow1, 2013/10/18 11:18:08]

No need for this, os.path.isfile will return false if the file does not exist as
well

[kustermann, 2013/10/18 14:12:13]

Done.

+              assert os.path.isfile(dmg_installer)
+
+              # Archive to old bucket
+              # TODO(kustermann/ricow): Remove all the old archiving code,
+              # once everything points to the new location.
+              if gsu.Copy(dmg_installer, gsu_editor_dmg):
+                raise Exception("gsutil command failed, aborting.")
+
+              # Archive to new bucket
+              # NOTE: This is a little bit hackisch, we fetch the editor from
+              # the old bucket and archive the dmg to the new bucket here.
+              DartArchiveUploadInstaller(arch, dmg_installer, 'dmg',
+                  release_type=bot_utils.ReleaseType.SIGNED)
+      else:
+        raise Exception(
+            "We currently cannot build installers for %s" % builder_name)

[ricow1, 2013/10/18 11:18:08]

builder_name -> SYSTEM?

[kustermann, 2013/10/18 14:12:13]

Done.

+    def build_editor():
+      ant_property_file = tempfile.NamedTemporaryFile(suffix='.property',
+                                                      prefix='AntProperties',
+                                                      delete=False)
+      ant_property_file.close()
+      ant = AntWrapper(ant_property_file.name, os.path.join(antpath, 'bin'),
+                       bzip2libpath)
+
+      ant.RunAnt(os.getcwd(), '', '', '', '',
+                 '', '', buildos, ['-diagnostics'])
+
+      if not os.path.exists(buildout):
+        os.makedirs(buildout)
+
+      # clean out old build artifacts
+      for f in os.listdir(buildout):
+        if ('dartsdk-' in f) or ('darteditor-' in f) or ('dart-editor-' in f):
+          os.remove(join(buildout, f))
+
+      InstallDartium(buildroot, buildout, buildos, gsu)
+
+      if not PLUGINS_BUILD:
+        StartBuildStep('create_sdk')
+        EnsureDirectoryExists(buildout)
+        try:
+          sdk_zip = CreateSDK(buildout)
+        except:
+          BuildStepFailure()
 
 
-    if builder_name.startswith('dart-editor-linux'):
-      StartBuildStep('api_docs')
-      try:
-        CreateApiDocs(buildout)
-      except:
-        BuildStepFailure()
+      if builder_name.startswith('dart-editor-linux'):
+        StartBuildStep('api_docs')
+        try:
+          CreateApiDocs(buildout)
+        except:
+          BuildStepFailure()
 
-    StartBuildStep(builder_name)
+      StartBuildStep(builder_name)
 
-    if PLUGINS_BUILD:
-      status = BuildUpdateSite(ant, revision, builder_name, buildroot, buildout,
-                               editorpath, buildos)
-      return status
-
-    with utils.TempDir('ExtraArtifacts') as extra_artifacts:
-      #tell the ant script where to write the sdk zip file so it can
-      #be expanded later
-      status = ant.RunAnt('.', 'build_rcp.xml', revision, builder_name,
-                          buildroot, buildout, editorpath, buildos,
-                          sdk_zip=sdk_zip,
-                          running_on_bot=running_on_buildbot,
-                          extra_artifacts=extra_artifacts)
-      #the ant script writes a property file in a known location so
-      #we can read it.
-      properties = ReadPropertyFile(buildos, ant_property_file.name)
-
-      if not properties:
-        raise Exception('no data was found in file {%s}'
-                        % ant_property_file.name)
-      if status:
-        if properties['build.runtime']:
-          PrintErrorLog(properties['build.runtime'])
+      if PLUGINS_BUILD:
+        status = BuildUpdateSite(ant, revision, builder_name, buildroot,
+                                 buildout, editorpath, buildos)
         return status
 
-      #For the dart-editor build, return at this point.
-      #We don't need to install the sdk+dartium, run tests, or copy to google
-      #storage.
-      if not buildos:
-        print 'skipping sdk and dartium steps for dart-editor build'
+      with utils.TempDir('ExtraArtifacts') as extra_artifacts:
+        #tell the ant script where to write the sdk zip file so it can

[ricow1, 2013/10/18 11:18:08]

I know this is not you, but fix the comment #tell -> # Tell
space before be below
+ below

+        #be expanded later
+        status = ant.RunAnt('.', 'build_rcp.xml', revision, builder_name,
+                            buildroot, buildout, editorpath, buildos,
+                            sdk_zip=sdk_zip,
+                            running_on_bot=running_on_buildbot,
+                            extra_artifacts=extra_artifacts)
+        #the ant script writes a property file in a known location so
+        #we can read it.
+        properties = ReadPropertyFile(buildos, ant_property_file.name)
+
+        if not properties:
+          raise Exception('no data was found in file {%s}'
+                          % ant_property_file.name)
+        if status:
+          if properties['build.runtime']:
+            PrintErrorLog(properties['build.runtime'])
+          return status
+
+        #For the dart-editor build, return at this point.
+        #We don't need to install the sdk+dartium, run tests, or copy to google
+        #storage.
+        if not buildos:
+          print 'skipping sdk and dartium steps for dart-editor build'
+          return 0
+
+        #This is an override for local testing
+        force_run_install = os.environ.get('FORCE_RUN_INSTALL')
+
+        if force_run_install or (not PLUGINS_BUILD):
+          InstallSdk(buildroot, buildout, buildos, buildout)
+          InstallDartium(buildroot, buildout, buildos, gsu)
+
+        if status:
+          return status
+
+        if not build_skip_tests:
+          RunEditorTests(buildout, buildos)
+
+        if buildos:
+          StartBuildStep('upload_artifacts')
+
+          _InstallArtifacts(buildout, buildos, extra_artifacts)
+
+          # dart-editor-linux.gtk.x86.zip --> darteditor-linux-32.zip
+          RenameRcpZipFiles(buildout)
+
+          PostProcessEditorBuilds(buildout)
+
+          if running_on_buildbot:
+            version_file = _FindVersionFile(buildout)
+            if version_file:
+              UploadFile(version_file, False)
+              DartArchiveUploadVersionFile(version_file)
+
+            found_zips = _FindRcpZipFiles(buildout)
+            for zipfile in found_zips:
+              UploadFile(zipfile)
+              DartArchiveUploadEditorZipFile(zipfile)
+
         return 0
 
-      #This is an override for local testing
-      force_run_install = os.environ.get('FORCE_RUN_INSTALL')
-
-      if force_run_install or (not PLUGINS_BUILD):
-        InstallSdk(buildroot, buildout, buildos, buildout)
-        InstallDartium(buildroot, buildout, buildos, gsu)
-
-      if status:
-        return status
-
-      if not build_skip_tests:
-        RunEditorTests(buildout, buildos)
-
-      if buildos:
-        StartBuildStep('upload_artifacts')
-
-        _InstallArtifacts(buildout, buildos, extra_artifacts)
-
-        # dart-editor-linux.gtk.x86.zip --> darteditor-linux-32.zip
-        RenameRcpZipFiles(buildout)
-
-        PostProcessEditorBuilds(buildout)
-
-        if running_on_buildbot:
-          version_file = _FindVersionFile(buildout)
-          if version_file:
-            UploadFile(version_file, False)
-            DartArchiveUploadVersionFile(version_file)
-
-          found_zips = _FindRcpZipFiles(buildout)
-          for zipfile in found_zips:
-            UploadFile(zipfile)
-            DartArchiveUploadEditorZipFile(zipfile)
-
-      return 0
+    if BUILD_INSTALLER:
+      build_installer()
+    else:
+      build_editor()
   finally:
     if ant_property_file is not None:
       print 'cleaning up temp file {0}'.format(ant_property_file.name)
       os.remove(ant_property_file.name)
     print 'cleaning up {0}'.format(buildroot)
     shutil.rmtree(buildroot, True)
     print 'Build Done'
 
 
 def ReadPropertyFile(buildos, property_file):
@@ skipping 165 matching lines @@
 
     # Download and unzip dartium
     unzip_dir = os.path.join(tmp_dir,
       os.path.splitext(os.path.basename(dartiumFile))[0])
     if not os.path.exists(unzip_dir):
       os.makedirs(unzip_dir)
 
     # Always download as local_name.zip
     tmp_zip_file = os.path.join(tmp_dir, "%s.zip" % local_name)
     if not os.path.exists(tmp_zip_file):
-      gsu.Copy(dartiumFile, tmp_zip_file, False)
+      if gsu.Copy(dartiumFile, tmp_zip_file, False):
+        raise Exception("gsutil command failed, aborting.")
       # Dartium is unzipped into unzip_dir/dartium-*/
       dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
       dartium_zip.UnZip(unzip_dir)
 
     dart_zip_path = join(buildout, rcpZipFile)
     dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
 
     add_path = glob.glob(join(unzip_dir, 'dartium-*'))[0]
     if system == 'windows':
       # TODO(ricow/kustermann): This is hackisch. We should make a generic
@@ skipping 57 matching lines @@
         #download and unzip dartium
         unzip_dir = os.path.join(tmp_dir,
           os.path.splitext(os.path.basename(dartiumFile))[0])
         if not os.path.exists(unzip_dir):
           os.makedirs(unzip_dir)
         # Always download as searchString.zip
         basename = "%s.zip" % searchString
         tmp_zip_file = os.path.join(tmp_dir, basename)
 
         if not os.path.exists(tmp_zip_file):
-          gsu.Copy(dartiumFile, tmp_zip_file, False)
+          if gsu.Copy(dartiumFile, tmp_zip_file, False):
+            raise Exception("gsutil command failed, aborting.")
 
           # Upload dartium zip to make sure we have consistent dartium downloads
           UploadFile(tmp_zip_file)
 
           # Dartium is unzipped into ~ unzip_dir/dartium-win-full-7665.7665
           dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
           dartium_zip.UnZip(unzip_dir)
         else:
           dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
 
@@ skipping 473 matching lines @@
   """delete the given file - do not re-throw any exceptions that occur"""
   if os.path.exists(f):
     try:
       os.remove(f)
     except OSError:
       print 'error deleting %s' % f
 
 
 if __name__ == '__main__':
   sys.exit(main())