Make editor builders archive to gs://dart-archive/ in addition to the other GCS buckets

dart/editor/build/build.py

Index: dart/editor/build/build.py
diff --git a/dart/editor/build/build.py b/dart/editor/build/build.py
index 5b1b481666e885bd6f0560d68bfdfed63a5d1611..8f6665043a4be2abc065aa005f9c69234282f94d 100755
--- a/dart/editor/build/build.py
+++ b/dart/editor/build/build.py
@@ -28,18 +28,85 @@ GSU_PATH_LATEST = None
 GSU_API_DOCS_PATH = None
 GSU_API_DOCS_BUCKET = 'gs://dartlang-api-docs'
 
+CHANNEL = None
+PLUGINS_BUILD = None
 REVISION = None
+SYSTEM = None
 TRUNK_BUILD = None
-PLUGINS_BUILD = None
 
 NO_UPLOAD = None
 
-def GetUtils():
-  '''Dynamically load the tools/utils.py python module.'''
-  dart_dir = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
-  return imp.load_source('utils', os.path.join(dart_dir, 'tools', 'utils.py'))
+DART_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
+utils = imp.load_source('utils', os.path.join(DART_DIR, 'tools', 'utils.py'))
+bot_utils = imp.load_source('bot_utils',
+    os.path.join(DART_DIR, 'tools', 'bots', 'bot_utils.py'))
+
+def DartArchiveFile(local_path, remote_path, create_md5sum=False):
+  # Copy it to the new unified gs://dart-archive bucket
+  # TODO(kustermann/ricow): Remove all the old archiving code, once everything
+  # points to the new location
+  gsutil = bot_utils.GSUtil()
+  gsutil.execute(['cp', '-a', 'public-read', local_path, remote_path])

[ricow1, 2013/09/16 11:51:30]

how about a copyPublic method in GSUtil class?

[kustermann, 2013/09/16 14:15:54]

Added upload/remove with named parameters for public/recursive.

+  if create_md5sum:
+    # 'local_path' may have a different filename than 'remote_path'. So we need
+    # to make sure the *.md5sum file contains the correct name.
+    assert '/' in remote_path and not remote_path.endswith('/')
+    mangled_filename = remote_path[remote_path.rfind('/') + 1:]
+    local_md5sum = bot_utils.CreateChecksumFile(local_path, mangled_filename)
+    gsutil.execute(['cp', '-a', 'public-read', local_md5sum,
+        remote_path + '.md5sum'])
+
+def DartArchiveUploadEditorZipFile(zipfile):
+  if CHANNEL == 'trunk': return

[ricow1, 2013/09/16 11:51:30]

add a todo here saying why we do this

[kustermann, 2013/09/16 14:15:54]

Done.

+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  gsutil = bot_utils.GSUtil()
+
+  basename = os.path.basename(zipfile)
+  system = None
+  arch = None
+  if basename.startswith('darteditor-linux'):

[ricow1, 2013/09/16 11:51:30]

shouldn't we pass this info along instead of extracting it from the filename?
If the reason is that we want to make as little addition to existing code as
possible we should have a todo to remove this

[kustermann, 2013/09/16 14:15:54]

I don't know what you mean by passing it along.

bot_utils expect to get an "system"/"arch" parameter -- and we extract these two
parts here.

I don't want to change the name of the zip file, since we want to keep doing
trunk builds/archives for a while (and I don't want to touch these pieces)

+    system = 'linux'
+  elif basename.startswith('darteditor-mac'):
+    system = 'macos'
+  elif basename.startswith('darteditor-win'):
+    system = 'windows'
+
+  if basename.endswith('-32.zip'):
+    arch = 'ia32'
+  elif basename.endswith('-64.zip'):
+    arch = 'x64'
+
+  if system and arch:
+    for revision in [REVISION, 'latest']:
+      editor_dir = namer.editor_directory(revision)
+      destination_zipfile = namer.editor_zipfilename(system, arch)
+      DartArchiveFile(zipfile, editor_dir + '/' + destination_zipfile,

[ricow1, 2013/09/16 11:51:30]

how about a namer.editor_zipfilepath instead, that just returns the full gs://
path?

[kustermann, 2013/09/16 14:15:54]

Added filepath functions in addition to filename/directory functions. 

+          create_md5sum=True)

[ricow1, 2013/09/16 11:51:30]

else:
  throw an error?

[kustermann, 2013/09/16 14:15:54]

I added an assert (I thought we could call this function with 'tar.gz' as well
-- now I think we don't).

+
+def DartArchiveUploadUpdateSite(local_path):
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  gsutil = bot_utils.GSUtil()
+  for revision in [REVISION, 'latest']:
+    update_site_dir = namer.editor_eclipse_update_directory(revision)
+    try:
+      gsutil.execute(['rm', '-R', update_site_dir])

[ricow1, 2013/09/16 11:51:30]

isn't this asking for a race condition?

[kustermann, 2013/09/16 14:15:54]

Not AFAIK. Please explain.

[ricow1, 2013/09/16 15:11:28]

If people are using the web interface to browse the docs?

+    except:
+      # Ignore this, in the general case there is nothing.
+      pass
+    gsutil.execute(['cp', '-a', 'public-read', '-R',
+        local_path, update_site_dir])
+
+def DartArchiveUploadSDKs(system, sdk32_zip, sdk64_zip):
+  if CHANNEL == 'trunk': return
+  namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
+  for revision in [REVISION, 'latest']:
+    sdk_dir = namer.sdk_directory(revision)
+    path32 = sdk_dir + '/' + namer.sdk_zipfilename(system, 'ia32', 'release')

[ricow1, 2013/09/16 11:51:30]

the latest one will already be there right, will this not throw an error?

[kustermann, 2013/09/16 14:15:54]

Not AFAIK. Please explain.

[ricow1, 2013/09/16 15:11:28]

latest will already be there

+    path64 = sdk_dir + '/' + namer.sdk_zipfilename(system, 'x64', 'release')
+    DartArchiveFile(sdk32_zip, path32, create_md5sum=True)
+    DartArchiveFile(sdk64_zip, path64, create_md5sum=True)
 
-utils = GetUtils()
 
 class AntWrapper(object):
   """A wrapper for ant build invocations"""
@@ -179,15 +246,17 @@ def BuildOptions():
 def main():
   """Main entry point for the build program."""
   global BUILD_OS
+  global CHANNEL
   global DART_PATH
-  global TOOLS_PATH
-  global GSU_PATH_REV
   global GSU_API_DOCS_PATH
   global GSU_PATH_LATEST
+  global GSU_PATH_REV
+  global NO_UPLOAD
+  global PLUGINS_BUILD
   global REVISION
+  global SYSTEM
+  global TOOLS_PATH
   global TRUNK_BUILD
-  global PLUGINS_BUILD
-  global NO_UPLOAD
 
   if not sys.argv:
     print 'Script pathname not known, giving up.'
@@ -298,12 +367,22 @@ def main():
       print 'Could not find username'
       return 6
 
-    # dart-editor[-trunk], dart-editor-(win/mac/linux)[-trunk]
+    # dart-editor[-trunk], dart-editor-(win/mac/linux)[-trunk/be/dev/stable]
     builder_name = str(options.name)
 
-    TRUNK_BUILD = builder_name.endswith("-trunk")
-    PLUGINS_BUILD = (builder_name == 'dart-editor' or
-                     builder_name == 'dart-editor-trunk')
+    EDITOR_REGEXP = (r'^dart-editor(-(?P<system>(win|mac|linux)))?' +
+                      '(-(?P<channel>(trunk|be|dev|stable)))?$')
+    match = re.match(EDITOR_REGEXP, builder_name)
+    if not match:
+      raise Exception("Buildername '%s' does not match pattern '%s'."
+                      % (builder_name, EDITOR_REGEXP))
+
+    CHANNEL = match.groupdict()['channel'] or 'be'
+    SYSTEM = match.groupdict()['system']
+
+    TRUNK_BUILD = CHANNEL == 'trunk'
+    PLUGINS_BUILD = SYSTEM is None
+    REVISION = revision
 
     build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
     sdk_environment = os.environ
@@ -318,7 +397,6 @@ def main():
       running_on_buildbot = False
       sdk_environment['DART_LOCAL_BUILD'] = 'dart-editor-archive-testing'
 
-    REVISION = revision
     GSU_PATH_REV = '%s/%s' % (to_bucket, REVISION)
     GSU_PATH_LATEST = '%s/%s' % (to_bucket, 'latest')
     GSU_API_DOCS_PATH = '%s/%s' % (GSU_API_DOCS_BUCKET, REVISION)
@@ -407,10 +485,16 @@ def main():
           version_file = _FindVersionFile(buildout)
           if version_file:
             UploadFile(version_file, False)
+            if CHANNEL != 'trunk':
+              namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)

[ricow1, 2013/09/16 11:51:30]

Extract this to a method as well to be consistent

[kustermann, 2013/09/16 14:15:54]

Extracted this and another one. Refactored bot_utils a bit.

+              for revision in [REVISION, 'latest']:
+                DartArchiveFile(version_file, namer.version_file(revision),
+                    create_md5sum=False)
 
           found_zips = _FindRcpZipFiles(buildout)
           for zipfile in found_zips:
             UploadFile(zipfile)
+            DartArchiveUploadEditorZipFile(zipfile)
 
       return 0
   finally:
@@ -613,7 +697,7 @@ def InstallDartium(buildroot, buildout, buildos, gsu):
 
         if not os.path.exists(tmp_zip_file):
           gsu.Copy(dartiumFile, tmp_zip_file, False)
-   
+
           # Upload dartium zip to make sure we have consistent dartium downloads
           UploadFile(tmp_zip_file)
 
@@ -706,32 +790,6 @@ def PostProcessEditorBuilds(out_dir):
       os.remove(infofile)
 
 
-def CalculateChecksum(filename):
-  """Calculate the MD5 checksum for filename."""
-
-  md5 = hashlib.md5()
-
-  with open(filename, 'rb') as f:
-    data = f.read(65536)
-    while len(data) > 0:
-      md5.update(data)
-      data = f.read(65536)
-
-  return md5.hexdigest()
-
-
-def CreateChecksumFile(filename):
-  """Create and upload an MD5 checksum file for filename."""
-
-  checksum = CalculateChecksum(filename)
-  checksum_filename = '%s.md5sum' % filename
-
-  with open(checksum_filename, 'w') as f:
-    f.write('%s *%s' % (checksum, os.path.basename(filename)))
-
-  return checksum_filename
-
-
 def RunEditorTests(buildout, buildos):
   StartBuildStep('run_tests')
 
@@ -839,9 +897,10 @@ def UploadSite(buildout, gsPath) :
   Gsutil(['cp', '-a', 'public-read',
           r'file://' + join(buildout, 'buildRepo', 'index.html'),
           join(gsPath,'index.html')])
+
   # recursively copy update site contents
   UploadDirectory(glob.glob(join(buildout, 'buildRepo', '*')), gsPath)
-
+  DartArchiveUploadUpdateSite(join(buildout, 'buildRepo'))
 
 def CreateApiDocs(buildLocation):
   """Zip up api_docs, upload it, and upload the raw tree of docs"""
@@ -863,6 +922,13 @@ def CreateApiDocs(buildLocation):
   # upload to continuous/svn_rev and to continuous/latest
   UploadFile(api_zip, False)
 
+  if CHANNEL != 'trunk':
+    namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)

[ricow1, 2013/09/16 11:51:30]

also put this into a method

[kustermann, 2013/09/16 14:15:54]

Done.

+    for revision in [REVISION, 'latest']:
+      destination = (namer.apidocs_directory(revision) + '/' +
+          namer.apidocs_zipfile())
+      DartArchiveFile(api_zip, destination, create_md5sum=False)
+
 
 def CreateSDK(sdkpath):
   """Create the dart-sdk's for the current OS"""
@@ -874,7 +940,6 @@ def CreateSDK(sdkpath):
   if BUILD_OS == 'win32':
     return CreateWin32SDK(sdkpath)
 
-
 def CreateLinuxSDK(sdkpath):
   sdkdir32 = join(DART_PATH, utils.GetBuildRoot('linux', 'release', 'ia32'),
                   'dart-sdk')
@@ -899,6 +964,8 @@ def CreateLinuxSDK(sdkpath):
   UploadFile(sdk64_zip)
   UploadFile(sdk64_tgz)
 
+  DartArchiveUploadSDKs('linux', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
@@ -925,6 +992,8 @@ def CreateMacosSDK(sdkpath):
   UploadFile(sdk32_tgz)
   UploadFile(sdk64_tgz)
 
+  DartArchiveUploadSDKs('macos', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
@@ -945,6 +1014,8 @@ def CreateWin32SDK(sdkpath):
   UploadFile(sdk32_zip)
   UploadFile(sdk64_zip)
 
+  DartArchiveUploadSDKs('win32', sdk32_zip, sdk64_zip)
+
   return sdk32_zip
 
 
@@ -992,12 +1063,12 @@ def UploadFile(targetFile, createChecksum=True):
 
   if (NO_UPLOAD):
     return
-    
+
   filePathRev = "%s/%s" % (GSU_PATH_REV, os.path.basename(targetFile))
   filePathLatest = "%s/%s" % (GSU_PATH_LATEST, os.path.basename(targetFile))
 
-  if (createChecksum):
-    checksum = CreateChecksumFile(targetFile)
+  if createChecksum:
+    checksum = bot_utils.CreateChecksumFile(targetFile)
 
     checksumRev = "%s/%s" % (GSU_PATH_REV, os.path.basename(checksum))
     checksumLatest = "%s/%s" % (GSU_PATH_LATEST, os.path.basename(checksum))