Index: dart/editor/build/promote.py |
=================================================================== |
--- dart/editor/build/promote.py (revision 30146) |
+++ dart/editor/build/promote.py (working copy) |
@@ -1,12 +1,11 @@ |
#!/usr/bin/env python |
-# |
+# |
# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
# for details. All rights reserved. Use of this source code is governed by a |
# BSD-style license that can be found in the LICENSE file. |
-# Dart Editor promote and google storage cleanup tools. |
+# Dart Editor promote tools. |
-import gsutil |
import imp |
import optparse |
import os |
@@ -16,189 +15,71 @@ |
from os.path import join |
-CONTINUOUS = 'gs://dart-editor-archive-continuous' |
-TRUNK = 'gs://dart-editor-archive-trunk' |
-TESTING = 'gs://dart-editor-archive-testing' |
-INTEGRATION = 'gs://dart-editor-archive-integration' |
-RELEASE = 'gs://dart-editor-archive-release' |
-INTERNAL = 'gs://dart-editor-archive-internal' |
- |
DART_PATH = os.path.abspath(os.path.join(__file__, '..', '..', '..')) |
BOT_UTILS = os.path.abspath(os.path.join( |
DART_PATH, 'tools', 'bots', 'bot_utils.py')) |
bot_utils = imp.load_source('bot_utils', BOT_UTILS) |
-def _BuildOptions(): |
- """Setup the argument processing for this program. |
+def BuildOptions(): |
+ usage = """usage: %prog promote [options] |
+ where: |
+ promote - Will promote builds from raw/signed locations to release |
+ locations. |
- Returns: |
- the OptionParser to process the CLI |
+ Example: Promote revision r29962 on dev channel: |
+ python editor/build/promote.py promote --channel=dev --revision=29962 |
""" |
- usage = """usage: %prog [options] cleanup|promote |
- where: |
- cleanup will cleanup the Google storage continuous bucket |
- promote will promote code between different stages |
- Examples: |
- cleanup saving the last 150 revisions |
- python gsTool.py cleanup --keepcount=150 |
- |
- promote revision 567 from trunk to integration |
- python gsTool.py promote --trunk --revision=567""" |
- |
- |
result = optparse.OptionParser(usage=usage) |
- result.set_default('gsbucketuri', 'gs://dart-editor-archive-continuous') |
- result.set_default('keepcount', 1000) |
- result.set_default('dryrun', False) |
- result.set_default('continuous', False) |
- result.set_default('trunk', False) |
- result.set_default('integration', False) |
- result.set_default('testing', False) |
- group = optparse.OptionGroup(result, 'Cleanup', |
- 'options used to cleanup Google Storage') |
- group.add_option('--keepcount', |
- type='int', |
- help='Numer of Builds to keep.', |
- action='store') |
- result.add_option_group(group) |
- group = optparse.OptionGroup(result, 'Promote', |
- 'options used to promote code') |
- group.add_option('--revision', |
- help='The svn revision to promote', |
- action='store') |
- group.add_option('--continuous', |
- help='Promote from continuous', |
- action='store_true') |
- group.add_option('--trunk', |
- help='Promote from trunk', |
- action='store_true') |
- group.add_option('--internal', |
- help='Promote from trunk to internal', |
- action='store_true') |
- group.add_option('--integration', |
- help='Promote from integration', |
- action='store_true') |
- group.add_option('--channel', type='string', |
- help='Promote from this channel', |
- default=None) |
+ group = optparse.OptionGroup( |
+ result, 'Promote', 'options used to promote code') |
+ group.add_option( |
+ '--revision', help='The svn revision to promote', action='store') |
+ group.add_option( |
+ '--channel', type='string', help='Channel to promote.', default=None) |
result.add_option_group(group) |
- result.add_option('--gsbucketuri', |
- help='Dart Continuous Google Storage bucket URI.', |
- action='store') |
- result.add_option('--gsutilloc', |
- help='location of gsutil the program', |
- action='store') |
- result.add_option('--dryrun', help='don\'t do anything that would change' |
- ' Google Storage', |
- action='store_true') |
- result.add_option('--testing', help='user test bucket in ' |
- ' Google Storage', |
- action='store_true') |
- |
return result |
def main(): |
- """Main entry point for Google Storage Tools.""" |
- |
- parser = _BuildOptions() |
+ parser = BuildOptions() |
(options, args) = parser.parse_args() |
- if not args: |
- print 'At least one command must be specified' |
+ def die(msg): |
+ print msg |
parser.print_help() |
sys.exit(1) |
+ if not args: |
+ die('At least one command must be specified') |
+ |
if args[0] == 'promote': |
command = 'promote' |
if options.revision is None: |
- print 'You must specify a --revision to specify which revision to promote' |
- parser.print_help() |
- sys.exit(3) |
+ die('You must specify a --revision to specify which revision to promote') |
# Make sure revision is a valid integer |
try: |
_ = int(options.revision) |
except: |
- print 'You must supply a valid integer argument to --revision to promote' |
- parser.print_help() |
- sys.exit(3) |
+ die('You must supply a valid integer argument to --revision to promote') |
- # Make sure options.channel is a valid channel if given |
- if options.channel: |
- if options.channel not in bot_utils.Channel.ALL_CHANNELS: |
- print 'You must supply a valid channel to --channel to promote' |
- parser.print_help() |
- sys.exit(3) |
- |
- if not (options.continuous or options.integration or |
- options.testing or options.trunk or options.internal or |
- options.channel): |
- print ('Specify --continuous, --integration, --testing, --trunk or ' |
- '--channel=be/dev/stable') |
- parser.print_help() |
- sys.exit(4) |
- if options.continuous and options.integration: |
- print 'continuous and integration can not be specified at the same time' |
- parser.print_help() |
- sys.exit(5) |
- if (options.continuous or options.integration) and options.testing: |
- print """Warning --continuous or --integration and --testing are |
- specified. The --testing flag will take precedence and all data will |
- go to the testing bucket {0}""".format(TESTING) |
- elif args[0] == 'cleanup': |
- command = 'cleanup' |
- if options.keepcount is None: |
- print 'You must specify --keepcount' |
- parser.print_help() |
- sys.exit(6) |
+ # Make sure options.channel is a valid |
+ if not options.channel: |
+ die('Specify --channel=be/dev/stable') |
+ if options.channel not in bot_utils.Channel.ALL_CHANNELS: |
+ die('You must supply a valid channel to --channel to promote') |
else: |
- print 'Invalid command specified: {0}. See help below'.format(args[0]) |
- parser.print_help() |
- sys.exit(2) |
+ die('Invalid command specified: {0}. See help below'.format(args[0])) |
- gsu = gsutil.GsUtil(options.dryrun, options.gsutilloc) |
- if options.testing: |
- bucket_from = CONTINUOUS |
- bucket_to = TESTING |
- print """The --testing attribute is specified. All data will go to the |
- testing bucket {0}. Press enter to continue""".format(TESTING) |
- raw_input('Press Enter to continue') |
- elif options.continuous: |
- bucket_from = CONTINUOUS |
- bucket_to = INTEGRATION |
- elif options.trunk: |
- bucket_from = TRUNK |
- bucket_to = INTEGRATION |
- elif options.integration: |
- bucket_from = INTEGRATION |
- bucket_to = RELEASE |
- elif options.internal: |
- bucket_from = TRUNK |
- bucket_to = INTERNAL |
+ if command == 'promote': |
+ _PromoteDartArchiveBuild(options.channel, options.revision) |
- if command == 'cleanup': |
- #if the testing flag is set remove the date from the testing bucket |
- if options.testing: |
- bucket = TESTING |
- #otherwise use the value passed as --gsbucketuri |
- else: |
- bucket = options.gsbucketuri |
- version_dirs = _ReadBucket(gsu, '{0}/{1}'.format(bucket, '[0-9]*')) |
- _RemoveElements(gsu, bucket, version_dirs, options.keepcount) |
- elif command == 'promote': |
- if options.channel: |
- _PromoteDartArchiveBuild(options.channel, options.revision) |
- else: |
- _PromoteBuild(options.revision, bucket_from, bucket_to) |
- _UpdateDocs() |
- |
-def _UpdateDocs(): |
+def UpdateDocs(): |
try: |
print 'Updating docs' |
url = "http://api.dartlang.org/docs/releases/latest/?force_reload=true" |
@@ -209,94 +90,7 @@ |
print 'Could not update api docs, please manually update them' |
print 'Failed with: %s' % e |
-def _ReadBucket(gsu, bucket): |
- """Read the contents of a Google Storage Bucket. |
- Args: |
- gsu: the location of the gsutil program |
- bucket: the bucket to read the contents of |
- |
- Returns: |
- a list of bucket entries excluding all entries starting with "latest" |
- """ |
- _PrintSeparator('_ReadBucket({0}, {1})'.format(gsu, bucket)) |
- elements = [] |
- items = gsu.ReadBucket(bucket) |
- for item in items: |
- dirpaths = item.split('/') |
- if len(dirpaths) >= 3: |
- dirname = dirpaths[3] |
- if dirname != 'latest': |
- try: |
- dirnum = int(dirname) |
- if not dirnum in elements: |
- elements.append(dirnum) |
- except ValueError: |
- pass |
- |
- return elements |
- |
- |
-def _RemoveElements(gsu, bucket, version_dirs, keepcount): |
- """Remove the selected elements from Google Storage. |
- |
- Args: |
- gsu: the gsutil program to run |
- bucket: the bucket to remove the dirs from |
- version_dirs: the dictionary of elements to remove keyed by |
- svn version number |
- keepcount: the number of elements to keep |
- """ |
- _PrintSeparator('_RemoveElements({0}, version_dirs,' |
- ' {1}'.format(gsu, keepcount)) |
- version_dirs_size = len(version_dirs) |
- delete_count = version_dirs_size - keepcount |
- if delete_count > 0: |
- count = 0 |
- version_dirs.sort() |
- for gs_dir in version_dirs: |
- if count < delete_count: |
- gsu.RemoveAll('{0}/{1}/*'.format(bucket, gs_dir)) |
- else: |
- print 'version {0}/{1} will be saved'.format(bucket, gs_dir) |
- count += 1 |
- else: |
- print ('nothing to delete because that are only {0} elemens in the list' |
- ' and the keep count is set to {0}').format(len(version_dirs), |
- keepcount) |
- |
- |
-def _PromoteBuild(revision, from_bucket, to_bucket): |
- """Promote a build from one bucket to another. |
- |
- Args: |
- revision: the revision to promote |
- from_bucket: the bucket to promote from |
- to_bucket: the bucket to promote to |
- """ |
- |
- # print the gsutil version |
- _Gsutil(['version']) |
- |
- src = '%s/%s/' % (from_bucket, revision) |
- srcVersion = src + 'VERSION' |
- |
- # copy from continuous/1234 to trunk/1234 |
- dest = '%s/%s/' % (to_bucket, revision) |
- destUpdate = dest + 'eclipse-update/' |
- print 'copying: %s -> %s' % (src, dest) |
- _Gsutil(['cp', '-a', 'public-read', srcVersion, destUpdate + 'features/']) |
- _Gsutil(['cp', '-a', 'public-read', srcVersion, destUpdate + 'plugins/']) |
- _Gsutil(['cp', '-r', '-a', 'public-read', src + '*', dest]) |
- |
- # copy from continuous/1234 to trunk/latest |
- dest = '%s/%s/' % (to_bucket, 'latest') |
- destUpdate = dest + 'eclipse-update/' |
- print 'copying: %s -> %s' % (src, dest) |
- _Gsutil(['cp', '-a', 'public-read', srcVersion, destUpdate + 'features/']) |
- _Gsutil(['cp', '-a', 'public-read', srcVersion, destUpdate + 'plugins/']) |
- _Gsutil(['cp', '-r', '-a', 'public-read', src + '*', dest]) |
- |
def _PromoteDartArchiveBuild(channel, revision): |
# These namer objects will be used to create GCS object URIs. For the |
# structure we use, please see tools/bots/bot_utils.py:GCSNamer |
@@ -315,35 +109,35 @@ |
def remove_gs_directory(gs_path): |
safety_check_on_gs_path(gs_path, to_revision, channel) |
- _Gsutil(['-m', 'rm', '-R', '-f', gs_path]) |
+ Gsutil(['-m', 'rm', '-R', '-f', gs_path]) |
# Copy VERSION file. |
from_loc = raw_namer.version_filepath(revision) |
to_loc = release_namer.version_filepath(to_revision) |
- _Gsutil(['cp', '-a', 'public-read', from_loc, to_loc]) |
+ Gsutil(['cp', '-a', 'public-read', from_loc, to_loc]) |
# Copy sdk directory. |
from_loc = raw_namer.sdk_directory(revision) |
to_loc = release_namer.sdk_directory(to_revision) |
remove_gs_directory(to_loc) |
- _Gsutil(['-m', 'cp', '-a', 'public-read', '-R', from_loc, to_loc]) |
+ Gsutil(['-m', 'cp', '-a', 'public-read', '-R', from_loc, to_loc]) |
# Copy eclipse update directory. |
from_loc = raw_namer.editor_eclipse_update_directory(revision) |
to_loc = release_namer.editor_eclipse_update_directory(to_revision) |
remove_gs_directory(to_loc) |
- _Gsutil(['-m', 'cp', '-a', 'public-read', '-R', from_loc, to_loc]) |
+ Gsutil(['-m', 'cp', '-a', 'public-read', '-R', from_loc, to_loc]) |
# Copy api-docs zipfile. |
from_loc = raw_namer.apidocs_zipfilepath(revision) |
to_loc = release_namer.apidocs_zipfilepath(to_revision) |
- _Gsutil(['-m', 'cp', '-a', 'public-read', from_loc, to_loc]) |
+ Gsutil(['-m', 'cp', '-a', 'public-read', from_loc, to_loc]) |
# Copy dartium directory. |
from_loc = raw_namer.dartium_directory(revision) |
to_loc = release_namer.dartium_directory(to_revision) |
remove_gs_directory(to_loc) |
- _Gsutil(['-m', 'cp', '-a', 'public-read', '-R', from_loc, to_loc]) |
+ Gsutil(['-m', 'cp', '-a', 'public-read', '-R', from_loc, to_loc]) |
# Copy editor zip files. |
target_editor_dir = release_namer.editor_directory(to_revision) |
@@ -356,38 +150,26 @@ |
from_namer = signed_namer |
from_loc = from_namer.editor_zipfilepath(revision, system, arch) |
to_loc = release_namer.editor_zipfilepath(to_revision, system, arch) |
- _Gsutil(['cp', '-a', 'public-read', from_loc, to_loc]) |
- _Gsutil(['cp', '-a', 'public-read', from_loc + '.md5sum', |
+ Gsutil(['cp', '-a', 'public-read', from_loc, to_loc]) |
+ Gsutil(['cp', '-a', 'public-read', from_loc + '.md5sum', |
to_loc + '.md5sum']) |
+ # Copy signed editor installers for macos/windows. |
+ for system, extension in [('windows', 'msi'), ('macos', 'dmg')]: |
+ for arch in ['ia32', 'x64']: |
+ from_loc = signed_namer.editor_installer_filepath( |
+ revision, system, arch, extension) |
+ to_loc = release_namer.editor_installer_filepath( |
+ to_revision, system, arch, extension) |
+ Gsutil(['cp', '-a', 'public-read', from_loc, to_loc]) |
+ |
promote(revision) |
promote('latest') |
-def _PrintSeparator(text): |
- print '================================' |
- print '== %s' % text |
- |
- |
-def _PrintFailure(text): |
- print '*****************************' |
- print '** %s' % text |
- print '*****************************' |
- |
- |
-def _Gsutil(cmd): |
+def Gsutil(cmd): |
gsutilTool = join(DART_PATH, 'third_party', 'gsutil', 'gsutil') |
- return _ExecuteCommand([sys.executable, gsutilTool] + cmd) |
+ bot_utils.run([sys.executable, gsutilTool] + cmd) |
-def _ExecuteCommand(cmd, directory=None): |
- """Execute the given command.""" |
- if directory is not None: |
- cwd = os.getcwd() |
- os.chdir(directory) |
- subprocess.call(cmd, env=os.environ) |
- if directory is not None: |
- os.chdir(cwd) |
- |
- |
if __name__ == '__main__': |
sys.exit(main()) |