| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/python | |
| 2 | |
| 3 # Copyright (c) 2014 The Chromium Authors. All rights reserved. | |
| 4 # Use of this source code is governed by a BSD-style license that can be | |
| 5 # found in the LICENSE file. | |
| 6 | |
| 7 """Steps to archive dartium, content_shell, and chromedriver from buildbots. | |
| 8 | |
| 9 Imported by buildbot_annotated_steps.py | |
| 10 """ | |
| 11 | |
| 12 import imp | |
| 13 import os | |
| 14 import platform | |
| 15 import re | |
| 16 import subprocess | |
| 17 import sys | |
| 18 | |
| 19 import dartium_bot_utils | |
| 20 import archive | |
| 21 | |
| 22 BUILDER_NAME = 'BUILDBOT_BUILDERNAME' | |
| 23 REVISION = 'BUILDBOT_REVISION' | |
| 24 BUILDER_PATTERN = (r'^(dartium)-(mac|lucid64|lucid32|win)' | |
| 25 r'-(full|inc|debug)(-ninja)?(-(be|dev|stable|integration))?$') | |
| 26 NEW_BUILDER_PATTERN = ( | |
| 27 r'^dartium-(mac|linux|win)-(ia32|x64)(-inc)?-(be|dev|stable|integration)$') | |
| 28 | |
| 29 if platform.system() == 'Windows': | |
| 30 GSUTIL = 'e:/b/build/scripts/slave/gsutil.bat' | |
| 31 else: | |
| 32 GSUTIL = '/b/build/scripts/slave/gsutil' | |
| 33 ACL = 'public-read' | |
| 34 GS_SITE = 'gs://' | |
| 35 GS_URL = 'https://sandbox.google.com/storage/' | |
| 36 GS_DIR = 'dartium-archive' | |
| 37 LATEST = 'latest' | |
| 38 CONTINUOUS = 'continuous' | |
| 39 | |
| 40 SRC_PATH = dartium_bot_utils.srcPath() | |
| 41 DART_PATH = os.path.join(SRC_PATH, 'dart') | |
| 42 | |
| 43 bot_utils = imp.load_source('bot_utils', | |
| 44 os.path.join(DART_PATH, 'tools', 'bots', 'bot_utils.py')) | |
| 45 | |
| 46 class BuildInfo(object): | |
| 47 """ | |
| 48 name: A name for the build - the buildbot host if a buildbot. | |
| 49 mode: 'Debug' or 'Release' | |
| 50 arch: target architecture | |
| 51 channel: the channel this build is happening on | |
| 52 is_full: True if this is a full build. | |
| 53 is_incremental: True if this is an incremental build. | |
| 54 | |
| 55 """ | |
| 56 def __init__(self, revision, version): | |
| 57 | |
| 58 self.revision = revision | |
| 59 self.version = version | |
| 60 self.name = os.environ[BUILDER_NAME] | |
| 61 pattern = re.match(NEW_BUILDER_PATTERN, self.name) | |
| 62 if pattern: | |
| 63 self.arch = pattern.group(2) | |
| 64 self.mode = 'Release' | |
| 65 self.is_incremental = (pattern.group(3) == '-inc') | |
| 66 self.is_full = not self.is_incremental | |
| 67 self.channel = pattern.group(4) | |
| 68 else: | |
| 69 pattern = re.match(BUILDER_PATTERN, self.name) | |
| 70 assert pattern | |
| 71 self.arch = 'x64' if pattern.group(2) == 'lucid64' else 'ia32' | |
| 72 self.mode = 'Debug' if pattern.group(3) == 'debug' else 'Release' | |
| 73 self.is_incremental = '-inc' in self.name | |
| 74 self.is_full = pattern.group(3) == 'full' | |
| 75 self.channel = pattern.group(6) if pattern.group(6) else 'be' | |
| 76 | |
| 77 | |
| 78 def ArchiveAndUpload(info, archive_latest=False): | |
| 79 print '@@@BUILD_STEP dartium_generate_archive@@@' | |
| 80 cwd = os.getcwd() | |
| 81 | |
| 82 dartium_bucket = info.name | |
| 83 drt_bucket = dartium_bucket.replace('dartium', 'drt') | |
| 84 chromedriver_bucket = dartium_bucket.replace('dartium', 'chromedriver') | |
| 85 dartium_archive = dartium_bucket + '-' + info.version | |
| 86 drt_archive = drt_bucket + '-' + info.version | |
| 87 chromedriver_archive = chromedriver_bucket + '-' + info.version | |
| 88 dartium_zip, drt_zip, chromedriver_zip = archive.Archive( | |
| 89 SRC_PATH, | |
| 90 info.mode, | |
| 91 dartium_archive, | |
| 92 drt_archive, | |
| 93 chromedriver_archive) | |
| 94 | |
| 95 status = 0 | |
| 96 # Upload bleeding-edge builds to old dartium-archive bucket | |
| 97 if info.channel == 'be': | |
| 98 status = (OldUpload('dartium', dartium_bucket, | |
| 99 os.path.abspath(dartium_zip), | |
| 100 archive_latest=archive_latest) | |
| 101 or OldUpload('drt', drt_bucket, | |
| 102 os.path.abspath(drt_zip), | |
| 103 archive_latest=archive_latest) | |
| 104 or OldUpload('chromedriver', chromedriver_bucket, | |
| 105 os.path.abspath(chromedriver_zip), | |
| 106 archive_latest=archive_latest)) | |
| 107 | |
| 108 # Upload to new dart-archive bucket using GCSNamer, but not incremental | |
| 109 # or perf builder builds. | |
| 110 if not info.is_incremental: | |
| 111 Upload('dartium', os.path.abspath(dartium_zip), | |
| 112 info, archive_latest=archive_latest) | |
| 113 Upload('drt', os.path.abspath(drt_zip), | |
| 114 info, archive_latest=archive_latest) | |
| 115 Upload('chromedriver', os.path.abspath(chromedriver_zip), | |
| 116 info, archive_latest=archive_latest) | |
| 117 | |
| 118 os.chdir(cwd) | |
| 119 if status != 0: | |
| 120 print '@@@STEP_FAILURE@@@' | |
| 121 return status | |
| 122 | |
| 123 | |
| 124 def OldUpload(module, bucket, zip_file, archive_latest=False): | |
| 125 """Upload a zip file to the old bucket gs://dartium-archive/ | |
| 126 """ | |
| 127 # TODO(whesse): Remove the old archiving code (OldUpload, OldUploadFile, | |
| 128 # and constants they use) once everything points to the new location. | |
| 129 status = 0 | |
| 130 _, filename = os.path.split(zip_file) | |
| 131 if not archive_latest: | |
| 132 target = '/'.join([GS_DIR, bucket, filename]) | |
| 133 print '@@@BUILD_STEP %s_upload_archive_old@@@' % module | |
| 134 status = OldUploadFile(zip_file, GS_SITE + target) | |
| 135 print '@@@STEP_LINK@download@' + GS_URL + target + '@@@' | |
| 136 else: | |
| 137 print '@@@BUILD_STEP %s_upload_latest_old@@@' % module | |
| 138 # Clear latest for this build type. | |
| 139 old = '/'.join([GS_DIR, LATEST, bucket + '-*']) | |
| 140 old_archives = ListArchives(GS_SITE + old) | |
| 141 | |
| 142 # Upload the new latest and remove unnecessary old ones. | |
| 143 target = GS_SITE + '/'.join([GS_DIR, LATEST, filename]) | |
| 144 status = OldUploadFile(zip_file, target) | |
| 145 if status == 0: | |
| 146 RemoveArchives( | |
| 147 [iarch for iarch in old_archives if iarch != target]) | |
| 148 else: | |
| 149 print 'Upload failed' | |
| 150 | |
| 151 # Upload unversioned name to continuous site for incremental | |
| 152 # builds. | |
| 153 if '-inc' in bucket: | |
| 154 continuous_name = bucket[:bucket.find('-inc')] | |
| 155 target = GS_SITE + '/'.join([GS_DIR, CONTINUOUS, | |
| 156 continuous_name + '.zip']) | |
| 157 status = OldUploadFile(zip_file, target) | |
| 158 | |
| 159 print ('@@@BUILD_STEP %s_upload_archive is over (status = %s)@@@' % | |
| 160 (module, status)) | |
| 161 return status | |
| 162 | |
| 163 | |
| 164 def Upload(module, zip_file, info, archive_latest=False): | |
| 165 """Upload a zip file to cloud storage bucket gs://dart-archive/ | |
| 166 """ | |
| 167 print '@@@BUILD_STEP %s_upload_archive @@@' % module | |
| 168 revision = 'latest' if archive_latest else info.revision | |
| 169 name = module.replace('drt', 'content_shell') | |
| 170 namer = bot_utils.GCSNamer(info.channel, bot_utils.ReleaseType.RAW) | |
| 171 remote_path = namer.dartium_variant_zipfilepath(revision, | |
| 172 name, | |
| 173 sys.platform, | |
| 174 info.arch, | |
| 175 info.mode.lower()) | |
| 176 UploadFile(zip_file, remote_path, checksum_files=True) | |
| 177 | |
| 178 print '@@@STEP_LINK@download@' + remote_path + '@@@' | |
| 179 | |
| 180 | |
| 181 def OldUploadFile(source, target): | |
| 182 """Upload an archive zip file to Google storage. | |
| 183 """ | |
| 184 | |
| 185 # Upload file. | |
| 186 cmd = [GSUTIL, 'cp', source, target] | |
| 187 (status, output) = ExecuteCommand(cmd) | |
| 188 if status != 0: | |
| 189 return status | |
| 190 print 'Uploaded: ' + output | |
| 191 | |
| 192 # Set ACL. | |
| 193 if ACL is not None: | |
| 194 cmd = [GSUTIL, 'acl', 'set', ACL, target] | |
| 195 (status, output) = ExecuteCommand(cmd) | |
| 196 return status | |
| 197 | |
| 198 | |
| 199 def UploadFile(local_path, remote_path, checksum_files=False): | |
| 200 # Copy it to the new unified gs://dart-archive bucket | |
| 201 gsutil = bot_utils.GSUtil() | |
| 202 gsutil.upload(local_path, remote_path, public=True) | |
| 203 if checksum_files: | |
| 204 # 'local_path' may have a different filename than 'remote_path'. So we need | |
| 205 # to make sure the *.md5sum file contains the correct name. | |
| 206 assert '/' in remote_path and not remote_path.endswith('/') | |
| 207 mangled_filename = remote_path[remote_path.rfind('/') + 1:] | |
| 208 local_md5sum = bot_utils.CreateMD5ChecksumFile(local_path, | |
| 209 mangled_filename) | |
| 210 gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True) | |
| 211 local_sha256 = bot_utils.CreateSha256ChecksumFile(local_path, | |
| 212 mangled_filename) | |
| 213 gsutil.upload(local_sha256, remote_path + '.sha256sum', public=True) | |
| 214 | |
| 215 def ExecuteCommand(cmd): | |
| 216 """Execute a command in a subprocess. | |
| 217 """ | |
| 218 print 'Executing: ' + ' '.join(cmd) | |
| 219 try: | |
| 220 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| 221 (output, error) = pipe.communicate() | |
| 222 if pipe.returncode != 0: | |
| 223 print 'Execution failed: ' + str(error) | |
| 224 return (pipe.returncode, output) | |
| 225 except: | |
| 226 import traceback | |
| 227 print 'Execution raised exception:', traceback.format_exc() | |
| 228 return (-1, '') | |
| 229 | |
| 230 | |
| 231 def UploadDartTestsResults(layout_test_results_dir, name, version, | |
| 232 component, checked): | |
| 233 """Uploads test results to google storage. | |
| 234 """ | |
| 235 print ('@@@BUILD_STEP archive %s_layout_%s_tests results@@@' % | |
| 236 (component, checked)) | |
| 237 dir_name = os.path.dirname(layout_test_results_dir) | |
| 238 base_name = os.path.basename(layout_test_results_dir) | |
| 239 cwd = os.getcwd() | |
| 240 try: | |
| 241 os.chdir(dir_name) | |
| 242 | |
| 243 archive_name = 'layout_test_results.zip' | |
| 244 archive.ZipDir(archive_name, base_name) | |
| 245 | |
| 246 target = '/'.join([GS_DIR, 'layout-test-results', name, component + '-' + | |
| 247 checked + '-' + version + '.zip']) | |
| 248 status = OldUploadFile(os.path.abspath(archive_name), GS_SITE + target) | |
| 249 os.remove(archive_name) | |
| 250 if status == 0: | |
| 251 print ('@@@STEP_LINK@download@' + GS_URL + target + '@@@') | |
| 252 else: | |
| 253 print '@@@STEP_FAILURE@@@' | |
| 254 except: | |
| 255 print '@@@STEP_FAILURE@@@' | |
| 256 os.chdir(cwd) | |
| 257 | |
| 258 | |
| 259 def ListArchives(pattern): | |
| 260 """List the contents in Google storage matching the file pattern. | |
| 261 """ | |
| 262 cmd = [GSUTIL, 'ls', pattern] | |
| 263 (status, output) = ExecuteCommand(cmd) | |
| 264 if status != 0: | |
| 265 return [] | |
| 266 return output.split(os.linesep) | |
| 267 | |
| 268 | |
| 269 def RemoveArchives(archives): | |
| 270 """Remove the list of archives in Google storage. | |
| 271 """ | |
| 272 for archive in archives: | |
| 273 if archive.find(GS_SITE) == 0: | |
| 274 cmd = [GSUTIL, 'rm', archive.rstrip()] | |
| 275 (status, _) = ExecuteCommand(cmd) | |
| 276 if status != 0: | |
| 277 return status | |
| 278 return 0 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2976303002:
Remove Dartium for TIP of origin/master (Closed)
