| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/python | |
| 2 | |
| 3 # Copyright (c) 2011 The Chromium Authors. All rights reserved. | |
| 4 # Use of this source code is governed by a BSD-style license that can be | |
| 5 # found in the LICENSE file. | |
| 6 | |
| 7 """Chromium buildbot steps | |
| 8 | |
| 9 Run the Dart layout tests. | |
| 10 """ | |
| 11 | |
| 12 import os | |
| 13 import platform | |
| 14 import re | |
| 15 import shutil | |
| 16 import socket | |
| 17 import subprocess | |
| 18 import sys | |
| 19 import imp | |
| 20 | |
| 21 BUILDER_NAME = 'BUILDBOT_BUILDERNAME' | |
| 22 REVISION = 'BUILDBOT_REVISION' | |
| 23 BUILDER_PATTERN = (r'^dartium-(mac|lucid64|lucid32|win)' | |
| 24 r'-(full|inc|debug)(-ninja)?(-(be|dev|stable|integration))?$'
) | |
| 25 | |
| 26 if platform.system() == 'Windows': | |
| 27 GSUTIL = 'e:/b/build/scripts/slave/gsutil.bat' | |
| 28 else: | |
| 29 GSUTIL = '/b/build/scripts/slave/gsutil' | |
| 30 ACL = 'public-read' | |
| 31 GS_SITE = 'gs://' | |
| 32 GS_URL = 'https://sandbox.google.com/storage/' | |
| 33 GS_DIR = 'dartium-archive' | |
| 34 LATEST = 'latest' | |
| 35 CONTINUOUS = 'continuous' | |
| 36 | |
| 37 REVISION_FILE = 'chrome/browser/ui/webui/dartvm_revision.h' | |
| 38 | |
| 39 # Add dartium tools and build/util to python path. | |
| 40 SRC_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| 41 TOOLS_PATH = os.path.join(SRC_PATH, 'dartium_tools') | |
| 42 DART_PATH = os.path.join(SRC_PATH, 'dart') | |
| 43 BUILD_UTIL_PATH = os.path.join(SRC_PATH, 'build/util') | |
| 44 # We limit testing on drt since it takes a long time to run | |
| 45 DRT_FILTER = 'html' | |
| 46 | |
| 47 | |
| 48 sys.path.extend([TOOLS_PATH, BUILD_UTIL_PATH]) | |
| 49 import archive | |
| 50 import utils | |
| 51 | |
| 52 bot_utils = imp.load_source('bot_utils', | |
| 53 os.path.join(DART_PATH, 'tools', 'bots', 'bot_utils.py')) | |
| 54 | |
| 55 def DartArchiveFile(local_path, remote_path, create_md5sum=False): | |
| 56 # Copy it to the new unified gs://dart-archive bucket | |
| 57 # TODO(kustermann/ricow): Remove all the old archiving code, once everything | |
| 58 # points to the new location | |
| 59 gsutil = bot_utils.GSUtil() | |
| 60 gsutil.upload(local_path, remote_path, public=True) | |
| 61 if create_md5sum: | |
| 62 # 'local_path' may have a different filename than 'remote_path'. So we need | |
| 63 # to make sure the *.md5sum file contains the correct name. | |
| 64 assert '/' in remote_path and not remote_path.endswith('/') | |
| 65 mangled_filename = remote_path[remote_path.rfind('/') + 1:] | |
| 66 local_md5sum = bot_utils.CreateChecksumFile(local_path, mangled_filename) | |
| 67 gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True) | |
| 68 | |
| 69 def UploadDartiumVariant(revision, name, channel, arch, mode, zip_file): | |
| 70 name = name.replace('drt', 'content_shell') | |
| 71 system = sys.platform | |
| 72 | |
| 73 namer = bot_utils.GCSNamer(channel, bot_utils.ReleaseType.RAW) | |
| 74 remote_path = namer.dartium_variant_zipfilepath(revision, name, system, arch, | |
| 75 mode) | |
| 76 DartArchiveFile(zip_file, remote_path, create_md5sum=True) | |
| 77 return remote_path | |
| 78 | |
| 79 def ExecuteCommand(cmd): | |
| 80 """Execute a command in a subprocess. | |
| 81 """ | |
| 82 print 'Executing: ' + ' '.join(cmd) | |
| 83 try: | |
| 84 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| 85 (output, error) = pipe.communicate() | |
| 86 if pipe.returncode != 0: | |
| 87 print 'Execution failed: ' + str(error) | |
| 88 return (pipe.returncode, output) | |
| 89 except: | |
| 90 import traceback | |
| 91 print 'Execution raised exception:', traceback.format_exc() | |
| 92 return (-1, '') | |
| 93 | |
| 94 | |
| 95 # TODO: Instead of returning a tuple we should make a class with these fields. | |
| 96 def GetBuildInfo(): | |
| 97 """Returns a tuple (name, dart_revision, version, mode, arch, channel, | |
| 98 is_full) where: | |
| 99 - name: A name for the build - the buildbot host if a buildbot. | |
| 100 - dart_revision: The dart revision. | |
| 101 - version: A version string corresponding to this build. | |
| 102 - mode: 'Debug' or 'Release' | |
| 103 - arch: target architecture | |
| 104 - channel: the channel this build is happening on | |
| 105 - is_full: True if this is a full build. | |
| 106 """ | |
| 107 os.chdir(SRC_PATH) | |
| 108 | |
| 109 name = None | |
| 110 version = None | |
| 111 mode = 'Release' | |
| 112 | |
| 113 # Populate via builder environment variables. | |
| 114 name = os.environ[BUILDER_NAME] | |
| 115 | |
| 116 # We need to chdir() to src/dart in order to get the correct revision number. | |
| 117 with utils.ChangedWorkingDirectory(DART_PATH): | |
| 118 dart_tools_utils = imp.load_source('dart_tools_utils', | |
| 119 os.path.join('tools', 'utils.py')) | |
| 120 dart_revision = dart_tools_utils.GetSVNRevision() | |
| 121 | |
| 122 version = dart_revision + '.0' | |
| 123 is_incremental = '-inc' in name | |
| 124 is_win_ninja = 'win-inc-ninja' in name | |
| 125 is_full = False | |
| 126 | |
| 127 pattern = re.match(BUILDER_PATTERN, name) | |
| 128 assert pattern | |
| 129 arch = 'x64' if pattern.group(1) == 'lucid64' else 'ia32' | |
| 130 if pattern.group(2) == 'debug': | |
| 131 mode = 'Debug' | |
| 132 is_full = pattern.group(2) == 'full' | |
| 133 channel = pattern.group(5) | |
| 134 if not channel: | |
| 135 channel = 'be' | |
| 136 | |
| 137 # Fall back if not on builder. | |
| 138 if not name: | |
| 139 name = socket.gethostname().split('.')[0] | |
| 140 | |
| 141 return (name, dart_revision, version, mode, arch, channel, is_full, | |
| 142 is_incremental, is_win_ninja) | |
| 143 | |
| 144 | |
| 145 def RunDartTests(mode, component, suite, arch, checked, test_filter=None, | |
| 146 is_win_ninja=False): | |
| 147 """Runs the Dart WebKit Layout tests. | |
| 148 """ | |
| 149 cmd = [sys.executable] | |
| 150 script = os.path.join(TOOLS_PATH, 'test.py') | |
| 151 cmd.append(script) | |
| 152 cmd.append('--buildbot') | |
| 153 cmd.append('--mode=' + mode) | |
| 154 cmd.append('--component=' + component) | |
| 155 cmd.append('--suite=' + suite) | |
| 156 cmd.append('--arch=' + arch) | |
| 157 cmd.append('--' + checked) | |
| 158 cmd.append('--no-show-results') | |
| 159 | |
| 160 if is_win_ninja: | |
| 161 cmd.append('--win-ninja-build') | |
| 162 | |
| 163 if test_filter: | |
| 164 cmd.append('--test-filter=' + test_filter) | |
| 165 | |
| 166 status = subprocess.call(cmd) | |
| 167 if status != 0: | |
| 168 print '@@@STEP_FAILURE@@@' | |
| 169 return status | |
| 170 | |
| 171 | |
| 172 def UploadDartTestsResults(layout_test_results_dir, name, version, | |
| 173 component, checked): | |
| 174 """Uploads test results to google storage. | |
| 175 """ | |
| 176 print ('@@@BUILD_STEP archive %s_layout_%s_tests results@@@' % | |
| 177 (component, checked)) | |
| 178 dir_name = os.path.dirname(layout_test_results_dir) | |
| 179 base_name = os.path.basename(layout_test_results_dir) | |
| 180 cwd = os.getcwd() | |
| 181 os.chdir(dir_name) | |
| 182 | |
| 183 archive_name = 'layout_test_results.zip' | |
| 184 archive.ZipDir(archive_name, base_name) | |
| 185 | |
| 186 target = '/'.join([GS_DIR, 'layout-test-results', name, component + '-' + | |
| 187 checked + '-' + version + '.zip']) | |
| 188 status = UploadArchive(os.path.abspath(archive_name), GS_SITE + target) | |
| 189 os.remove(archive_name) | |
| 190 if status == 0: | |
| 191 print ('@@@STEP_LINK@download@' + GS_URL + target + '@@@') | |
| 192 else: | |
| 193 print '@@@STEP_FAILURE@@@' | |
| 194 os.chdir(cwd) | |
| 195 | |
| 196 | |
| 197 def ListArchives(pattern): | |
| 198 """List the contents in Google storage matching the file pattern. | |
| 199 """ | |
| 200 cmd = [GSUTIL, 'ls', pattern] | |
| 201 (status, output) = ExecuteCommand(cmd) | |
| 202 if status != 0: | |
| 203 return [] | |
| 204 return output.split(os.linesep) | |
| 205 | |
| 206 | |
| 207 def RemoveArchives(archives): | |
| 208 """Remove the list of archives in Google storage. | |
| 209 """ | |
| 210 for archive in archives: | |
| 211 if archive.find(GS_SITE) == 0: | |
| 212 cmd = [GSUTIL, 'rm', archive.rstrip()] | |
| 213 (status, _) = ExecuteCommand(cmd) | |
| 214 if status != 0: | |
| 215 return status | |
| 216 return 0 | |
| 217 | |
| 218 | |
| 219 def UploadArchive(source, target): | |
| 220 """Upload an archive zip file to Google storage. | |
| 221 """ | |
| 222 | |
| 223 # Upload file. | |
| 224 cmd = [GSUTIL, 'cp', source, target] | |
| 225 (status, output) = ExecuteCommand(cmd) | |
| 226 if status != 0: | |
| 227 return status | |
| 228 print 'Uploaded: ' + output | |
| 229 | |
| 230 # Set ACL. | |
| 231 if ACL is not None: | |
| 232 cmd = [GSUTIL, 'setacl', ACL, target] | |
| 233 (status, output) = ExecuteCommand(cmd) | |
| 234 return status | |
| 235 | |
| 236 | |
| 237 def main(): | |
| 238 (dartium_bucket, dart_revision, version, mode, arch, channel, | |
| 239 is_full, is_incremental, is_win_ninja) = GetBuildInfo() | |
| 240 drt_bucket = dartium_bucket.replace('dartium', 'drt') | |
| 241 chromedriver_bucket = dartium_bucket.replace('dartium', 'chromedriver') | |
| 242 | |
| 243 def archiveAndUpload(archive_latest=False): | |
| 244 print '@@@BUILD_STEP dartium_generate_archive@@@' | |
| 245 cwd = os.getcwd() | |
| 246 dartium_archive = dartium_bucket + '-' + version | |
| 247 drt_archive = drt_bucket + '-' + version | |
| 248 chromedriver_archive = chromedriver_bucket + '-' + version | |
| 249 dartium_zip, drt_zip, chromedriver_zip = \ | |
| 250 archive.Archive(SRC_PATH, mode, dartium_archive, | |
| 251 drt_archive, chromedriver_archive, | |
| 252 is_win_ninja=is_win_ninja) | |
| 253 status = upload('dartium', dartium_bucket, os.path.abspath(dartium_zip), | |
| 254 archive_latest=archive_latest) | |
| 255 if status == 0: | |
| 256 status = upload('drt', drt_bucket, os.path.abspath(drt_zip), | |
| 257 archive_latest=archive_latest) | |
| 258 if status == 0: | |
| 259 status = upload('chromedriver', chromedriver_bucket, | |
| 260 os.path.abspath(chromedriver_zip), | |
| 261 archive_latest=archive_latest) | |
| 262 os.chdir(cwd) | |
| 263 if status != 0: | |
| 264 print '@@@STEP_FAILURE@@@' | |
| 265 return status | |
| 266 | |
| 267 def upload(module, bucket, zip_file, archive_latest=False): | |
| 268 status = 0 | |
| 269 | |
| 270 # We archive to the new location on all builders except for -inc builders. | |
| 271 if not is_incremental: | |
| 272 print '@@@BUILD_STEP %s_upload_archive_new @@@' % module | |
| 273 # We archive the full builds to gs://dart-archive/ | |
| 274 revision = 'latest' if archive_latest else dart_revision | |
| 275 remote_path = UploadDartiumVariant(revision, module, channel, arch, | |
| 276 mode.lower(), zip_file) | |
| 277 print '@@@STEP_LINK@download@' + remote_path + '@@@' | |
| 278 | |
| 279 # We archive to the old locations only for bleeding_edge builders | |
| 280 if channel == 'be': | |
| 281 _, filename = os.path.split(zip_file) | |
| 282 if not archive_latest: | |
| 283 target = '/'.join([GS_DIR, bucket, filename]) | |
| 284 print '@@@BUILD_STEP %s_upload_archive@@@' % module | |
| 285 status = UploadArchive(zip_file, GS_SITE + target) | |
| 286 print '@@@STEP_LINK@download@' + GS_URL + target + '@@@' | |
| 287 else: | |
| 288 print '@@@BUILD_STEP %s_upload_latest@@@' % module | |
| 289 # Clear latest for this build type. | |
| 290 old = '/'.join([GS_DIR, LATEST, bucket + '-*']) | |
| 291 old_archives = ListArchives(GS_SITE + old) | |
| 292 | |
| 293 # Upload the new latest and remove unnecessary old ones. | |
| 294 target = GS_SITE + '/'.join([GS_DIR, LATEST, filename]) | |
| 295 status = UploadArchive(zip_file, target) | |
| 296 if status == 0: | |
| 297 RemoveArchives( | |
| 298 [iarch for iarch in old_archives if iarch != target]) | |
| 299 else: | |
| 300 print 'Upload failed' | |
| 301 | |
| 302 # Upload unversioned name to continuous site for incremental | |
| 303 # builds. | |
| 304 if '-inc' in bucket: | |
| 305 continuous_name = bucket[:bucket.find('-inc')] | |
| 306 target = GS_SITE + '/'.join([GS_DIR, CONTINUOUS, | |
| 307 continuous_name + '.zip']) | |
| 308 status = UploadArchive(zip_file, target) | |
| 309 | |
| 310 print ('@@@BUILD_STEP %s_upload_archive is over (status = %s)@@@' % | |
| 311 (module, status)) | |
| 312 | |
| 313 return status | |
| 314 | |
| 315 def test(component, suite, checked, test_filter=None): | |
| 316 """Test a particular component (e.g., dartium or frog). | |
| 317 """ | |
| 318 print '@@@BUILD_STEP %s_%s_%s_tests@@@' % (component, suite, checked) | |
| 319 sys.stdout.flush() | |
| 320 layout_test_results_dir = os.path.join(SRC_PATH, 'webkit', mode, | |
| 321 'layout-test-results') | |
| 322 shutil.rmtree(layout_test_results_dir, ignore_errors=True) | |
| 323 status = RunDartTests(mode, component, suite, arch, checked, | |
| 324 test_filter=test_filter, is_win_ninja=is_win_ninja) | |
| 325 | |
| 326 if suite == 'layout' and status != 0: | |
| 327 UploadDartTestsResults(layout_test_results_dir, dartium_bucket, version, | |
| 328 component, checked) | |
| 329 return status | |
| 330 | |
| 331 result = 0 | |
| 332 | |
| 333 # Archive to the revision bucket unless integration build | |
| 334 if channel != 'integration': | |
| 335 result = archiveAndUpload(archive_latest=False) | |
| 336 | |
| 337 # On dev/stable we archive to the latest bucket as well | |
| 338 if channel != 'be': | |
| 339 result = archiveAndUpload(archive_latest=True) or result | |
| 340 | |
| 341 # Run layout tests | |
| 342 if mode == 'Release' or platform.system() != 'Darwin': | |
| 343 result = test('drt', 'layout', 'unchecked') or result | |
| 344 result = test('drt', 'layout', 'checked') or result | |
| 345 | |
| 346 # Run dartium tests | |
| 347 result = test('dartium', 'core', 'unchecked') or result | |
| 348 result = test('dartium', 'core', 'checked') or result | |
| 349 | |
| 350 # Run ContentShell tests | |
| 351 # NOTE: We don't run ContentShell tests on dartium-*-inc builders to keep | |
| 352 # cycle times down. | |
| 353 if not is_incremental: | |
| 354 # If we run all checked tests on dartium, we restrict the number of | |
| 355 # unchecked tests on drt to DRT_FILTER | |
| 356 result = test('drt', 'core', 'unchecked', test_filter=DRT_FILTER) or result | |
| 357 result = test('drt', 'core', 'checked') or result | |
| 358 | |
| 359 # On the 'be' channel, we only archive to the latest bucket if all tests ran | |
| 360 # successfull. | |
| 361 if result == 0 and channel == 'be': | |
| 362 result = archiveAndUpload(archive_latest=True) or result | |
| 363 | |
| 364 # BIG HACK | |
| 365 # Normal ninja clobbering does not work due to symlinks/python on windows | |
| 366 # Full clobbering before building does not work since it will destroy | |
| 367 # the ninja build files | |
| 368 # So we basically clobber at the end here | |
| 369 if is_full and platform.system() == 'Windows': | |
| 370 print '@@@BUILD_STEP Dartium hackish clobber@@@' | |
| 371 shutil.rmtree(os.path.join(SRC_PATH, 'out'), ignore_errors=True) | |
| 372 | |
| 373 if __name__ == '__main__': | |
| 374 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 239993009:
Revert accidental dartium code push (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/src