| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Front end tool to manage .isolate files and corresponding tests. | |
| 7 | |
| 8 Run ./isolate.py --help for more detailed information. | |
| 9 | |
| 10 See more information at | |
| 11 http://dev.chromium.org/developers/testing/isolated-testing | |
| 12 """ | |
| 13 | |
| 14 import binascii | |
| 15 import copy | |
| 16 import hashlib | |
| 17 import logging | |
| 18 import optparse | |
| 19 import os | |
| 20 import posixpath | |
| 21 import re | |
| 22 import stat | |
| 23 import subprocess | |
| 24 import sys | |
| 25 import time | |
| 26 import urllib | |
| 27 import urllib2 | |
| 28 | |
| 29 import trace_inputs | |
| 30 import run_test_from_archive | |
| 31 from run_test_from_archive import get_flavor | |
| 32 | |
| 33 # Used by process_input(). | |
| 34 NO_INFO, STATS_ONLY, WITH_HASH = range(56, 59) | |
| 35 SHA_1_NULL = hashlib.sha1().hexdigest() | |
| 36 | |
| 37 PATH_VARIABLES = ('DEPTH', 'PRODUCT_DIR') | |
| 38 DEFAULT_OSES = ('linux', 'mac', 'win') | |
| 39 | |
| 40 # Files that should be 0-length when mapped. | |
| 41 KEY_TOUCHED = 'isolate_dependency_touched' | |
| 42 # Files that should be tracked by the build tool. | |
| 43 KEY_TRACKED = 'isolate_dependency_tracked' | |
| 44 # Files that should not be tracked by the build tool. | |
| 45 KEY_UNTRACKED = 'isolate_dependency_untracked' | |
| 46 | |
| 47 _GIT_PATH = os.path.sep + '.git' | |
| 48 _SVN_PATH = os.path.sep + '.svn' | |
| 49 | |
| 50 # The maximum number of upload attempts to try when uploading a single file. | |
| 51 MAX_UPLOAD_ATTEMPTS = 5 | |
| 52 | |
| 53 # The minimum size of files to upload directly to the blobstore. | |
| 54 MIN_SIZE_FOR_DIRECT_BLOBSTORE = 20 * 8 | |
| 55 | |
| 56 | |
| 57 class ExecutionError(Exception): | |
| 58 """A generic error occurred.""" | |
| 59 def __str__(self): | |
| 60 return self.args[0] | |
| 61 | |
| 62 | |
| 63 ### Path handling code. | |
| 64 | |
| 65 | |
| 66 def relpath(path, root): | |
| 67 """os.path.relpath() that keeps trailing os.path.sep.""" | |
| 68 out = os.path.relpath(path, root) | |
| 69 if path.endswith(os.path.sep): | |
| 70 out += os.path.sep | |
| 71 return out | |
| 72 | |
| 73 | |
| 74 def normpath(path): | |
| 75 """os.path.normpath() that keeps trailing os.path.sep.""" | |
| 76 out = os.path.normpath(path) | |
| 77 if path.endswith(os.path.sep): | |
| 78 out += os.path.sep | |
| 79 return out | |
| 80 | |
| 81 | |
| 82 def posix_relpath(path, root): | |
| 83 """posix.relpath() that keeps trailing slash.""" | |
| 84 out = posixpath.relpath(path, root) | |
| 85 if path.endswith('/'): | |
| 86 out += '/' | |
| 87 return out | |
| 88 | |
| 89 | |
| 90 def cleanup_path(x): | |
| 91 """Cleans up a relative path. Converts any os.path.sep to '/' on Windows.""" | |
| 92 if x: | |
| 93 x = x.rstrip(os.path.sep).replace(os.path.sep, '/') | |
| 94 if x == '.': | |
| 95 x = '' | |
| 96 if x: | |
| 97 x += '/' | |
| 98 return x | |
| 99 | |
| 100 | |
| 101 def default_blacklist(f): | |
| 102 """Filters unimportant files normally ignored.""" | |
| 103 return ( | |
| 104 f.endswith(('.pyc', '.run_test_cases', 'testserver.log')) or | |
| 105 _GIT_PATH in f or | |
| 106 _SVN_PATH in f or | |
| 107 f in ('.git', '.svn')) | |
| 108 | |
| 109 | |
| 110 def expand_directory_and_symlink(indir, relfile, blacklist): | |
| 111 """Expands a single input. It can result in multiple outputs. | |
| 112 | |
| 113 This function is recursive when relfile is a directory or a symlink. | |
| 114 | |
| 115 Note: this code doesn't properly handle recursive symlink like one created | |
| 116 with: | |
| 117 ln -s .. foo | |
| 118 """ | |
| 119 if os.path.isabs(relfile): | |
| 120 raise run_test_from_archive.MappingError( | |
| 121 'Can\'t map absolute path %s' % relfile) | |
| 122 | |
| 123 infile = normpath(os.path.join(indir, relfile)) | |
| 124 if not infile.startswith(indir): | |
| 125 raise run_test_from_archive.MappingError( | |
| 126 'Can\'t map file %s outside %s' % (infile, indir)) | |
| 127 | |
| 128 if sys.platform != 'win32': | |
| 129 # Look if any item in relfile is a symlink. | |
| 130 base, symlink, rest = trace_inputs.split_at_symlink(indir, relfile) | |
| 131 if symlink: | |
| 132 # Append everything pointed by the symlink. If the symlink is recursive, | |
| 133 # this code blows up. | |
| 134 symlink_relfile = os.path.join(base, symlink) | |
| 135 symlink_path = os.path.join(indir, symlink_relfile) | |
| 136 pointed = os.readlink(symlink_path) | |
| 137 dest_infile = normpath( | |
| 138 os.path.join(os.path.dirname(symlink_path), pointed)) | |
| 139 if rest: | |
| 140 dest_infile = trace_inputs.safe_join(dest_infile, rest) | |
| 141 if not dest_infile.startswith(indir): | |
| 142 raise run_test_from_archive.MappingError( | |
| 143 'Can\'t map symlink reference %s (from %s) ->%s outside of %s' % | |
| 144 (symlink_relfile, relfile, dest_infile, indir)) | |
| 145 if infile.startswith(dest_infile): | |
| 146 raise run_test_from_archive.MappingError( | |
| 147 'Can\'t map recursive symlink reference %s->%s' % | |
| 148 (symlink_relfile, dest_infile)) | |
| 149 dest_relfile = dest_infile[len(indir)+1:] | |
| 150 logging.info('Found symlink: %s -> %s' % (symlink_relfile, dest_relfile)) | |
| 151 out = expand_directory_and_symlink(indir, dest_relfile, blacklist) | |
| 152 # Add the symlink itself. | |
| 153 out.append(symlink_relfile) | |
| 154 return out | |
| 155 | |
| 156 if relfile.endswith(os.path.sep): | |
| 157 if not os.path.isdir(infile): | |
| 158 raise run_test_from_archive.MappingError( | |
| 159 '%s is not a directory but ends with "%s"' % (infile, os.path.sep)) | |
| 160 | |
| 161 outfiles = [] | |
| 162 for filename in os.listdir(infile): | |
| 163 inner_relfile = os.path.join(relfile, filename) | |
| 164 if blacklist(inner_relfile): | |
| 165 continue | |
| 166 if os.path.isdir(os.path.join(indir, inner_relfile)): | |
| 167 inner_relfile += os.path.sep | |
| 168 outfiles.extend( | |
| 169 expand_directory_and_symlink(indir, inner_relfile, blacklist)) | |
| 170 return outfiles | |
| 171 else: | |
| 172 # Always add individual files even if they were blacklisted. | |
| 173 if os.path.isdir(infile): | |
| 174 raise run_test_from_archive.MappingError( | |
| 175 'Input directory %s must have a trailing slash' % infile) | |
| 176 | |
| 177 if not os.path.isfile(infile): | |
| 178 raise run_test_from_archive.MappingError( | |
| 179 'Input file %s doesn\'t exist' % infile) | |
| 180 | |
| 181 return [relfile] | |
| 182 | |
| 183 | |
| 184 def expand_directories_and_symlinks(indir, infiles, blacklist): | |
| 185 """Expands the directories and the symlinks, applies the blacklist and | |
| 186 verifies files exist. | |
| 187 | |
| 188 Files are specified in os native path separator. | |
| 189 """ | |
| 190 outfiles = [] | |
| 191 for relfile in infiles: | |
| 192 outfiles.extend(expand_directory_and_symlink(indir, relfile, blacklist)) | |
| 193 return outfiles | |
| 194 | |
| 195 | |
| 196 def recreate_tree(outdir, indir, infiles, action, as_sha1): | |
| 197 """Creates a new tree with only the input files in it. | |
| 198 | |
| 199 Arguments: | |
| 200 outdir: Output directory to create the files in. | |
| 201 indir: Root directory the infiles are based in. | |
| 202 infiles: dict of files to map from |indir| to |outdir|. | |
| 203 action: See assert below. | |
| 204 as_sha1: Output filename is the sha1 instead of relfile. | |
| 205 """ | |
| 206 logging.info( | |
| 207 'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_sha1=%s)' % | |
| 208 (outdir, indir, len(infiles), action, as_sha1)) | |
| 209 | |
| 210 assert action in ( | |
| 211 run_test_from_archive.HARDLINK, | |
| 212 run_test_from_archive.SYMLINK, | |
| 213 run_test_from_archive.COPY) | |
| 214 outdir = os.path.normpath(outdir) | |
| 215 if not os.path.isdir(outdir): | |
| 216 logging.info ('Creating %s' % outdir) | |
| 217 os.makedirs(outdir) | |
| 218 # Do not call abspath until the directory exists. | |
| 219 outdir = os.path.abspath(outdir) | |
| 220 | |
| 221 for relfile, metadata in infiles.iteritems(): | |
| 222 infile = os.path.join(indir, relfile) | |
| 223 if as_sha1: | |
| 224 # Do the hashtable specific checks. | |
| 225 if 'link' in metadata: | |
| 226 # Skip links when storing a hashtable. | |
| 227 continue | |
| 228 outfile = os.path.join(outdir, metadata['sha-1']) | |
| 229 if os.path.isfile(outfile): | |
| 230 # Just do a quick check that the file size matches. No need to stat() | |
| 231 # again the input file, grab the value from the dict. | |
| 232 if metadata['size'] == os.stat(outfile).st_size: | |
| 233 continue | |
| 234 else: | |
| 235 logging.warn('Overwritting %s' % metadata['sha-1']) | |
| 236 os.remove(outfile) | |
| 237 else: | |
| 238 outfile = os.path.join(outdir, relfile) | |
| 239 outsubdir = os.path.dirname(outfile) | |
| 240 if not os.path.isdir(outsubdir): | |
| 241 os.makedirs(outsubdir) | |
| 242 | |
| 243 # TODO(csharp): Fix crbug.com/150823 and enable the touched logic again. | |
| 244 # if metadata.get('touched_only') == True: | |
| 245 # open(outfile, 'ab').close() | |
| 246 if 'link' in metadata: | |
| 247 pointed = metadata['link'] | |
| 248 logging.debug('Symlink: %s -> %s' % (outfile, pointed)) | |
| 249 os.symlink(pointed, outfile) | |
| 250 else: | |
| 251 run_test_from_archive.link_file(outfile, infile, action) | |
| 252 | |
| 253 | |
| 254 def encode_multipart_formdata(fields, files, | |
| 255 mime_mapper=lambda _: 'application/octet-stream'): | |
| 256 """Encodes a Multipart form data object. | |
| 257 | |
| 258 Args: | |
| 259 fields: a sequence (name, value) elements for | |
| 260 regular form fields. | |
| 261 files: a sequence of (name, filename, value) elements for data to be | |
| 262 uploaded as files. | |
| 263 mime_mapper: function to return the mime type from the filename. | |
| 264 Returns: | |
| 265 content_type: for httplib.HTTP instance | |
| 266 body: for httplib.HTTP instance | |
| 267 """ | |
| 268 boundary = hashlib.md5(str(time.time())).hexdigest() | |
| 269 body_list = [] | |
| 270 for (key, value) in fields: | |
| 271 body_list.append('--' + boundary) | |
| 272 body_list.append('Content-Disposition: form-data; name="%s"' % key) | |
| 273 body_list.append('') | |
| 274 body_list.append(value) | |
| 275 body_list.append('--' + boundary) | |
| 276 body_list.append('') | |
| 277 for (key, filename, value) in files: | |
| 278 body_list.append('--' + boundary) | |
| 279 body_list.append('Content-Disposition: form-data; name="%s"; ' | |
| 280 'filename="%s"' % (key, filename)) | |
| 281 body_list.append('Content-Type: %s' % mime_mapper(filename)) | |
| 282 body_list.append('') | |
| 283 body_list.append(value) | |
| 284 body_list.append('--' + boundary) | |
| 285 body_list.append('') | |
| 286 if body_list: | |
| 287 body_list[-2] += '--' | |
| 288 body = '\r\n'.join(body_list) | |
| 289 content_type = 'multipart/form-data; boundary=%s' % boundary | |
| 290 return content_type, body | |
| 291 | |
| 292 | |
| 293 def upload_hash_content(url, params=None, payload=None, | |
| 294 content_type='application/octet-stream'): | |
| 295 """Uploads the given hash contents. | |
| 296 | |
| 297 Arguments: | |
| 298 url: The url to upload the hash contents to. | |
| 299 params: The params to include with the upload. | |
| 300 payload: The data to upload. | |
| 301 content_type: The content_type of the data being uploaded. | |
| 302 """ | |
| 303 if params: | |
| 304 url = url + '?' + urllib.urlencode(params) | |
| 305 request = urllib2.Request(url, data=payload) | |
| 306 request.add_header('Content-Type', content_type) | |
| 307 request.add_header('Content-Length', len(payload or '')) | |
| 308 | |
| 309 return urllib2.urlopen(request) | |
| 310 | |
| 311 | |
| 312 def upload_hash_content_to_blobstore(generate_upload_url, params, | |
| 313 hash_data): | |
| 314 """Uploads the given hash contents directly to the blobsotre via a generated | |
| 315 url. | |
| 316 | |
| 317 Arguments: | |
| 318 generate_upload_url: The url to get the new upload url from. | |
| 319 params: The params to include with the upload. | |
| 320 hash_contents: The contents to upload. | |
| 321 """ | |
| 322 content_type, body = encode_multipart_formdata( | |
| 323 params.items(), [('hash_contents', 'hash_contest', hash_data)]) | |
| 324 | |
| 325 logging.debug('Generating url to directly upload file to blobstore') | |
| 326 response = urllib2.urlopen(generate_upload_url) | |
| 327 upload_url = response.read() | |
| 328 | |
| 329 if not upload_url: | |
| 330 logging.error('Unable to generate upload url') | |
| 331 return | |
| 332 | |
| 333 return upload_hash_content(upload_url, payload=body, | |
| 334 content_type=content_type) | |
| 335 | |
| 336 | |
| 337 class UploadRemote(run_test_from_archive.Remote): | |
| 338 @staticmethod | |
| 339 def get_file_handler(base_url): | |
| 340 def upload_file(hash_data, hash_key): | |
| 341 params = {'hash_key': hash_key} | |
| 342 if len(hash_data) > MIN_SIZE_FOR_DIRECT_BLOBSTORE: | |
| 343 upload_hash_content_to_blobstore( | |
| 344 base_url.rstrip('/') + '/content/generate_blobstore_url', | |
| 345 params, hash_data) | |
| 346 else: | |
| 347 upload_hash_content( | |
| 348 base_url.rstrip('/') + '/content/store', params, hash_data) | |
| 349 return upload_file | |
| 350 | |
| 351 | |
| 352 def url_open(url, data=None, max_retries=MAX_UPLOAD_ATTEMPTS): | |
| 353 """Opens the given url with the given data, repeating up to max_retries | |
| 354 times if it encounters an error. | |
| 355 | |
| 356 Arguments: | |
| 357 url: The url to open. | |
| 358 data: The data to send to the url. | |
| 359 max_retries: The maximum number of times to try connecting to the url. | |
| 360 | |
| 361 Returns: | |
| 362 The response from the url, or it raises an exception it it failed to get | |
| 363 a response. | |
| 364 """ | |
| 365 for _ in range(max_retries): | |
| 366 try: | |
| 367 response = urllib2.urlopen(url, data=data) | |
| 368 except urllib2.URLError as e: | |
| 369 logging.warning('Unable to connect to %s, error msg: %s', url, e) | |
| 370 time.sleep(1) | |
| 371 | |
| 372 # If we get no response from the server after max_retries, assume it | |
| 373 # is down and raise an exception | |
| 374 if response is None: | |
| 375 raise run_test_from_archive.MappingError('Unable to connect to server, %s, ' | |
| 376 'to see which files are presents' % | |
| 377 url) | |
| 378 | |
| 379 return response | |
| 380 | |
| 381 | |
| 382 def update_files_to_upload(query_url, queries, files_to_upload): | |
| 383 """Queries the server to see which files from this batch already exist there. | |
| 384 | |
| 385 Arguments: | |
| 386 queries: The hash files to potential upload to the server. | |
| 387 files_to_upload: Any new files that need to be upload are added to | |
| 388 this list. | |
| 389 """ | |
| 390 body = ''.join( | |
| 391 (binascii.unhexlify(meta_data['sha-1']) for (_, meta_data) in queries)) | |
| 392 response = url_open(query_url, data=body).read() | |
| 393 if len(queries) != len(response): | |
| 394 raise run_test_from_archive.MappingError( | |
| 395 'Got an incorrect number of responses from the server. Expected %d, ' | |
| 396 'but got %d' % (len(queries), len(response))) | |
| 397 | |
| 398 for i in range(len(response)): | |
| 399 if response[i] == chr(0): | |
| 400 files_to_upload.append(queries[i]) | |
| 401 else: | |
| 402 logging.debug('Hash for %s already exists on the server, no need ' | |
| 403 'to upload again', queries[i][0]) | |
| 404 | |
| 405 | |
| 406 def upload_sha1_tree(base_url, indir, infiles): | |
| 407 """Uploads the given tree to the given url. | |
| 408 | |
| 409 Arguments: | |
| 410 base_url: The base url, it is assume that |base_url|/has/ can be used to | |
| 411 query if an element was already uploaded, and |base_url|/store/ | |
| 412 can be used to upload a new element. | |
| 413 indir: Root directory the infiles are based in. | |
| 414 infiles: dict of files to map from |indir| to |outdir|. | |
| 415 """ | |
| 416 logging.info('upload tree(base_url=%s, indir=%s, files=%d)' % | |
| 417 (base_url, indir, len(infiles))) | |
| 418 | |
| 419 # Generate the list of files that need to be uploaded (since some may already | |
| 420 # be on the server. | |
| 421 base_url = base_url.rstrip('/') | |
| 422 contains_hash_url = base_url + '/content/contains' | |
| 423 to_upload = [] | |
| 424 next_queries = [] | |
| 425 for relfile, metadata in infiles.iteritems(): | |
| 426 if 'link' in metadata: | |
| 427 # Skip links when uploading. | |
| 428 continue | |
| 429 | |
| 430 next_queries.append((relfile, metadata)) | |
| 431 if len(next_queries) == 1000: | |
| 432 update_files_to_upload(contains_hash_url, next_queries, to_upload) | |
| 433 next_queries = [] | |
| 434 | |
| 435 if next_queries: | |
| 436 update_files_to_upload(contains_hash_url, next_queries, to_upload) | |
| 437 | |
| 438 | |
| 439 # Upload the required files. | |
| 440 remote_uploader = UploadRemote(base_url) | |
| 441 for relfile, metadata in to_upload: | |
| 442 # TODO(csharp): Fix crbug.com/150823 and enable the touched logic again. | |
| 443 # if metadata.get('touched_only') == True: | |
| 444 # hash_data = '' | |
| 445 infile = os.path.join(indir, relfile) | |
| 446 with open(infile, 'rb') as f: | |
| 447 hash_data = f.read() | |
| 448 remote_uploader.add_item(run_test_from_archive.Remote.MED, | |
| 449 hash_data, | |
| 450 metadata['sha-1']) | |
| 451 remote_uploader.join() | |
| 452 | |
| 453 exception = remote_uploader.next_exception() | |
| 454 if exception: | |
| 455 while exception: | |
| 456 logging.error('Error uploading file to server:\n%s', exception[1]) | |
| 457 exception = remote_uploader.next_exception() | |
| 458 raise run_test_from_archive.MappingError( | |
| 459 'Encountered errors uploading hash contents to server. See logs for ' | |
| 460 'exact failures') | |
| 461 | |
| 462 | |
| 463 def process_input(filepath, prevdict, level, read_only): | |
| 464 """Processes an input file, a dependency, and return meta data about it. | |
| 465 | |
| 466 Arguments: | |
| 467 - filepath: File to act on. | |
| 468 - prevdict: the previous dictionary. It is used to retrieve the cached sha-1 | |
| 469 to skip recalculating the hash. | |
| 470 - level: determines the amount of information retrieved. | |
| 471 - read_only: If True, the file mode is manipulated. In practice, only save | |
| 472 one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r). On | |
| 473 windows, mode is not set since all files are 'executable' by | |
| 474 default. | |
| 475 | |
| 476 Behaviors: | |
| 477 - NO_INFO retrieves no information. | |
| 478 - STATS_ONLY retrieves the file mode, file size, file timestamp, file link | |
| 479 destination if it is a file link. | |
| 480 - WITH_HASH retrieves all of STATS_ONLY plus the sha-1 of the content of the | |
| 481 file. | |
| 482 """ | |
| 483 assert level in (NO_INFO, STATS_ONLY, WITH_HASH) | |
| 484 out = {} | |
| 485 # TODO(csharp): Fix crbug.com/150823 and enable the touched logic again. | |
| 486 # if prevdict.get('touched_only') == True: | |
| 487 # # The file's content is ignored. Skip the time and hard code mode. | |
| 488 # if get_flavor() != 'win': | |
| 489 # out['mode'] = stat.S_IRUSR | stat.S_IRGRP | |
| 490 # out['size'] = 0 | |
| 491 # out['sha-1'] = SHA_1_NULL | |
| 492 # out['touched_only'] = True | |
| 493 # return out | |
| 494 | |
| 495 if level >= STATS_ONLY: | |
| 496 try: | |
| 497 filestats = os.lstat(filepath) | |
| 498 except OSError: | |
| 499 # The file is not present. | |
| 500 raise run_test_from_archive.MappingError('%s is missing' % filepath) | |
| 501 is_link = stat.S_ISLNK(filestats.st_mode) | |
| 502 if get_flavor() != 'win': | |
| 503 # Ignore file mode on Windows since it's not really useful there. | |
| 504 filemode = stat.S_IMODE(filestats.st_mode) | |
| 505 # Remove write access for group and all access to 'others'. | |
| 506 filemode &= ~(stat.S_IWGRP | stat.S_IRWXO) | |
| 507 if read_only: | |
| 508 filemode &= ~stat.S_IWUSR | |
| 509 if filemode & stat.S_IXUSR: | |
| 510 filemode |= stat.S_IXGRP | |
| 511 else: | |
| 512 filemode &= ~stat.S_IXGRP | |
| 513 out['mode'] = filemode | |
| 514 if not is_link: | |
| 515 out['size'] = filestats.st_size | |
| 516 # Used to skip recalculating the hash. Use the most recent update time. | |
| 517 out['timestamp'] = int(round(filestats.st_mtime)) | |
| 518 # If the timestamp wasn't updated, carry on the sha-1. | |
| 519 if prevdict.get('timestamp') == out['timestamp']: | |
| 520 if 'sha-1' in prevdict: | |
| 521 # Reuse the previous hash. | |
| 522 out['sha-1'] = prevdict['sha-1'] | |
| 523 if 'link' in prevdict: | |
| 524 # Reuse the previous link destination. | |
| 525 out['link'] = prevdict['link'] | |
| 526 if is_link and not 'link' in out: | |
| 527 # A symlink, store the link destination. | |
| 528 out['link'] = os.readlink(filepath) | |
| 529 | |
| 530 if level >= WITH_HASH and not out.get('sha-1') and not out.get('link'): | |
| 531 if not is_link: | |
| 532 with open(filepath, 'rb') as f: | |
| 533 out['sha-1'] = hashlib.sha1(f.read()).hexdigest() | |
| 534 return out | |
| 535 | |
| 536 | |
| 537 ### Variable stuff. | |
| 538 | |
| 539 | |
| 540 def result_to_state(filename): | |
| 541 """Replaces the file's extension.""" | |
| 542 return filename.rsplit('.', 1)[0] + '.state' | |
| 543 | |
| 544 | |
| 545 def determine_root_dir(relative_root, infiles): | |
| 546 """For a list of infiles, determines the deepest root directory that is | |
| 547 referenced indirectly. | |
| 548 | |
| 549 All arguments must be using os.path.sep. | |
| 550 """ | |
| 551 # The trick used to determine the root directory is to look at "how far" back | |
| 552 # up it is looking up. | |
| 553 deepest_root = relative_root | |
| 554 for i in infiles: | |
| 555 x = relative_root | |
| 556 while i.startswith('..' + os.path.sep): | |
| 557 i = i[3:] | |
| 558 assert not i.startswith(os.path.sep) | |
| 559 x = os.path.dirname(x) | |
| 560 if deepest_root.startswith(x): | |
| 561 deepest_root = x | |
| 562 logging.debug( | |
| 563 'determine_root_dir(%s, %d files) -> %s' % ( | |
| 564 relative_root, len(infiles), deepest_root)) | |
| 565 return deepest_root | |
| 566 | |
| 567 | |
| 568 def replace_variable(part, variables): | |
| 569 m = re.match(r'<\(([A-Z_]+)\)', part) | |
| 570 if m: | |
| 571 if m.group(1) not in variables: | |
| 572 raise ExecutionError( | |
| 573 'Variable "%s" was not found in %s.\nDid you forget to specify ' | |
| 574 '--variable?' % (m.group(1), variables)) | |
| 575 return variables[m.group(1)] | |
| 576 return part | |
| 577 | |
| 578 | |
| 579 def process_variables(variables, relative_base_dir): | |
| 580 """Processes path variables as a special case and returns a copy of the dict. | |
| 581 | |
| 582 For each 'path' variable: first normalizes it, verifies it exists, converts it | |
| 583 to an absolute path, then sets it as relative to relative_base_dir. | |
| 584 """ | |
| 585 variables = variables.copy() | |
| 586 for i in PATH_VARIABLES: | |
| 587 if i not in variables: | |
| 588 continue | |
| 589 variable = os.path.normpath(variables[i]) | |
| 590 if not os.path.isdir(variable): | |
| 591 raise ExecutionError('%s=%s is not a directory' % (i, variable)) | |
| 592 # Variables could contain / or \ on windows. Always normalize to | |
| 593 # os.path.sep. | |
| 594 variable = os.path.abspath(variable.replace('/', os.path.sep)) | |
| 595 # All variables are relative to the .isolate file. | |
| 596 variables[i] = os.path.relpath(variable, relative_base_dir) | |
| 597 return variables | |
| 598 | |
| 599 | |
| 600 def eval_variables(item, variables): | |
| 601 """Replaces the .isolate variables in a string item. | |
| 602 | |
| 603 Note that the .isolate format is a subset of the .gyp dialect. | |
| 604 """ | |
| 605 return ''.join( | |
| 606 replace_variable(p, variables) for p in re.split(r'(<\([A-Z_]+\))', item)) | |
| 607 | |
| 608 | |
| 609 def classify_files(root_dir, tracked, untracked): | |
| 610 """Converts the list of files into a .isolate 'variables' dictionary. | |
| 611 | |
| 612 Arguments: | |
| 613 - tracked: list of files names to generate a dictionary out of that should | |
| 614 probably be tracked. | |
| 615 - untracked: list of files names that must not be tracked. | |
| 616 """ | |
| 617 # These directories are not guaranteed to be always present on every builder. | |
| 618 OPTIONAL_DIRECTORIES = ( | |
| 619 'test/data/plugin', | |
| 620 'third_party/WebKit/LayoutTests', | |
| 621 ) | |
| 622 | |
| 623 new_tracked = [] | |
| 624 new_untracked = list(untracked) | |
| 625 | |
| 626 def should_be_tracked(filepath): | |
| 627 """Returns True if it is a file without whitespace in a non-optional | |
| 628 directory that has no symlink in its path. | |
| 629 """ | |
| 630 if filepath.endswith('/'): | |
| 631 return False | |
| 632 if ' ' in filepath: | |
| 633 return False | |
| 634 if any(i in filepath for i in OPTIONAL_DIRECTORIES): | |
| 635 return False | |
| 636 # Look if any element in the path is a symlink. | |
| 637 split = filepath.split('/') | |
| 638 for i in range(len(split)): | |
| 639 if os.path.islink(os.path.join(root_dir, '/'.join(split[:i+1]))): | |
| 640 return False | |
| 641 return True | |
| 642 | |
| 643 for filepath in sorted(tracked): | |
| 644 if should_be_tracked(filepath): | |
| 645 new_tracked.append(filepath) | |
| 646 else: | |
| 647 # Anything else. | |
| 648 new_untracked.append(filepath) | |
| 649 | |
| 650 variables = {} | |
| 651 if new_tracked: | |
| 652 variables[KEY_TRACKED] = sorted(new_tracked) | |
| 653 if new_untracked: | |
| 654 variables[KEY_UNTRACKED] = sorted(new_untracked) | |
| 655 return variables | |
| 656 | |
| 657 | |
| 658 def generate_simplified( | |
| 659 tracked, untracked, touched, root_dir, variables, relative_cwd): | |
| 660 """Generates a clean and complete .isolate 'variables' dictionary. | |
| 661 | |
| 662 Cleans up and extracts only files from within root_dir then processes | |
| 663 variables and relative_cwd. | |
| 664 """ | |
| 665 logging.info( | |
| 666 'generate_simplified(%d files, %s, %s, %s)' % | |
| 667 (len(tracked) + len(untracked) + len(touched), | |
| 668 root_dir, variables, relative_cwd)) | |
| 669 # Constants. | |
| 670 # Skip log in PRODUCT_DIR. Note that these are applied on '/' style path | |
| 671 # separator. | |
| 672 LOG_FILE = re.compile(r'^\<\(PRODUCT_DIR\)\/[^\/]+\.log$') | |
| 673 EXECUTABLE = re.compile( | |
| 674 r'^(\<\(PRODUCT_DIR\)\/[^\/\.]+)' + | |
| 675 re.escape(variables.get('EXECUTABLE_SUFFIX', '')) + | |
| 676 r'$') | |
| 677 | |
| 678 # Preparation work. | |
| 679 relative_cwd = cleanup_path(relative_cwd) | |
| 680 # Creates the right set of variables here. We only care about PATH_VARIABLES. | |
| 681 variables = dict( | |
| 682 ('<(%s)' % k, variables[k].replace(os.path.sep, '/')) | |
| 683 for k in PATH_VARIABLES if k in variables) | |
| 684 | |
| 685 # Actual work: Process the files. | |
| 686 # TODO(maruel): if all the files in a directory are in part tracked and in | |
| 687 # part untracked, the directory will not be extracted. Tracked files should be | |
| 688 # 'promoted' to be untracked as needed. | |
| 689 tracked = trace_inputs.extract_directories( | |
| 690 root_dir, tracked, default_blacklist) | |
| 691 untracked = trace_inputs.extract_directories( | |
| 692 root_dir, untracked, default_blacklist) | |
| 693 # touched is not compressed, otherwise it would result in files to be archived | |
| 694 # that we don't need. | |
| 695 | |
| 696 def fix(f): | |
| 697 """Bases the file on the most restrictive variable.""" | |
| 698 logging.debug('fix(%s)' % f) | |
| 699 # Important, GYP stores the files with / and not \. | |
| 700 f = f.replace(os.path.sep, '/') | |
| 701 # If it's not already a variable. | |
| 702 if not f.startswith('<'): | |
| 703 # relative_cwd is usually the directory containing the gyp file. It may be | |
| 704 # empty if the whole directory containing the gyp file is needed. | |
| 705 f = posix_relpath(f, relative_cwd) or './' | |
| 706 | |
| 707 for variable, root_path in variables.iteritems(): | |
| 708 if f.startswith(root_path): | |
| 709 f = variable + f[len(root_path):] | |
| 710 break | |
| 711 | |
| 712 # Now strips off known files we want to ignore and to any specific mangling | |
| 713 # as necessary. It's easier to do it here than generate a blacklist. | |
| 714 match = EXECUTABLE.match(f) | |
| 715 if match: | |
| 716 return match.group(1) + '<(EXECUTABLE_SUFFIX)' | |
| 717 | |
| 718 # Blacklist logs and 'First Run' in the PRODUCT_DIR. First Run is not | |
| 719 # created by the compile, but by the test itself. | |
| 720 if LOG_FILE.match(f) or f == '<(PRODUCT_DIR)/First Run': | |
| 721 return None | |
| 722 | |
| 723 if sys.platform == 'darwin': | |
| 724 # On OSX, the name of the output is dependent on gyp define, it can be | |
| 725 # 'Google Chrome.app' or 'Chromium.app', same for 'XXX | |
| 726 # Framework.framework'. Furthermore, they are versioned with a gyp | |
| 727 # variable. To lower the complexity of the .isolate file, remove all the | |
| 728 # individual entries that show up under any of the 4 entries and replace | |
| 729 # them with the directory itself. Overall, this results in a bit more | |
| 730 # files than strictly necessary. | |
| 731 OSX_BUNDLES = ( | |
| 732 '<(PRODUCT_DIR)/Chromium Framework.framework/', | |
| 733 '<(PRODUCT_DIR)/Chromium.app/', | |
| 734 '<(PRODUCT_DIR)/Google Chrome Framework.framework/', | |
| 735 '<(PRODUCT_DIR)/Google Chrome.app/', | |
| 736 ) | |
| 737 for prefix in OSX_BUNDLES: | |
| 738 if f.startswith(prefix): | |
| 739 # Note this result in duplicate values, so the a set() must be used to | |
| 740 # remove duplicates. | |
| 741 return prefix | |
| 742 | |
| 743 return f | |
| 744 | |
| 745 tracked = set(filter(None, (fix(f.path) for f in tracked))) | |
| 746 untracked = set(filter(None, (fix(f.path) for f in untracked))) | |
| 747 touched = set(filter(None, (fix(f.path) for f in touched))) | |
| 748 out = classify_files(root_dir, tracked, untracked) | |
| 749 if touched: | |
| 750 out[KEY_TOUCHED] = sorted(touched) | |
| 751 return out | |
| 752 | |
| 753 | |
| 754 def generate_isolate( | |
| 755 tracked, untracked, touched, root_dir, variables, relative_cwd): | |
| 756 """Generates a clean and complete .isolate file.""" | |
| 757 result = generate_simplified( | |
| 758 tracked, untracked, touched, root_dir, variables, relative_cwd) | |
| 759 return { | |
| 760 'conditions': [ | |
| 761 ['OS=="%s"' % get_flavor(), { | |
| 762 'variables': result, | |
| 763 }], | |
| 764 ], | |
| 765 } | |
| 766 | |
| 767 | |
| 768 def split_touched(files): | |
| 769 """Splits files that are touched vs files that are read.""" | |
| 770 tracked = [] | |
| 771 touched = [] | |
| 772 for f in files: | |
| 773 if f.size: | |
| 774 tracked.append(f) | |
| 775 else: | |
| 776 touched.append(f) | |
| 777 return tracked, touched | |
| 778 | |
| 779 | |
| 780 def pretty_print(variables, stdout): | |
| 781 """Outputs a gyp compatible list from the decoded variables. | |
| 782 | |
| 783 Similar to pprint.print() but with NIH syndrome. | |
| 784 """ | |
| 785 # Order the dictionary keys by these keys in priority. | |
| 786 ORDER = ( | |
| 787 'variables', 'condition', 'command', 'relative_cwd', 'read_only', | |
| 788 KEY_TRACKED, KEY_UNTRACKED) | |
| 789 | |
| 790 def sorting_key(x): | |
| 791 """Gives priority to 'most important' keys before the others.""" | |
| 792 if x in ORDER: | |
| 793 return str(ORDER.index(x)) | |
| 794 return x | |
| 795 | |
| 796 def loop_list(indent, items): | |
| 797 for item in items: | |
| 798 if isinstance(item, basestring): | |
| 799 stdout.write('%s\'%s\',\n' % (indent, item)) | |
| 800 elif isinstance(item, dict): | |
| 801 stdout.write('%s{\n' % indent) | |
| 802 loop_dict(indent + ' ', item) | |
| 803 stdout.write('%s},\n' % indent) | |
| 804 elif isinstance(item, list): | |
| 805 # A list inside a list will write the first item embedded. | |
| 806 stdout.write('%s[' % indent) | |
| 807 for index, i in enumerate(item): | |
| 808 if isinstance(i, basestring): | |
| 809 stdout.write( | |
| 810 '\'%s\', ' % i.replace('\\', '\\\\').replace('\'', '\\\'')) | |
| 811 elif isinstance(i, dict): | |
| 812 stdout.write('{\n') | |
| 813 loop_dict(indent + ' ', i) | |
| 814 if index != len(item) - 1: | |
| 815 x = ', ' | |
| 816 else: | |
| 817 x = '' | |
| 818 stdout.write('%s}%s' % (indent, x)) | |
| 819 else: | |
| 820 assert False | |
| 821 stdout.write('],\n') | |
| 822 else: | |
| 823 assert False | |
| 824 | |
| 825 def loop_dict(indent, items): | |
| 826 for key in sorted(items, key=sorting_key): | |
| 827 item = items[key] | |
| 828 stdout.write("%s'%s': " % (indent, key)) | |
| 829 if isinstance(item, dict): | |
| 830 stdout.write('{\n') | |
| 831 loop_dict(indent + ' ', item) | |
| 832 stdout.write(indent + '},\n') | |
| 833 elif isinstance(item, list): | |
| 834 stdout.write('[\n') | |
| 835 loop_list(indent + ' ', item) | |
| 836 stdout.write(indent + '],\n') | |
| 837 elif isinstance(item, basestring): | |
| 838 stdout.write( | |
| 839 '\'%s\',\n' % item.replace('\\', '\\\\').replace('\'', '\\\'')) | |
| 840 elif item in (True, False, None): | |
| 841 stdout.write('%s\n' % item) | |
| 842 else: | |
| 843 assert False, item | |
| 844 | |
| 845 stdout.write('{\n') | |
| 846 loop_dict(' ', variables) | |
| 847 stdout.write('}\n') | |
| 848 | |
| 849 | |
| 850 def union(lhs, rhs): | |
| 851 """Merges two compatible datastructures composed of dict/list/set.""" | |
| 852 assert lhs is not None or rhs is not None | |
| 853 if lhs is None: | |
| 854 return copy.deepcopy(rhs) | |
| 855 if rhs is None: | |
| 856 return copy.deepcopy(lhs) | |
| 857 assert type(lhs) == type(rhs), (lhs, rhs) | |
| 858 if hasattr(lhs, 'union'): | |
| 859 # Includes set, OSSettings and Configs. | |
| 860 return lhs.union(rhs) | |
| 861 if isinstance(lhs, dict): | |
| 862 return dict((k, union(lhs.get(k), rhs.get(k))) for k in set(lhs).union(rhs)) | |
| 863 elif isinstance(lhs, list): | |
| 864 # Do not go inside the list. | |
| 865 return lhs + rhs | |
| 866 assert False, type(lhs) | |
| 867 | |
| 868 | |
| 869 def extract_comment(content): | |
| 870 """Extracts file level comment.""" | |
| 871 out = [] | |
| 872 for line in content.splitlines(True): | |
| 873 if line.startswith('#'): | |
| 874 out.append(line) | |
| 875 else: | |
| 876 break | |
| 877 return ''.join(out) | |
| 878 | |
| 879 | |
| 880 def eval_content(content): | |
| 881 """Evaluates a python file and return the value defined in it. | |
| 882 | |
| 883 Used in practice for .isolate files. | |
| 884 """ | |
| 885 globs = {'__builtins__': None} | |
| 886 locs = {} | |
| 887 value = eval(content, globs, locs) | |
| 888 assert locs == {}, locs | |
| 889 assert globs == {'__builtins__': None}, globs | |
| 890 return value | |
| 891 | |
| 892 | |
| 893 def verify_variables(variables): | |
| 894 """Verifies the |variables| dictionary is in the expected format.""" | |
| 895 VALID_VARIABLES = [ | |
| 896 KEY_TOUCHED, | |
| 897 KEY_TRACKED, | |
| 898 KEY_UNTRACKED, | |
| 899 'command', | |
| 900 'read_only', | |
| 901 ] | |
| 902 assert isinstance(variables, dict), variables | |
| 903 assert set(VALID_VARIABLES).issuperset(set(variables)), variables.keys() | |
| 904 for name, value in variables.iteritems(): | |
| 905 if name == 'read_only': | |
| 906 assert value in (True, False, None), value | |
| 907 else: | |
| 908 assert isinstance(value, list), value | |
| 909 assert all(isinstance(i, basestring) for i in value), value | |
| 910 | |
| 911 | |
| 912 def verify_condition(condition): | |
| 913 """Verifies the |condition| dictionary is in the expected format.""" | |
| 914 VALID_INSIDE_CONDITION = ['variables'] | |
| 915 assert isinstance(condition, list), condition | |
| 916 assert 2 <= len(condition) <= 3, condition | |
| 917 assert re.match(r'OS==\"([a-z]+)\"', condition[0]), condition[0] | |
| 918 for c in condition[1:]: | |
| 919 assert isinstance(c, dict), c | |
| 920 assert set(VALID_INSIDE_CONDITION).issuperset(set(c)), c.keys() | |
| 921 verify_variables(c.get('variables', {})) | |
| 922 | |
| 923 | |
| 924 def verify_root(value): | |
| 925 VALID_ROOTS = ['variables', 'conditions'] | |
| 926 assert isinstance(value, dict), value | |
| 927 assert set(VALID_ROOTS).issuperset(set(value)), value.keys() | |
| 928 verify_variables(value.get('variables', {})) | |
| 929 | |
| 930 conditions = value.get('conditions', []) | |
| 931 assert isinstance(conditions, list), conditions | |
| 932 for condition in conditions: | |
| 933 verify_condition(condition) | |
| 934 | |
| 935 | |
| 936 def remove_weak_dependencies(values, key, item, item_oses): | |
| 937 """Remove any oses from this key if the item is already under a strong key.""" | |
| 938 if key == KEY_TOUCHED: | |
| 939 for stronger_key in (KEY_TRACKED, KEY_UNTRACKED): | |
| 940 oses = values.get(stronger_key, {}).get(item, None) | |
| 941 if oses: | |
| 942 item_oses -= oses | |
| 943 | |
| 944 return item_oses | |
| 945 | |
| 946 | |
| 947 def invert_map(variables): | |
| 948 """Converts a dict(OS, dict(deptype, list(dependencies)) to a flattened view. | |
| 949 | |
| 950 Returns a tuple of: | |
| 951 1. dict(deptype, dict(dependency, set(OSes)) for easier processing. | |
| 952 2. All the OSes found as a set. | |
| 953 """ | |
| 954 KEYS = ( | |
| 955 KEY_TOUCHED, | |
| 956 KEY_TRACKED, | |
| 957 KEY_UNTRACKED, | |
| 958 'command', | |
| 959 'read_only', | |
| 960 ) | |
| 961 out = dict((key, {}) for key in KEYS) | |
| 962 for os_name, values in variables.iteritems(): | |
| 963 for key in (KEY_TOUCHED, KEY_TRACKED, KEY_UNTRACKED): | |
| 964 for item in values.get(key, []): | |
| 965 out[key].setdefault(item, set()).add(os_name) | |
| 966 | |
| 967 # command needs special handling. | |
| 968 command = tuple(values.get('command', [])) | |
| 969 out['command'].setdefault(command, set()).add(os_name) | |
| 970 | |
| 971 # read_only needs special handling. | |
| 972 out['read_only'].setdefault(values.get('read_only'), set()).add(os_name) | |
| 973 return out, set(variables) | |
| 974 | |
| 975 | |
| 976 def reduce_inputs(values, oses): | |
| 977 """Reduces the invert_map() output to the strictest minimum list. | |
| 978 | |
| 979 1. Construct the inverse map first. | |
| 980 2. Look at each individual file and directory, map where they are used and | |
| 981 reconstruct the inverse dictionary. | |
| 982 3. Do not convert back to negative if only 2 OSes were merged. | |
| 983 | |
| 984 Returns a tuple of: | |
| 985 1. the minimized dictionary | |
| 986 2. oses passed through as-is. | |
| 987 """ | |
| 988 KEYS = ( | |
| 989 KEY_TOUCHED, | |
| 990 KEY_TRACKED, | |
| 991 KEY_UNTRACKED, | |
| 992 'command', | |
| 993 'read_only', | |
| 994 ) | |
| 995 out = dict((key, {}) for key in KEYS) | |
| 996 assert all(oses), oses | |
| 997 if len(oses) > 2: | |
| 998 for key in KEYS: | |
| 999 for item, item_oses in values.get(key, {}).iteritems(): | |
| 1000 item_oses = remove_weak_dependencies(values, key, item, item_oses) | |
| 1001 if not item_oses: | |
| 1002 continue | |
| 1003 | |
| 1004 # Converts all oses.difference('foo') to '!foo'. | |
| 1005 assert all(item_oses), item_oses | |
| 1006 missing = oses.difference(item_oses) | |
| 1007 if len(missing) == 1: | |
| 1008 # Replace it with a negative. | |
| 1009 out[key][item] = set(['!' + tuple(missing)[0]]) | |
| 1010 elif not missing: | |
| 1011 out[key][item] = set([None]) | |
| 1012 else: | |
| 1013 out[key][item] = set(item_oses) | |
| 1014 else: | |
| 1015 for key in KEYS: | |
| 1016 for item, item_oses in values.get(key, {}).iteritems(): | |
| 1017 item_oses = remove_weak_dependencies(values, key, item, item_oses) | |
| 1018 if not item_oses: | |
| 1019 continue | |
| 1020 | |
| 1021 # Converts all oses.difference('foo') to '!foo'. | |
| 1022 assert None not in item_oses, item_oses | |
| 1023 out[key][item] = set(item_oses) | |
| 1024 return out, oses | |
| 1025 | |
| 1026 | |
| 1027 def convert_map_to_isolate_dict(values, oses): | |
| 1028 """Regenerates back a .isolate configuration dict from files and dirs | |
| 1029 mappings generated from reduce_inputs(). | |
| 1030 """ | |
| 1031 # First, inverse the mapping to make it dict first. | |
| 1032 config = {} | |
| 1033 for key in values: | |
| 1034 for item, oses in values[key].iteritems(): | |
| 1035 if item is None: | |
| 1036 # For read_only default. | |
| 1037 continue | |
| 1038 for cond_os in oses: | |
| 1039 cond_key = None if cond_os is None else cond_os.lstrip('!') | |
| 1040 # Insert the if/else dicts. | |
| 1041 condition_values = config.setdefault(cond_key, [{}, {}]) | |
| 1042 # If condition is negative, use index 1, else use index 0. | |
| 1043 cond_value = condition_values[int((cond_os or '').startswith('!'))] | |
| 1044 variables = cond_value.setdefault('variables', {}) | |
| 1045 | |
| 1046 if item in (True, False): | |
| 1047 # One-off for read_only. | |
| 1048 variables[key] = item | |
| 1049 else: | |
| 1050 if isinstance(item, tuple) and item: | |
| 1051 # One-off for command. | |
| 1052 # Do not merge lists and do not sort! | |
| 1053 # Note that item is a tuple. | |
| 1054 assert key not in variables | |
| 1055 variables[key] = list(item) | |
| 1056 elif item: | |
| 1057 # The list of items (files or dirs). Append the new item and keep | |
| 1058 # the list sorted. | |
| 1059 l = variables.setdefault(key, []) | |
| 1060 l.append(item) | |
| 1061 l.sort() | |
| 1062 | |
| 1063 out = {} | |
| 1064 for o in sorted(config): | |
| 1065 d = config[o] | |
| 1066 if o is None: | |
| 1067 assert not d[1] | |
| 1068 out = union(out, d[0]) | |
| 1069 else: | |
| 1070 c = out.setdefault('conditions', []) | |
| 1071 if d[1]: | |
| 1072 c.append(['OS=="%s"' % o] + d) | |
| 1073 else: | |
| 1074 c.append(['OS=="%s"' % o] + d[0:1]) | |
| 1075 return out | |
| 1076 | |
| 1077 | |
| 1078 ### Internal state files. | |
| 1079 | |
| 1080 | |
| 1081 class OSSettings(object): | |
| 1082 """Represents the dependencies for an OS. The structure is immutable. | |
| 1083 | |
| 1084 It's the .isolate settings for a specific file. | |
| 1085 """ | |
| 1086 def __init__(self, name, values): | |
| 1087 self.name = name | |
| 1088 verify_variables(values) | |
| 1089 self.touched = sorted(values.get(KEY_TOUCHED, [])) | |
| 1090 self.tracked = sorted(values.get(KEY_TRACKED, [])) | |
| 1091 self.untracked = sorted(values.get(KEY_UNTRACKED, [])) | |
| 1092 self.command = values.get('command', [])[:] | |
| 1093 self.read_only = values.get('read_only') | |
| 1094 | |
| 1095 def union(self, rhs): | |
| 1096 assert self.name == rhs.name | |
| 1097 assert not (self.command and rhs.command) | |
| 1098 var = { | |
| 1099 KEY_TOUCHED: sorted(self.touched + rhs.touched), | |
| 1100 KEY_TRACKED: sorted(self.tracked + rhs.tracked), | |
| 1101 KEY_UNTRACKED: sorted(self.untracked + rhs.untracked), | |
| 1102 'command': self.command or rhs.command, | |
| 1103 'read_only': rhs.read_only if self.read_only is None else self.read_only, | |
| 1104 } | |
| 1105 return OSSettings(self.name, var) | |
| 1106 | |
| 1107 def flatten(self): | |
| 1108 out = {} | |
| 1109 if self.command: | |
| 1110 out['command'] = self.command | |
| 1111 if self.touched: | |
| 1112 out[KEY_TOUCHED] = self.touched | |
| 1113 if self.tracked: | |
| 1114 out[KEY_TRACKED] = self.tracked | |
| 1115 if self.untracked: | |
| 1116 out[KEY_UNTRACKED] = self.untracked | |
| 1117 if self.read_only is not None: | |
| 1118 out['read_only'] = self.read_only | |
| 1119 return out | |
| 1120 | |
| 1121 | |
| 1122 class Configs(object): | |
| 1123 """Represents a processed .isolate file. | |
| 1124 | |
| 1125 Stores the file in a processed way, split by each the OS-specific | |
| 1126 configurations. | |
| 1127 | |
| 1128 The self.per_os[None] member contains all the 'else' clauses plus the default | |
| 1129 values. It is not included in the flatten() result. | |
| 1130 """ | |
| 1131 def __init__(self, oses, file_comment): | |
| 1132 self.file_comment = file_comment | |
| 1133 self.per_os = { | |
| 1134 None: OSSettings(None, {}), | |
| 1135 } | |
| 1136 self.per_os.update(dict((name, OSSettings(name, {})) for name in oses)) | |
| 1137 | |
| 1138 def union(self, rhs): | |
| 1139 items = list(set(self.per_os.keys() + rhs.per_os.keys())) | |
| 1140 # Takes the first file comment, prefering lhs. | |
| 1141 out = Configs(items, self.file_comment or rhs.file_comment) | |
| 1142 for key in items: | |
| 1143 out.per_os[key] = union(self.per_os.get(key), rhs.per_os.get(key)) | |
| 1144 return out | |
| 1145 | |
| 1146 def add_globals(self, values): | |
| 1147 for key in self.per_os: | |
| 1148 self.per_os[key] = self.per_os[key].union(OSSettings(key, values)) | |
| 1149 | |
| 1150 def add_values(self, for_os, values): | |
| 1151 self.per_os[for_os] = self.per_os[for_os].union(OSSettings(for_os, values)) | |
| 1152 | |
| 1153 def add_negative_values(self, for_os, values): | |
| 1154 """Includes the variables to all OSes except |for_os|. | |
| 1155 | |
| 1156 This includes 'None' so unknown OSes gets it too. | |
| 1157 """ | |
| 1158 for key in self.per_os: | |
| 1159 if key != for_os: | |
| 1160 self.per_os[key] = self.per_os[key].union(OSSettings(key, values)) | |
| 1161 | |
| 1162 def flatten(self): | |
| 1163 """Returns a flat dictionary representation of the configuration. | |
| 1164 | |
| 1165 Skips None pseudo-OS. | |
| 1166 """ | |
| 1167 return dict( | |
| 1168 (k, v.flatten()) for k, v in self.per_os.iteritems() if k is not None) | |
| 1169 | |
| 1170 | |
| 1171 def load_isolate_as_config(value, file_comment, default_oses): | |
| 1172 """Parses one .isolate file and returns a Configs() instance. | |
| 1173 | |
| 1174 |value| is the loaded dictionary that was defined in the gyp file. | |
| 1175 | |
| 1176 The expected format is strict, anything diverting from the format below will | |
| 1177 throw an assert: | |
| 1178 { | |
| 1179 'variables': { | |
| 1180 'command': [ | |
| 1181 ... | |
| 1182 ], | |
| 1183 'isolate_dependency_tracked': [ | |
| 1184 ... | |
| 1185 ], | |
| 1186 'isolate_dependency_untracked': [ | |
| 1187 ... | |
| 1188 ], | |
| 1189 'read_only': False, | |
| 1190 }, | |
| 1191 'conditions': [ | |
| 1192 ['OS=="<os>"', { | |
| 1193 'variables': { | |
| 1194 ... | |
| 1195 }, | |
| 1196 }, { # else | |
| 1197 'variables': { | |
| 1198 ... | |
| 1199 }, | |
| 1200 }], | |
| 1201 ... | |
| 1202 ], | |
| 1203 } | |
| 1204 """ | |
| 1205 verify_root(value) | |
| 1206 | |
| 1207 # Scan to get the list of OSes. | |
| 1208 conditions = value.get('conditions', []) | |
| 1209 oses = set(re.match(r'OS==\"([a-z]+)\"', c[0]).group(1) for c in conditions) | |
| 1210 oses = oses.union(default_oses) | |
| 1211 configs = Configs(oses, file_comment) | |
| 1212 | |
| 1213 # Global level variables. | |
| 1214 configs.add_globals(value.get('variables', {})) | |
| 1215 | |
| 1216 # OS specific variables. | |
| 1217 for condition in conditions: | |
| 1218 condition_os = re.match(r'OS==\"([a-z]+)\"', condition[0]).group(1) | |
| 1219 configs.add_values(condition_os, condition[1].get('variables', {})) | |
| 1220 if len(condition) > 2: | |
| 1221 configs.add_negative_values( | |
| 1222 condition_os, condition[2].get('variables', {})) | |
| 1223 return configs | |
| 1224 | |
| 1225 | |
| 1226 def load_isolate_for_flavor(content, flavor): | |
| 1227 """Loads the .isolate file and returns the information unprocessed. | |
| 1228 | |
| 1229 Returns the command, dependencies and read_only flag. The dependencies are | |
| 1230 fixed to use os.path.sep. | |
| 1231 """ | |
| 1232 # Load the .isolate file, process its conditions, retrieve the command and | |
| 1233 # dependencies. | |
| 1234 configs = load_isolate_as_config(eval_content(content), None, DEFAULT_OSES) | |
| 1235 config = configs.per_os.get(flavor) or configs.per_os.get(None) | |
| 1236 if not config: | |
| 1237 raise ExecutionError('Failed to load configuration for \'%s\'' % flavor) | |
| 1238 # Merge tracked and untracked dependencies, isolate.py doesn't care about the | |
| 1239 # trackability of the dependencies, only the build tool does. | |
| 1240 dependencies = [ | |
| 1241 f.replace('/', os.path.sep) for f in config.tracked + config.untracked | |
| 1242 ] | |
| 1243 touched = [f.replace('/', os.path.sep) for f in config.touched] | |
| 1244 return config.command, dependencies, touched, config.read_only | |
| 1245 | |
| 1246 | |
| 1247 class Flattenable(object): | |
| 1248 """Represents data that can be represented as a json file.""" | |
| 1249 MEMBERS = () | |
| 1250 | |
| 1251 def flatten(self): | |
| 1252 """Returns a json-serializable version of itself. | |
| 1253 | |
| 1254 Skips None entries. | |
| 1255 """ | |
| 1256 items = ((member, getattr(self, member)) for member in self.MEMBERS) | |
| 1257 return dict((member, value) for member, value in items if value is not None) | |
| 1258 | |
| 1259 @classmethod | |
| 1260 def load(cls, data): | |
| 1261 """Loads a flattened version.""" | |
| 1262 data = data.copy() | |
| 1263 out = cls() | |
| 1264 for member in out.MEMBERS: | |
| 1265 if member in data: | |
| 1266 # Access to a protected member XXX of a client class | |
| 1267 # pylint: disable=W0212 | |
| 1268 out._load_member(member, data.pop(member)) | |
| 1269 if data: | |
| 1270 raise ValueError( | |
| 1271 'Found unexpected entry %s while constructing an object %s' % | |
| 1272 (data, cls.__name__), data, cls.__name__) | |
| 1273 return out | |
| 1274 | |
| 1275 def _load_member(self, member, value): | |
| 1276 """Loads a member into self.""" | |
| 1277 setattr(self, member, value) | |
| 1278 | |
| 1279 @classmethod | |
| 1280 def load_file(cls, filename): | |
| 1281 """Loads the data from a file or return an empty instance.""" | |
| 1282 out = cls() | |
| 1283 try: | |
| 1284 out = cls.load(trace_inputs.read_json(filename)) | |
| 1285 logging.debug('Loaded %s(%s)' % (cls.__name__, filename)) | |
| 1286 except (IOError, ValueError): | |
| 1287 logging.warn('Failed to load %s' % filename) | |
| 1288 return out | |
| 1289 | |
| 1290 | |
| 1291 class Result(Flattenable): | |
| 1292 """Describes the content of a .result file. | |
| 1293 | |
| 1294 This file is used by run_test_from_archive.py so its content is strictly only | |
| 1295 what is necessary to run the test outside of a checkout. | |
| 1296 | |
| 1297 It is important to note that the 'files' dict keys are using native OS path | |
| 1298 separator instead of '/' used in .isolate file. | |
| 1299 """ | |
| 1300 MEMBERS = ( | |
| 1301 'command', | |
| 1302 'files', | |
| 1303 'os', | |
| 1304 'read_only', | |
| 1305 'relative_cwd', | |
| 1306 ) | |
| 1307 | |
| 1308 os = get_flavor() | |
| 1309 | |
| 1310 def __init__(self): | |
| 1311 super(Result, self).__init__() | |
| 1312 self.command = [] | |
| 1313 self.files = {} | |
| 1314 self.read_only = None | |
| 1315 self.relative_cwd = None | |
| 1316 | |
| 1317 def update(self, command, infiles, touched, read_only, relative_cwd): | |
| 1318 """Updates the result state with new information.""" | |
| 1319 self.command = command | |
| 1320 # Add new files. | |
| 1321 for f in infiles: | |
| 1322 self.files.setdefault(f, {}) | |
| 1323 for f in touched: | |
| 1324 self.files.setdefault(f, {})['touched_only'] = True | |
| 1325 # Prune extraneous files that are not a dependency anymore. | |
| 1326 for f in set(self.files).difference(set(infiles).union(touched)): | |
| 1327 del self.files[f] | |
| 1328 if read_only is not None: | |
| 1329 self.read_only = read_only | |
| 1330 self.relative_cwd = relative_cwd | |
| 1331 | |
| 1332 def _load_member(self, member, value): | |
| 1333 if member == 'os': | |
| 1334 if value != self.os: | |
| 1335 raise run_test_from_archive.ConfigError( | |
| 1336 'The .results file was created on another platform') | |
| 1337 else: | |
| 1338 super(Result, self)._load_member(member, value) | |
| 1339 | |
| 1340 def __str__(self): | |
| 1341 out = '%s(\n' % self.__class__.__name__ | |
| 1342 out += ' command: %s\n' % self.command | |
| 1343 out += ' files: %d\n' % len(self.files) | |
| 1344 out += ' read_only: %s\n' % self.read_only | |
| 1345 out += ' relative_cwd: %s)' % self.relative_cwd | |
| 1346 return out | |
| 1347 | |
| 1348 | |
| 1349 class SavedState(Flattenable): | |
| 1350 """Describes the content of a .state file. | |
| 1351 | |
| 1352 The items in this file are simply to improve the developer's life and aren't | |
| 1353 used by run_test_from_archive.py. This file can always be safely removed. | |
| 1354 | |
| 1355 isolate_file permits to find back root_dir, variables are used for stateful | |
| 1356 rerun. | |
| 1357 """ | |
| 1358 MEMBERS = ( | |
| 1359 'isolate_file', | |
| 1360 'variables', | |
| 1361 ) | |
| 1362 | |
| 1363 def __init__(self): | |
| 1364 super(SavedState, self).__init__() | |
| 1365 self.isolate_file = None | |
| 1366 self.variables = {} | |
| 1367 | |
| 1368 def update(self, isolate_file, variables): | |
| 1369 """Updates the saved state with new information.""" | |
| 1370 self.isolate_file = isolate_file | |
| 1371 self.variables.update(variables) | |
| 1372 | |
| 1373 @classmethod | |
| 1374 def load(cls, data): | |
| 1375 out = super(SavedState, cls).load(data) | |
| 1376 if out.isolate_file: | |
| 1377 out.isolate_file = trace_inputs.get_native_path_case(out.isolate_file) | |
| 1378 return out | |
| 1379 | |
| 1380 def __str__(self): | |
| 1381 out = '%s(\n' % self.__class__.__name__ | |
| 1382 out += ' isolate_file: %s\n' % self.isolate_file | |
| 1383 out += ' variables: %s' % ''.join( | |
| 1384 '\n %s=%s' % (k, self.variables[k]) for k in sorted(self.variables)) | |
| 1385 out += ')' | |
| 1386 return out | |
| 1387 | |
| 1388 | |
| 1389 class CompleteState(object): | |
| 1390 """Contains all the state to run the task at hand.""" | |
| 1391 def __init__(self, result_file, result, saved_state): | |
| 1392 super(CompleteState, self).__init__() | |
| 1393 self.result_file = result_file | |
| 1394 # Contains the data that will be used by run_test_from_archive.py | |
| 1395 self.result = result | |
| 1396 # Contains the data to ease developer's use-case but that is not strictly | |
| 1397 # necessary. | |
| 1398 self.saved_state = saved_state | |
| 1399 | |
| 1400 @classmethod | |
| 1401 def load_files(cls, result_file): | |
| 1402 """Loads state from disk.""" | |
| 1403 assert os.path.isabs(result_file), result_file | |
| 1404 return cls( | |
| 1405 result_file, | |
| 1406 Result.load_file(result_file), | |
| 1407 SavedState.load_file(result_to_state(result_file))) | |
| 1408 | |
| 1409 def load_isolate(self, isolate_file, variables): | |
| 1410 """Updates self.result and self.saved_state with information loaded from a | |
| 1411 .isolate file. | |
| 1412 | |
| 1413 Processes the loaded data, deduce root_dir, relative_cwd. | |
| 1414 """ | |
| 1415 # Make sure to not depend on os.getcwd(). | |
| 1416 assert os.path.isabs(isolate_file), isolate_file | |
| 1417 logging.info( | |
| 1418 'CompleteState.load_isolate(%s, %s)' % (isolate_file, variables)) | |
| 1419 relative_base_dir = os.path.dirname(isolate_file) | |
| 1420 | |
| 1421 # Processes the variables and update the saved state. | |
| 1422 variables = process_variables(variables, relative_base_dir) | |
| 1423 self.saved_state.update(isolate_file, variables) | |
| 1424 | |
| 1425 with open(isolate_file, 'r') as f: | |
| 1426 # At that point, variables are not replaced yet in command and infiles. | |
| 1427 # infiles may contain directory entries and is in posix style. | |
| 1428 command, infiles, touched, read_only = load_isolate_for_flavor( | |
| 1429 f.read(), get_flavor()) | |
| 1430 command = [eval_variables(i, self.saved_state.variables) for i in command] | |
| 1431 infiles = [eval_variables(f, self.saved_state.variables) for f in infiles] | |
| 1432 touched = [eval_variables(f, self.saved_state.variables) for f in touched] | |
| 1433 # root_dir is automatically determined by the deepest root accessed with the | |
| 1434 # form '../../foo/bar'. | |
| 1435 root_dir = determine_root_dir(relative_base_dir, infiles + touched) | |
| 1436 # The relative directory is automatically determined by the relative path | |
| 1437 # between root_dir and the directory containing the .isolate file, | |
| 1438 # isolate_base_dir. | |
| 1439 relative_cwd = os.path.relpath(relative_base_dir, root_dir) | |
| 1440 # Normalize the files based to root_dir. It is important to keep the | |
| 1441 # trailing os.path.sep at that step. | |
| 1442 infiles = [ | |
| 1443 relpath(normpath(os.path.join(relative_base_dir, f)), root_dir) | |
| 1444 for f in infiles | |
| 1445 ] | |
| 1446 touched = [ | |
| 1447 relpath(normpath(os.path.join(relative_base_dir, f)), root_dir) | |
| 1448 for f in touched | |
| 1449 ] | |
| 1450 # Expand the directories by listing each file inside. Up to now, trailing | |
| 1451 # os.path.sep must be kept. Do not expand 'touched'. | |
| 1452 infiles = expand_directories_and_symlinks( | |
| 1453 root_dir, | |
| 1454 infiles, | |
| 1455 lambda x: re.match(r'.*\.(git|svn|pyc)$', x)) | |
| 1456 | |
| 1457 # Finally, update the new stuff in the foo.result file, the file that is | |
| 1458 # used by run_test_from_archive.py. | |
| 1459 self.result.update(command, infiles, touched, read_only, relative_cwd) | |
| 1460 logging.debug(self) | |
| 1461 | |
| 1462 def process_inputs(self, level): | |
| 1463 """Updates self.result.files with the files' mode and hash. | |
| 1464 | |
| 1465 See process_input() for more information. | |
| 1466 """ | |
| 1467 for infile in sorted(self.result.files): | |
| 1468 filepath = os.path.join(self.root_dir, infile) | |
| 1469 self.result.files[infile] = process_input( | |
| 1470 filepath, self.result.files[infile], level, self.result.read_only) | |
| 1471 | |
| 1472 def save_files(self): | |
| 1473 """Saves both self.result and self.saved_state.""" | |
| 1474 logging.debug('Dumping to %s' % self.result_file) | |
| 1475 trace_inputs.write_json(self.result_file, self.result.flatten(), True) | |
| 1476 total_bytes = sum(i.get('size', 0) for i in self.result.files.itervalues()) | |
| 1477 if total_bytes: | |
| 1478 logging.debug('Total size: %d bytes' % total_bytes) | |
| 1479 saved_state_file = result_to_state(self.result_file) | |
| 1480 logging.debug('Dumping to %s' % saved_state_file) | |
| 1481 trace_inputs.write_json(saved_state_file, self.saved_state.flatten(), True) | |
| 1482 | |
| 1483 @property | |
| 1484 def root_dir(self): | |
| 1485 """isolate_file is always inside relative_cwd relative to root_dir.""" | |
| 1486 isolate_dir = os.path.dirname(self.saved_state.isolate_file) | |
| 1487 # Special case '.'. | |
| 1488 if self.result.relative_cwd == '.': | |
| 1489 return isolate_dir | |
| 1490 assert isolate_dir.endswith(self.result.relative_cwd), ( | |
| 1491 isolate_dir, self.result.relative_cwd) | |
| 1492 return isolate_dir[:-(len(self.result.relative_cwd) + 1)] | |
| 1493 | |
| 1494 @property | |
| 1495 def resultdir(self): | |
| 1496 """Directory containing the results, usually equivalent to the variable | |
| 1497 PRODUCT_DIR. | |
| 1498 """ | |
| 1499 return os.path.dirname(self.result_file) | |
| 1500 | |
| 1501 def __str__(self): | |
| 1502 def indent(data, indent_length): | |
| 1503 """Indents text.""" | |
| 1504 spacing = ' ' * indent_length | |
| 1505 return ''.join(spacing + l for l in str(data).splitlines(True)) | |
| 1506 | |
| 1507 out = '%s(\n' % self.__class__.__name__ | |
| 1508 out += ' root_dir: %s\n' % self.root_dir | |
| 1509 out += ' result: %s\n' % indent(self.result, 2) | |
| 1510 out += ' saved_state: %s)' % indent(self.saved_state, 2) | |
| 1511 return out | |
| 1512 | |
| 1513 | |
| 1514 def load_complete_state(options, level): | |
| 1515 """Loads a CompleteState. | |
| 1516 | |
| 1517 This includes data from .isolate, .result and .state files. | |
| 1518 | |
| 1519 Arguments: | |
| 1520 options: Options instance generated with OptionParserIsolate. | |
| 1521 level: Amount of data to fetch. | |
| 1522 """ | |
| 1523 if options.result: | |
| 1524 # Load the previous state if it was present. Namely, "foo.result" and | |
| 1525 # "foo.state". | |
| 1526 complete_state = CompleteState.load_files(options.result) | |
| 1527 else: | |
| 1528 # Constructs a dummy object that cannot be saved. Useful for temporary | |
| 1529 # commands like 'run'. | |
| 1530 complete_state = CompleteState(None, Result(), SavedState()) | |
| 1531 options.isolate = options.isolate or complete_state.saved_state.isolate_file | |
| 1532 if not options.isolate: | |
| 1533 raise ExecutionError('A .isolate file is required.') | |
| 1534 if (complete_state.saved_state.isolate_file and | |
| 1535 options.isolate != complete_state.saved_state.isolate_file): | |
| 1536 raise ExecutionError( | |
| 1537 '%s and %s do not match.' % ( | |
| 1538 options.isolate, complete_state.saved_state.isolate_file)) | |
| 1539 | |
| 1540 # Then load the .isolate and expands directories. | |
| 1541 complete_state.load_isolate(options.isolate, options.variables) | |
| 1542 | |
| 1543 # Regenerate complete_state.result.files. | |
| 1544 complete_state.process_inputs(level) | |
| 1545 return complete_state | |
| 1546 | |
| 1547 | |
| 1548 def read_trace_as_isolate_dict(complete_state): | |
| 1549 """Reads a trace and returns the .isolate dictionary.""" | |
| 1550 api = trace_inputs.get_api() | |
| 1551 logfile = complete_state.result_file + '.log' | |
| 1552 if not os.path.isfile(logfile): | |
| 1553 raise ExecutionError( | |
| 1554 'No log file \'%s\' to read, did you forget to \'trace\'?' % logfile) | |
| 1555 try: | |
| 1556 results = trace_inputs.load_trace( | |
| 1557 logfile, complete_state.root_dir, api, default_blacklist) | |
| 1558 tracked, touched = split_touched(results.existent) | |
| 1559 value = generate_isolate( | |
| 1560 tracked, | |
| 1561 [], | |
| 1562 touched, | |
| 1563 complete_state.root_dir, | |
| 1564 complete_state.saved_state.variables, | |
| 1565 complete_state.result.relative_cwd) | |
| 1566 return value | |
| 1567 except trace_inputs.TracingFailure, e: | |
| 1568 raise ExecutionError( | |
| 1569 'Reading traces failed for: %s\n%s' % | |
| 1570 (' '.join(complete_state.result.command), str(e))) | |
| 1571 | |
| 1572 | |
| 1573 def print_all(comment, data, stream): | |
| 1574 """Prints a complete .isolate file and its top-level file comment into a | |
| 1575 stream. | |
| 1576 """ | |
| 1577 if comment: | |
| 1578 stream.write(comment) | |
| 1579 pretty_print(data, stream) | |
| 1580 | |
| 1581 | |
| 1582 def merge(complete_state): | |
| 1583 """Reads a trace and merges it back into the source .isolate file.""" | |
| 1584 value = read_trace_as_isolate_dict(complete_state) | |
| 1585 | |
| 1586 # Now take that data and union it into the original .isolate file. | |
| 1587 with open(complete_state.saved_state.isolate_file, 'r') as f: | |
| 1588 prev_content = f.read() | |
| 1589 prev_config = load_isolate_as_config( | |
| 1590 eval_content(prev_content), | |
| 1591 extract_comment(prev_content), | |
| 1592 DEFAULT_OSES) | |
| 1593 new_config = load_isolate_as_config(value, '', DEFAULT_OSES) | |
| 1594 config = union(prev_config, new_config) | |
| 1595 # pylint: disable=E1103 | |
| 1596 data = convert_map_to_isolate_dict( | |
| 1597 *reduce_inputs(*invert_map(config.flatten()))) | |
| 1598 print 'Updating %s' % complete_state.saved_state.isolate_file | |
| 1599 with open(complete_state.saved_state.isolate_file, 'wb') as f: | |
| 1600 print_all(config.file_comment, data, f) | |
| 1601 | |
| 1602 | |
| 1603 def CMDcheck(args): | |
| 1604 """Checks that all the inputs are present and update .result.""" | |
| 1605 parser = OptionParserIsolate(command='check') | |
| 1606 options, _ = parser.parse_args(args) | |
| 1607 complete_state = load_complete_state(options, NO_INFO) | |
| 1608 | |
| 1609 # Nothing is done specifically. Just store the result and state. | |
| 1610 complete_state.save_files() | |
| 1611 return 0 | |
| 1612 | |
| 1613 | |
| 1614 def CMDhashtable(args): | |
| 1615 """Creates a hash table content addressed object store. | |
| 1616 | |
| 1617 All the files listed in the .result file are put in the output directory with | |
| 1618 the file name being the sha-1 of the file's content. | |
| 1619 """ | |
| 1620 parser = OptionParserIsolate(command='hashtable') | |
| 1621 options, _ = parser.parse_args(args) | |
| 1622 | |
| 1623 with run_test_from_archive.Profiler('GenerateHashtable'): | |
| 1624 success = False | |
| 1625 try: | |
| 1626 complete_state = load_complete_state(options, WITH_HASH) | |
| 1627 options.outdir = ( | |
| 1628 options.outdir or os.path.join(complete_state.resultdir, 'hashtable')) | |
| 1629 # Make sure that complete_state isn't modified until save_files() is | |
| 1630 # called, because any changes made to it here will propagate to the files | |
| 1631 # created (which is probably not intended). | |
| 1632 complete_state.save_files() | |
| 1633 | |
| 1634 logging.info('Creating content addressed object store with %d item', | |
| 1635 len(complete_state.result.files)) | |
| 1636 | |
| 1637 with open(complete_state.result_file, 'rb') as f: | |
| 1638 manifest_hash = hashlib.sha1(f.read()).hexdigest() | |
| 1639 manifest_metadata = {'sha-1': manifest_hash} | |
| 1640 | |
| 1641 infiles = complete_state.result.files | |
| 1642 infiles[complete_state.result_file] = manifest_metadata | |
| 1643 | |
| 1644 if re.match(r'^https?://.+$', options.outdir): | |
| 1645 upload_sha1_tree( | |
| 1646 base_url=options.outdir, | |
| 1647 indir=complete_state.root_dir, | |
| 1648 infiles=infiles) | |
| 1649 else: | |
| 1650 recreate_tree( | |
| 1651 outdir=options.outdir, | |
| 1652 indir=complete_state.root_dir, | |
| 1653 infiles=infiles, | |
| 1654 action=run_test_from_archive.HARDLINK, | |
| 1655 as_sha1=True) | |
| 1656 success = True | |
| 1657 finally: | |
| 1658 # If the command failed, delete the .results file if it exists. This is | |
| 1659 # important so no stale swarm job is executed. | |
| 1660 if not success and os.path.isfile(options.result): | |
| 1661 os.remove(options.result) | |
| 1662 | |
| 1663 | |
| 1664 def CMDnoop(args): | |
| 1665 """Touches --result but does nothing else. | |
| 1666 | |
| 1667 This mode is to help transition since some builders do not have all the test | |
| 1668 data files checked out. Touch result_file and exit silently. | |
| 1669 """ | |
| 1670 parser = OptionParserIsolate(command='noop') | |
| 1671 options, _ = parser.parse_args(args) | |
| 1672 # In particular, do not call load_complete_state(). | |
| 1673 open(options.result, 'a').close() | |
| 1674 return 0 | |
| 1675 | |
| 1676 | |
| 1677 def CMDmerge(args): | |
| 1678 """Reads and merges the data from the trace back into the original .isolate. | |
| 1679 | |
| 1680 Ignores --outdir. | |
| 1681 """ | |
| 1682 parser = OptionParserIsolate(command='merge', require_result=False) | |
| 1683 options, _ = parser.parse_args(args) | |
| 1684 complete_state = load_complete_state(options, NO_INFO) | |
| 1685 merge(complete_state) | |
| 1686 return 0 | |
| 1687 | |
| 1688 | |
| 1689 def CMDread(args): | |
| 1690 """Reads the trace file generated with command 'trace'. | |
| 1691 | |
| 1692 Ignores --outdir. | |
| 1693 """ | |
| 1694 parser = OptionParserIsolate(command='read', require_result=False) | |
| 1695 options, _ = parser.parse_args(args) | |
| 1696 complete_state = load_complete_state(options, NO_INFO) | |
| 1697 value = read_trace_as_isolate_dict(complete_state) | |
| 1698 pretty_print(value, sys.stdout) | |
| 1699 return 0 | |
| 1700 | |
| 1701 | |
| 1702 def CMDremap(args): | |
| 1703 """Creates a directory with all the dependencies mapped into it. | |
| 1704 | |
| 1705 Useful to test manually why a test is failing. The target executable is not | |
| 1706 run. | |
| 1707 """ | |
| 1708 parser = OptionParserIsolate(command='remap', require_result=False) | |
| 1709 options, _ = parser.parse_args(args) | |
| 1710 complete_state = load_complete_state(options, STATS_ONLY) | |
| 1711 | |
| 1712 if not options.outdir: | |
| 1713 options.outdir = run_test_from_archive.make_temp_dir( | |
| 1714 'isolate', complete_state.root_dir) | |
| 1715 else: | |
| 1716 if not os.path.isdir(options.outdir): | |
| 1717 os.makedirs(options.outdir) | |
| 1718 print 'Remapping into %s' % options.outdir | |
| 1719 if len(os.listdir(options.outdir)): | |
| 1720 raise ExecutionError('Can\'t remap in a non-empty directory') | |
| 1721 recreate_tree( | |
| 1722 outdir=options.outdir, | |
| 1723 indir=complete_state.root_dir, | |
| 1724 infiles=complete_state.result.files, | |
| 1725 action=run_test_from_archive.HARDLINK, | |
| 1726 as_sha1=False) | |
| 1727 if complete_state.result.read_only: | |
| 1728 run_test_from_archive.make_writable(options.outdir, True) | |
| 1729 | |
| 1730 if complete_state.result_file: | |
| 1731 complete_state.save_files() | |
| 1732 return 0 | |
| 1733 | |
| 1734 | |
| 1735 def CMDrun(args): | |
| 1736 """Runs the test executable in an isolated (temporary) directory. | |
| 1737 | |
| 1738 All the dependencies are mapped into the temporary directory and the | |
| 1739 directory is cleaned up after the target exits. Warning: if -outdir is | |
| 1740 specified, it is deleted upon exit. | |
| 1741 | |
| 1742 Argument processing stops at the first non-recognized argument and these | |
| 1743 arguments are appended to the command line of the target to run. For example, | |
| 1744 use: isolate.py -r foo.results -- --gtest_filter=Foo.Bar | |
| 1745 """ | |
| 1746 parser = OptionParserIsolate(command='run', require_result=False) | |
| 1747 parser.enable_interspersed_args() | |
| 1748 options, args = parser.parse_args(args) | |
| 1749 complete_state = load_complete_state(options, STATS_ONLY) | |
| 1750 cmd = complete_state.result.command + args | |
| 1751 if not cmd: | |
| 1752 raise ExecutionError('No command to run') | |
| 1753 cmd = trace_inputs.fix_python_path(cmd) | |
| 1754 try: | |
| 1755 if not options.outdir: | |
| 1756 options.outdir = run_test_from_archive.make_temp_dir( | |
| 1757 'isolate', complete_state.root_dir) | |
| 1758 else: | |
| 1759 if not os.path.isdir(options.outdir): | |
| 1760 os.makedirs(options.outdir) | |
| 1761 recreate_tree( | |
| 1762 outdir=options.outdir, | |
| 1763 indir=complete_state.root_dir, | |
| 1764 infiles=complete_state.result.files, | |
| 1765 action=run_test_from_archive.HARDLINK, | |
| 1766 as_sha1=False) | |
| 1767 cwd = os.path.normpath( | |
| 1768 os.path.join(options.outdir, complete_state.result.relative_cwd)) | |
| 1769 if not os.path.isdir(cwd): | |
| 1770 # It can happen when no files are mapped from the directory containing the | |
| 1771 # .isolate file. But the directory must exist to be the current working | |
| 1772 # directory. | |
| 1773 os.makedirs(cwd) | |
| 1774 if complete_state.result.read_only: | |
| 1775 run_test_from_archive.make_writable(options.outdir, True) | |
| 1776 logging.info('Running %s, cwd=%s' % (cmd, cwd)) | |
| 1777 result = subprocess.call(cmd, cwd=cwd) | |
| 1778 finally: | |
| 1779 if options.outdir: | |
| 1780 run_test_from_archive.rmtree(options.outdir) | |
| 1781 | |
| 1782 if complete_state.result_file: | |
| 1783 complete_state.save_files() | |
| 1784 return result | |
| 1785 | |
| 1786 | |
| 1787 def CMDtrace(args): | |
| 1788 """Traces the target using trace_inputs.py. | |
| 1789 | |
| 1790 It runs the executable without remapping it, and traces all the files it and | |
| 1791 its child processes access. Then the 'read' command can be used to generate an | |
| 1792 updated .isolate file out of it. | |
| 1793 | |
| 1794 Argument processing stops at the first non-recognized argument and these | |
| 1795 arguments are appended to the command line of the target to run. For example, | |
| 1796 use: isolate.py -r foo.results -- --gtest_filter=Foo.Bar | |
| 1797 """ | |
| 1798 parser = OptionParserIsolate(command='trace') | |
| 1799 parser.enable_interspersed_args() | |
| 1800 parser.add_option( | |
| 1801 '-m', '--merge', action='store_true', | |
| 1802 help='After tracing, merge the results back in the .isolate file') | |
| 1803 options, args = parser.parse_args(args) | |
| 1804 complete_state = load_complete_state(options, STATS_ONLY) | |
| 1805 cmd = complete_state.result.command + args | |
| 1806 if not cmd: | |
| 1807 raise ExecutionError('No command to run') | |
| 1808 cmd = trace_inputs.fix_python_path(cmd) | |
| 1809 cwd = os.path.normpath(os.path.join( | |
| 1810 complete_state.root_dir, complete_state.result.relative_cwd)) | |
| 1811 logging.info('Running %s, cwd=%s' % (cmd, cwd)) | |
| 1812 api = trace_inputs.get_api() | |
| 1813 logfile = complete_state.result_file + '.log' | |
| 1814 api.clean_trace(logfile) | |
| 1815 try: | |
| 1816 with api.get_tracer(logfile) as tracer: | |
| 1817 result, _ = tracer.trace( | |
| 1818 cmd, | |
| 1819 cwd, | |
| 1820 'default', | |
| 1821 True) | |
| 1822 except trace_inputs.TracingFailure, e: | |
| 1823 raise ExecutionError('Tracing failed for: %s\n%s' % (' '.join(cmd), str(e))) | |
| 1824 | |
| 1825 complete_state.save_files() | |
| 1826 | |
| 1827 if options.merge: | |
| 1828 merge(complete_state) | |
| 1829 | |
| 1830 return result | |
| 1831 | |
| 1832 | |
| 1833 class OptionParserIsolate(trace_inputs.OptionParserWithNiceDescription): | |
| 1834 """Adds automatic --isolate, --result, --out and --variables handling.""" | |
| 1835 def __init__(self, require_result=True, **kwargs): | |
| 1836 trace_inputs.OptionParserWithNiceDescription.__init__(self, **kwargs) | |
| 1837 default_variables = [('OS', get_flavor())] | |
| 1838 if sys.platform in ('win32', 'cygwin'): | |
| 1839 default_variables.append(('EXECUTABLE_SUFFIX', '.exe')) | |
| 1840 else: | |
| 1841 default_variables.append(('EXECUTABLE_SUFFIX', '')) | |
| 1842 group = optparse.OptionGroup(self, "Common options") | |
| 1843 group.add_option( | |
| 1844 '-r', '--result', | |
| 1845 metavar='FILE', | |
| 1846 help='.result file to store the json manifest') | |
| 1847 group.add_option( | |
| 1848 '-i', '--isolate', | |
| 1849 metavar='FILE', | |
| 1850 help='.isolate file to load the dependency data from') | |
| 1851 group.add_option( | |
| 1852 '-V', '--variable', | |
| 1853 nargs=2, | |
| 1854 action='append', | |
| 1855 default=default_variables, | |
| 1856 dest='variables', | |
| 1857 metavar='FOO BAR', | |
| 1858 help='Variables to process in the .isolate file, default: %default. ' | |
| 1859 'Variables are persistent accross calls, they are saved inside ' | |
| 1860 '<results>.state') | |
| 1861 group.add_option( | |
| 1862 '-o', '--outdir', metavar='DIR', | |
| 1863 help='Directory used to recreate the tree or store the hash table. ' | |
| 1864 'If the environment variable ISOLATE_HASH_TABLE_DIR exists, it ' | |
| 1865 'will be used. Otherwise, for run and remap, uses a /tmp ' | |
| 1866 'subdirectory. For the other modes, defaults to the directory ' | |
| 1867 'containing --result') | |
| 1868 self.add_option_group(group) | |
| 1869 self.require_result = require_result | |
| 1870 | |
| 1871 def parse_args(self, *args, **kwargs): | |
| 1872 """Makes sure the paths make sense. | |
| 1873 | |
| 1874 On Windows, / and \ are often mixed together in a path. | |
| 1875 """ | |
| 1876 options, args = trace_inputs.OptionParserWithNiceDescription.parse_args( | |
| 1877 self, *args, **kwargs) | |
| 1878 if not self.allow_interspersed_args and args: | |
| 1879 self.error('Unsupported argument: %s' % args) | |
| 1880 | |
| 1881 options.variables = dict(options.variables) | |
| 1882 | |
| 1883 if self.require_result and not options.result: | |
| 1884 self.error('--result is required.') | |
| 1885 if options.result and not options.result.endswith('.results'): | |
| 1886 self.error('--result value must end with \'.results\'') | |
| 1887 | |
| 1888 if options.result: | |
| 1889 options.result = os.path.abspath(options.result.replace('/', os.path.sep)) | |
| 1890 | |
| 1891 if options.isolate: | |
| 1892 options.isolate = trace_inputs.get_native_path_case( | |
| 1893 os.path.abspath( | |
| 1894 options.isolate.replace('/', os.path.sep))) | |
| 1895 | |
| 1896 if options.outdir and not re.match(r'^https?://.+$', options.outdir): | |
| 1897 options.outdir = os.path.abspath( | |
| 1898 options.outdir.replace('/', os.path.sep)) | |
| 1899 | |
| 1900 return options, args | |
| 1901 | |
| 1902 | |
| 1903 ### Glue code to make all the commands works magically. | |
| 1904 | |
| 1905 | |
| 1906 CMDhelp = trace_inputs.CMDhelp | |
| 1907 | |
| 1908 | |
| 1909 def main(argv): | |
| 1910 try: | |
| 1911 return trace_inputs.main_impl(argv) | |
| 1912 except ( | |
| 1913 ExecutionError, | |
| 1914 run_test_from_archive.MappingError, | |
| 1915 run_test_from_archive.ConfigError) as e: | |
| 1916 sys.stderr.write('\nError: ') | |
| 1917 sys.stderr.write(str(e)) | |
| 1918 sys.stderr.write('\n') | |
| 1919 return 1 | |
| 1920 | |
| 1921 | |
| 1922 if __name__ == '__main__': | |
| 1923 sys.exit(main(sys.argv[1:])) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 11045023:
Move src/tools/isolate to src/tools/swarm_client as a DEPS. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/src