| Index: swarm_client/isolate.py
|
| ===================================================================
|
| --- swarm_client/isolate.py (revision 235167)
|
| +++ swarm_client/isolate.py (working copy)
|
| @@ -1,2461 +0,0 @@
|
| -#!/usr/bin/env python
|
| -# Copyright (c) 2012 The Chromium Authors. All rights reserved.
|
| -# Use of this source code is governed by a BSD-style license that can be
|
| -# found in the LICENSE file.
|
| -
|
| -"""Front end tool to operate on .isolate files.
|
| -
|
| -This includes creating, merging or compiling them to generate a .isolated file.
|
| -
|
| -See more information at
|
| - https://code.google.com/p/swarming/wiki/IsolateDesign
|
| - https://code.google.com/p/swarming/wiki/IsolateUserGuide
|
| -"""
|
| -# Run ./isolate.py --help for more detailed information.
|
| -
|
| -import ast
|
| -import copy
|
| -import itertools
|
| -import logging
|
| -import optparse
|
| -import os
|
| -import posixpath
|
| -import re
|
| -import stat
|
| -import subprocess
|
| -import sys
|
| -
|
| -import isolateserver
|
| -import run_isolated
|
| -import trace_inputs
|
| -
|
| -# Import here directly so isolate is easier to use as a library.
|
| -from run_isolated import get_flavor
|
| -
|
| -from third_party import colorama
|
| -from third_party.depot_tools import fix_encoding
|
| -from third_party.depot_tools import subcommand
|
| -
|
| -from utils import file_path
|
| -from utils import tools
|
| -from utils import short_expression_finder
|
| -
|
| -
|
| -__version__ = '0.1.1'
|
| -
|
| -
|
| -PATH_VARIABLES = ('DEPTH', 'PRODUCT_DIR')
|
| -
|
| -# Files that should be 0-length when mapped.
|
| -KEY_TOUCHED = 'isolate_dependency_touched'
|
| -# Files that should be tracked by the build tool.
|
| -KEY_TRACKED = 'isolate_dependency_tracked'
|
| -# Files that should not be tracked by the build tool.
|
| -KEY_UNTRACKED = 'isolate_dependency_untracked'
|
| -
|
| -
|
| -class ExecutionError(Exception):
|
| - """A generic error occurred."""
|
| - def __str__(self):
|
| - return self.args[0]
|
| -
|
| -
|
| -### Path handling code.
|
| -
|
| -
|
| -DEFAULT_BLACKLIST = (
|
| - # Temporary vim or python files.
|
| - r'^.+\.(?:pyc|swp)$',
|
| - # .git or .svn directory.
|
| - r'^(?:.+' + re.escape(os.path.sep) + r'|)\.(?:git|svn)$',
|
| -)
|
| -
|
| -
|
| -# Chromium-specific.
|
| -DEFAULT_BLACKLIST += (
|
| - r'^.+\.(?:run_test_cases)$',
|
| - r'^(?:.+' + re.escape(os.path.sep) + r'|)testserver\.log$',
|
| -)
|
| -
|
| -
|
| -def relpath(path, root):
|
| - """os.path.relpath() that keeps trailing os.path.sep."""
|
| - out = os.path.relpath(path, root)
|
| - if path.endswith(os.path.sep):
|
| - out += os.path.sep
|
| - return out
|
| -
|
| -
|
| -def safe_relpath(filepath, basepath):
|
| - """Do not throw on Windows when filepath and basepath are on different drives.
|
| -
|
| - Different than relpath() above since this one doesn't keep the trailing
|
| - os.path.sep and it swallows exceptions on Windows and return the original
|
| - absolute path in the case of different drives.
|
| - """
|
| - try:
|
| - return os.path.relpath(filepath, basepath)
|
| - except ValueError:
|
| - assert sys.platform == 'win32'
|
| - return filepath
|
| -
|
| -
|
| -def normpath(path):
|
| - """os.path.normpath() that keeps trailing os.path.sep."""
|
| - out = os.path.normpath(path)
|
| - if path.endswith(os.path.sep):
|
| - out += os.path.sep
|
| - return out
|
| -
|
| -
|
| -def posix_relpath(path, root):
|
| - """posix.relpath() that keeps trailing slash."""
|
| - out = posixpath.relpath(path, root)
|
| - if path.endswith('/'):
|
| - out += '/'
|
| - return out
|
| -
|
| -
|
| -def cleanup_path(x):
|
| - """Cleans up a relative path. Converts any os.path.sep to '/' on Windows."""
|
| - if x:
|
| - x = x.rstrip(os.path.sep).replace(os.path.sep, '/')
|
| - if x == '.':
|
| - x = ''
|
| - if x:
|
| - x += '/'
|
| - return x
|
| -
|
| -
|
| -def is_url(path):
|
| - return bool(re.match(r'^https?://.+$', path))
|
| -
|
| -
|
| -def path_starts_with(prefix, path):
|
| - """Returns true if the components of the path |prefix| are the same as the
|
| - initial components of |path| (or all of the components of |path|). The paths
|
| - must be absolute.
|
| - """
|
| - assert os.path.isabs(prefix) and os.path.isabs(path)
|
| - prefix = os.path.normpath(prefix)
|
| - path = os.path.normpath(path)
|
| - assert prefix == file_path.get_native_path_case(prefix), prefix
|
| - assert path == file_path.get_native_path_case(path), path
|
| - prefix = prefix.rstrip(os.path.sep) + os.path.sep
|
| - path = path.rstrip(os.path.sep) + os.path.sep
|
| - return path.startswith(prefix)
|
| -
|
| -
|
| -def fix_native_path_case(root, path):
|
| - """Ensures that each component of |path| has the proper native case by
|
| - iterating slowly over the directory elements of |path|."""
|
| - native_case_path = root
|
| - for raw_part in path.split(os.sep):
|
| - if not raw_part or raw_part == '.':
|
| - break
|
| -
|
| - part = file_path.find_item_native_case(native_case_path, raw_part)
|
| - if not part:
|
| - raise isolateserver.MappingError(
|
| - 'Input file %s doesn\'t exist' %
|
| - os.path.join(native_case_path, raw_part))
|
| - native_case_path = os.path.join(native_case_path, part)
|
| -
|
| - return os.path.normpath(native_case_path)
|
| -
|
| -
|
| -def expand_symlinks(indir, relfile):
|
| - """Follows symlinks in |relfile|, but treating symlinks that point outside the
|
| - build tree as if they were ordinary directories/files. Returns the final
|
| - symlink-free target and a list of paths to symlinks encountered in the
|
| - process.
|
| -
|
| - The rule about symlinks outside the build tree is for the benefit of the
|
| - Chromium OS ebuild, which symlinks the output directory to an unrelated path
|
| - in the chroot.
|
| -
|
| - Fails when a directory loop is detected, although in theory we could support
|
| - that case.
|
| - """
|
| - is_directory = relfile.endswith(os.path.sep)
|
| - done = indir
|
| - todo = relfile.strip(os.path.sep)
|
| - symlinks = []
|
| -
|
| - while todo:
|
| - pre_symlink, symlink, post_symlink = file_path.split_at_symlink(
|
| - done, todo)
|
| - if not symlink:
|
| - todo = fix_native_path_case(done, todo)
|
| - done = os.path.join(done, todo)
|
| - break
|
| - symlink_path = os.path.join(done, pre_symlink, symlink)
|
| - post_symlink = post_symlink.lstrip(os.path.sep)
|
| - # readlink doesn't exist on Windows.
|
| - # pylint: disable=E1101
|
| - target = os.path.normpath(os.path.join(done, pre_symlink))
|
| - symlink_target = os.readlink(symlink_path)
|
| - if os.path.isabs(symlink_target):
|
| - # Absolute path are considered a normal directories. The use case is
|
| - # generally someone who puts the output directory on a separate drive.
|
| - target = symlink_target
|
| - else:
|
| - # The symlink itself could be using the wrong path case.
|
| - target = fix_native_path_case(target, symlink_target)
|
| -
|
| - if not os.path.exists(target):
|
| - raise isolateserver.MappingError(
|
| - 'Symlink target doesn\'t exist: %s -> %s' % (symlink_path, target))
|
| - target = file_path.get_native_path_case(target)
|
| - if not path_starts_with(indir, target):
|
| - done = symlink_path
|
| - todo = post_symlink
|
| - continue
|
| - if path_starts_with(target, symlink_path):
|
| - raise isolateserver.MappingError(
|
| - 'Can\'t map recursive symlink reference %s -> %s' %
|
| - (symlink_path, target))
|
| - logging.info('Found symlink: %s -> %s', symlink_path, target)
|
| - symlinks.append(os.path.relpath(symlink_path, indir))
|
| - # Treat the common prefix of the old and new paths as done, and start
|
| - # scanning again.
|
| - target = target.split(os.path.sep)
|
| - symlink_path = symlink_path.split(os.path.sep)
|
| - prefix_length = 0
|
| - for target_piece, symlink_path_piece in zip(target, symlink_path):
|
| - if target_piece == symlink_path_piece:
|
| - prefix_length += 1
|
| - else:
|
| - break
|
| - done = os.path.sep.join(target[:prefix_length])
|
| - todo = os.path.join(
|
| - os.path.sep.join(target[prefix_length:]), post_symlink)
|
| -
|
| - relfile = os.path.relpath(done, indir)
|
| - relfile = relfile.rstrip(os.path.sep) + is_directory * os.path.sep
|
| - return relfile, symlinks
|
| -
|
| -
|
| -def expand_directory_and_symlink(indir, relfile, blacklist, follow_symlinks):
|
| - """Expands a single input. It can result in multiple outputs.
|
| -
|
| - This function is recursive when relfile is a directory.
|
| -
|
| - Note: this code doesn't properly handle recursive symlink like one created
|
| - with:
|
| - ln -s .. foo
|
| - """
|
| - if os.path.isabs(relfile):
|
| - raise isolateserver.MappingError(
|
| - 'Can\'t map absolute path %s' % relfile)
|
| -
|
| - infile = normpath(os.path.join(indir, relfile))
|
| - if not infile.startswith(indir):
|
| - raise isolateserver.MappingError(
|
| - 'Can\'t map file %s outside %s' % (infile, indir))
|
| -
|
| - filepath = os.path.join(indir, relfile)
|
| - native_filepath = file_path.get_native_path_case(filepath)
|
| - if filepath != native_filepath:
|
| - # Special case './'.
|
| - if filepath != native_filepath + '.' + os.path.sep:
|
| - # Give up enforcing strict path case on OSX. Really, it's that sad. The
|
| - # case where it happens is very specific and hard to reproduce:
|
| - # get_native_path_case(
|
| - # u'Foo.framework/Versions/A/Resources/Something.nib') will return
|
| - # u'Foo.framework/Versions/A/resources/Something.nib', e.g. lowercase 'r'.
|
| - #
|
| - # Note that this is really something deep in OSX because running
|
| - # ls Foo.framework/Versions/A
|
| - # will print out 'Resources', while file_path.get_native_path_case()
|
| - # returns a lower case 'r'.
|
| - #
|
| - # So *something* is happening under the hood resulting in the command 'ls'
|
| - # and Carbon.File.FSPathMakeRef('path').FSRefMakePath() to disagree. We
|
| - # have no idea why.
|
| - if sys.platform != 'darwin':
|
| - raise isolateserver.MappingError(
|
| - 'File path doesn\'t equal native file path\n%s != %s' %
|
| - (filepath, native_filepath))
|
| -
|
| - symlinks = []
|
| - if follow_symlinks:
|
| - relfile, symlinks = expand_symlinks(indir, relfile)
|
| -
|
| - if relfile.endswith(os.path.sep):
|
| - if not os.path.isdir(infile):
|
| - raise isolateserver.MappingError(
|
| - '%s is not a directory but ends with "%s"' % (infile, os.path.sep))
|
| -
|
| - # Special case './'.
|
| - if relfile.startswith('.' + os.path.sep):
|
| - relfile = relfile[2:]
|
| - outfiles = symlinks
|
| - try:
|
| - for filename in os.listdir(infile):
|
| - inner_relfile = os.path.join(relfile, filename)
|
| - if blacklist(inner_relfile):
|
| - continue
|
| - if os.path.isdir(os.path.join(indir, inner_relfile)):
|
| - inner_relfile += os.path.sep
|
| - outfiles.extend(
|
| - expand_directory_and_symlink(indir, inner_relfile, blacklist,
|
| - follow_symlinks))
|
| - return outfiles
|
| - except OSError as e:
|
| - raise isolateserver.MappingError(
|
| - 'Unable to iterate over directory %s.\n%s' % (infile, e))
|
| - else:
|
| - # Always add individual files even if they were blacklisted.
|
| - if os.path.isdir(infile):
|
| - raise isolateserver.MappingError(
|
| - 'Input directory %s must have a trailing slash' % infile)
|
| -
|
| - if not os.path.isfile(infile):
|
| - raise isolateserver.MappingError(
|
| - 'Input file %s doesn\'t exist' % infile)
|
| -
|
| - return symlinks + [relfile]
|
| -
|
| -
|
| -def expand_directories_and_symlinks(indir, infiles, blacklist,
|
| - follow_symlinks, ignore_broken_items):
|
| - """Expands the directories and the symlinks, applies the blacklist and
|
| - verifies files exist.
|
| -
|
| - Files are specified in os native path separator.
|
| - """
|
| - outfiles = []
|
| - for relfile in infiles:
|
| - try:
|
| - outfiles.extend(expand_directory_and_symlink(indir, relfile, blacklist,
|
| - follow_symlinks))
|
| - except isolateserver.MappingError as e:
|
| - if ignore_broken_items:
|
| - logging.info('warning: %s', e)
|
| - else:
|
| - raise
|
| - return outfiles
|
| -
|
| -
|
| -def recreate_tree(outdir, indir, infiles, action, as_hash):
|
| - """Creates a new tree with only the input files in it.
|
| -
|
| - Arguments:
|
| - outdir: Output directory to create the files in.
|
| - indir: Root directory the infiles are based in.
|
| - infiles: dict of files to map from |indir| to |outdir|.
|
| - action: One of accepted action of run_isolated.link_file().
|
| - as_hash: Output filename is the hash instead of relfile.
|
| - """
|
| - logging.info(
|
| - 'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
|
| - (outdir, indir, len(infiles), action, as_hash))
|
| -
|
| - assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
|
| - if not os.path.isdir(outdir):
|
| - logging.info('Creating %s' % outdir)
|
| - os.makedirs(outdir)
|
| -
|
| - for relfile, metadata in infiles.iteritems():
|
| - infile = os.path.join(indir, relfile)
|
| - if as_hash:
|
| - # Do the hashtable specific checks.
|
| - if 'l' in metadata:
|
| - # Skip links when storing a hashtable.
|
| - continue
|
| - outfile = os.path.join(outdir, metadata['h'])
|
| - if os.path.isfile(outfile):
|
| - # Just do a quick check that the file size matches. No need to stat()
|
| - # again the input file, grab the value from the dict.
|
| - if not 's' in metadata:
|
| - raise isolateserver.MappingError(
|
| - 'Misconfigured item %s: %s' % (relfile, metadata))
|
| - if metadata['s'] == os.stat(outfile).st_size:
|
| - continue
|
| - else:
|
| - logging.warn('Overwritting %s' % metadata['h'])
|
| - os.remove(outfile)
|
| - else:
|
| - outfile = os.path.join(outdir, relfile)
|
| - outsubdir = os.path.dirname(outfile)
|
| - if not os.path.isdir(outsubdir):
|
| - os.makedirs(outsubdir)
|
| -
|
| - # TODO(csharp): Fix crbug.com/150823 and enable the touched logic again.
|
| - # if metadata.get('T') == True:
|
| - # open(outfile, 'ab').close()
|
| - if 'l' in metadata:
|
| - pointed = metadata['l']
|
| - logging.debug('Symlink: %s -> %s' % (outfile, pointed))
|
| - # symlink doesn't exist on Windows.
|
| - os.symlink(pointed, outfile) # pylint: disable=E1101
|
| - else:
|
| - run_isolated.link_file(outfile, infile, action)
|
| -
|
| -
|
| -def process_input(filepath, prevdict, read_only, flavor, algo):
|
| - """Processes an input file, a dependency, and return meta data about it.
|
| -
|
| - Arguments:
|
| - - filepath: File to act on.
|
| - - prevdict: the previous dictionary. It is used to retrieve the cached sha-1
|
| - to skip recalculating the hash.
|
| - - read_only: If True, the file mode is manipulated. In practice, only save
|
| - one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r). On
|
| - windows, mode is not set since all files are 'executable' by
|
| - default.
|
| - - algo: Hashing algorithm used.
|
| -
|
| - Behaviors:
|
| - - Retrieves the file mode, file size, file timestamp, file link
|
| - destination if it is a file link and calcultate the SHA-1 of the file's
|
| - content if the path points to a file and not a symlink.
|
| - """
|
| - out = {}
|
| - # TODO(csharp): Fix crbug.com/150823 and enable the touched logic again.
|
| - # if prevdict.get('T') == True:
|
| - # # The file's content is ignored. Skip the time and hard code mode.
|
| - # if get_flavor() != 'win':
|
| - # out['m'] = stat.S_IRUSR | stat.S_IRGRP
|
| - # out['s'] = 0
|
| - # out['h'] = algo().hexdigest()
|
| - # out['T'] = True
|
| - # return out
|
| -
|
| - # Always check the file stat and check if it is a link. The timestamp is used
|
| - # to know if the file's content/symlink destination should be looked into.
|
| - # E.g. only reuse from prevdict if the timestamp hasn't changed.
|
| - # There is the risk of the file's timestamp being reset to its last value
|
| - # manually while its content changed. We don't protect against that use case.
|
| - try:
|
| - filestats = os.lstat(filepath)
|
| - except OSError:
|
| - # The file is not present.
|
| - raise isolateserver.MappingError('%s is missing' % filepath)
|
| - is_link = stat.S_ISLNK(filestats.st_mode)
|
| -
|
| - if flavor != 'win':
|
| - # Ignore file mode on Windows since it's not really useful there.
|
| - filemode = stat.S_IMODE(filestats.st_mode)
|
| - # Remove write access for group and all access to 'others'.
|
| - filemode &= ~(stat.S_IWGRP | stat.S_IRWXO)
|
| - if read_only:
|
| - filemode &= ~stat.S_IWUSR
|
| - if filemode & stat.S_IXUSR:
|
| - filemode |= stat.S_IXGRP
|
| - else:
|
| - filemode &= ~stat.S_IXGRP
|
| - if not is_link:
|
| - out['m'] = filemode
|
| -
|
| - # Used to skip recalculating the hash or link destination. Use the most recent
|
| - # update time.
|
| - # TODO(maruel): Save it in the .state file instead of .isolated so the
|
| - # .isolated file is deterministic.
|
| - out['t'] = int(round(filestats.st_mtime))
|
| -
|
| - if not is_link:
|
| - out['s'] = filestats.st_size
|
| - # If the timestamp wasn't updated and the file size is still the same, carry
|
| - # on the sha-1.
|
| - if (prevdict.get('t') == out['t'] and
|
| - prevdict.get('s') == out['s']):
|
| - # Reuse the previous hash if available.
|
| - out['h'] = prevdict.get('h')
|
| - if not out.get('h'):
|
| - out['h'] = isolateserver.hash_file(filepath, algo)
|
| - else:
|
| - # If the timestamp wasn't updated, carry on the link destination.
|
| - if prevdict.get('t') == out['t']:
|
| - # Reuse the previous link destination if available.
|
| - out['l'] = prevdict.get('l')
|
| - if out.get('l') is None:
|
| - # The link could be in an incorrect path case. In practice, this only
|
| - # happen on OSX on case insensitive HFS.
|
| - # TODO(maruel): It'd be better if it was only done once, in
|
| - # expand_directory_and_symlink(), so it would not be necessary to do again
|
| - # here.
|
| - symlink_value = os.readlink(filepath) # pylint: disable=E1101
|
| - filedir = file_path.get_native_path_case(os.path.dirname(filepath))
|
| - native_dest = fix_native_path_case(filedir, symlink_value)
|
| - out['l'] = os.path.relpath(native_dest, filedir)
|
| - return out
|
| -
|
| -
|
| -### Variable stuff.
|
| -
|
| -
|
| -def isolatedfile_to_state(filename):
|
| - """Replaces the file's extension."""
|
| - return filename + '.state'
|
| -
|
| -
|
| -def determine_root_dir(relative_root, infiles):
|
| - """For a list of infiles, determines the deepest root directory that is
|
| - referenced indirectly.
|
| -
|
| - All arguments must be using os.path.sep.
|
| - """
|
| - # The trick used to determine the root directory is to look at "how far" back
|
| - # up it is looking up.
|
| - deepest_root = relative_root
|
| - for i in infiles:
|
| - x = relative_root
|
| - while i.startswith('..' + os.path.sep):
|
| - i = i[3:]
|
| - assert not i.startswith(os.path.sep)
|
| - x = os.path.dirname(x)
|
| - if deepest_root.startswith(x):
|
| - deepest_root = x
|
| - logging.debug(
|
| - 'determine_root_dir(%s, %d files) -> %s' % (
|
| - relative_root, len(infiles), deepest_root))
|
| - return deepest_root
|
| -
|
| -
|
| -def replace_variable(part, variables):
|
| - m = re.match(r'<\(([A-Z_]+)\)', part)
|
| - if m:
|
| - if m.group(1) not in variables:
|
| - raise ExecutionError(
|
| - 'Variable "%s" was not found in %s.\nDid you forget to specify '
|
| - '--variable?' % (m.group(1), variables))
|
| - return variables[m.group(1)]
|
| - return part
|
| -
|
| -
|
| -def process_variables(cwd, variables, relative_base_dir):
|
| - """Processes path variables as a special case and returns a copy of the dict.
|
| -
|
| - For each 'path' variable: first normalizes it based on |cwd|, verifies it
|
| - exists then sets it as relative to relative_base_dir.
|
| - """
|
| - relative_base_dir = file_path.get_native_path_case(relative_base_dir)
|
| - variables = variables.copy()
|
| - for i in PATH_VARIABLES:
|
| - if i not in variables:
|
| - continue
|
| - variable = variables[i].strip()
|
| - # Variables could contain / or \ on windows. Always normalize to
|
| - # os.path.sep.
|
| - variable = variable.replace('/', os.path.sep)
|
| - variable = os.path.join(cwd, variable)
|
| - variable = os.path.normpath(variable)
|
| - variable = file_path.get_native_path_case(variable)
|
| - if not os.path.isdir(variable):
|
| - raise ExecutionError('%s=%s is not a directory' % (i, variable))
|
| -
|
| - # All variables are relative to the .isolate file.
|
| - variable = os.path.relpath(variable, relative_base_dir)
|
| - logging.debug(
|
| - 'Translated variable %s from %s to %s', i, variables[i], variable)
|
| - variables[i] = variable
|
| - return variables
|
| -
|
| -
|
| -def eval_variables(item, variables):
|
| - """Replaces the .isolate variables in a string item.
|
| -
|
| - Note that the .isolate format is a subset of the .gyp dialect.
|
| - """
|
| - return ''.join(
|
| - replace_variable(p, variables) for p in re.split(r'(<\([A-Z_]+\))', item))
|
| -
|
| -
|
| -def classify_files(root_dir, tracked, untracked):
|
| - """Converts the list of files into a .isolate 'variables' dictionary.
|
| -
|
| - Arguments:
|
| - - tracked: list of files names to generate a dictionary out of that should
|
| - probably be tracked.
|
| - - untracked: list of files names that must not be tracked.
|
| - """
|
| - # These directories are not guaranteed to be always present on every builder.
|
| - OPTIONAL_DIRECTORIES = (
|
| - 'test/data/plugin',
|
| - 'third_party/WebKit/LayoutTests',
|
| - )
|
| -
|
| - new_tracked = []
|
| - new_untracked = list(untracked)
|
| -
|
| - def should_be_tracked(filepath):
|
| - """Returns True if it is a file without whitespace in a non-optional
|
| - directory that has no symlink in its path.
|
| - """
|
| - if filepath.endswith('/'):
|
| - return False
|
| - if ' ' in filepath:
|
| - return False
|
| - if any(i in filepath for i in OPTIONAL_DIRECTORIES):
|
| - return False
|
| - # Look if any element in the path is a symlink.
|
| - split = filepath.split('/')
|
| - for i in range(len(split)):
|
| - if os.path.islink(os.path.join(root_dir, '/'.join(split[:i+1]))):
|
| - return False
|
| - return True
|
| -
|
| - for filepath in sorted(tracked):
|
| - if should_be_tracked(filepath):
|
| - new_tracked.append(filepath)
|
| - else:
|
| - # Anything else.
|
| - new_untracked.append(filepath)
|
| -
|
| - variables = {}
|
| - if new_tracked:
|
| - variables[KEY_TRACKED] = sorted(new_tracked)
|
| - if new_untracked:
|
| - variables[KEY_UNTRACKED] = sorted(new_untracked)
|
| - return variables
|
| -
|
| -
|
| -def chromium_fix(f, variables):
|
| - """Fixes an isolate dependnecy with Chromium-specific fixes."""
|
| - # Skip log in PRODUCT_DIR. Note that these are applied on '/' style path
|
| - # separator.
|
| - LOG_FILE = re.compile(r'^\<\(PRODUCT_DIR\)\/[^\/]+\.log$')
|
| - # Ignored items.
|
| - IGNORED_ITEMS = (
|
| - # http://crbug.com/160539, on Windows, it's in chrome/.
|
| - 'Media Cache/',
|
| - 'chrome/Media Cache/',
|
| - # 'First Run' is not created by the compile, but by the test itself.
|
| - '<(PRODUCT_DIR)/First Run')
|
| -
|
| - # Blacklist logs and other unimportant files.
|
| - if LOG_FILE.match(f) or f in IGNORED_ITEMS:
|
| - logging.debug('Ignoring %s', f)
|
| - return None
|
| -
|
| - EXECUTABLE = re.compile(
|
| - r'^(\<\(PRODUCT_DIR\)\/[^\/\.]+)' +
|
| - re.escape(variables.get('EXECUTABLE_SUFFIX', '')) +
|
| - r'$')
|
| - match = EXECUTABLE.match(f)
|
| - if match:
|
| - return match.group(1) + '<(EXECUTABLE_SUFFIX)'
|
| -
|
| - if sys.platform == 'darwin':
|
| - # On OSX, the name of the output is dependent on gyp define, it can be
|
| - # 'Google Chrome.app' or 'Chromium.app', same for 'XXX
|
| - # Framework.framework'. Furthermore, they are versioned with a gyp
|
| - # variable. To lower the complexity of the .isolate file, remove all the
|
| - # individual entries that show up under any of the 4 entries and replace
|
| - # them with the directory itself. Overall, this results in a bit more
|
| - # files than strictly necessary.
|
| - OSX_BUNDLES = (
|
| - '<(PRODUCT_DIR)/Chromium Framework.framework/',
|
| - '<(PRODUCT_DIR)/Chromium.app/',
|
| - '<(PRODUCT_DIR)/Google Chrome Framework.framework/',
|
| - '<(PRODUCT_DIR)/Google Chrome.app/',
|
| - )
|
| - for prefix in OSX_BUNDLES:
|
| - if f.startswith(prefix):
|
| - # Note this result in duplicate values, so the a set() must be used to
|
| - # remove duplicates.
|
| - return prefix
|
| - return f
|
| -
|
| -
|
| -def generate_simplified(
|
| - tracked, untracked, touched, root_dir, variables, relative_cwd,
|
| - trace_blacklist):
|
| - """Generates a clean and complete .isolate 'variables' dictionary.
|
| -
|
| - Cleans up and extracts only files from within root_dir then processes
|
| - variables and relative_cwd.
|
| - """
|
| - root_dir = os.path.realpath(root_dir)
|
| - logging.info(
|
| - 'generate_simplified(%d files, %s, %s, %s)' %
|
| - (len(tracked) + len(untracked) + len(touched),
|
| - root_dir, variables, relative_cwd))
|
| -
|
| - # Preparation work.
|
| - relative_cwd = cleanup_path(relative_cwd)
|
| - assert not os.path.isabs(relative_cwd), relative_cwd
|
| - # Creates the right set of variables here. We only care about PATH_VARIABLES.
|
| - path_variables = dict(
|
| - ('<(%s)' % k, variables[k].replace(os.path.sep, '/'))
|
| - for k in PATH_VARIABLES if k in variables)
|
| - variables = variables.copy()
|
| - variables.update(path_variables)
|
| -
|
| - # Actual work: Process the files.
|
| - # TODO(maruel): if all the files in a directory are in part tracked and in
|
| - # part untracked, the directory will not be extracted. Tracked files should be
|
| - # 'promoted' to be untracked as needed.
|
| - tracked = trace_inputs.extract_directories(
|
| - root_dir, tracked, trace_blacklist)
|
| - untracked = trace_inputs.extract_directories(
|
| - root_dir, untracked, trace_blacklist)
|
| - # touched is not compressed, otherwise it would result in files to be archived
|
| - # that we don't need.
|
| -
|
| - root_dir_posix = root_dir.replace(os.path.sep, '/')
|
| - def fix(f):
|
| - """Bases the file on the most restrictive variable."""
|
| - # Important, GYP stores the files with / and not \.
|
| - f = f.replace(os.path.sep, '/')
|
| - logging.debug('fix(%s)' % f)
|
| - # If it's not already a variable.
|
| - if not f.startswith('<'):
|
| - # relative_cwd is usually the directory containing the gyp file. It may be
|
| - # empty if the whole directory containing the gyp file is needed.
|
| - # Use absolute paths in case cwd_dir is outside of root_dir.
|
| - # Convert the whole thing to / since it's isolate's speak.
|
| - f = posix_relpath(
|
| - posixpath.join(root_dir_posix, f),
|
| - posixpath.join(root_dir_posix, relative_cwd)) or './'
|
| -
|
| - for variable, root_path in path_variables.iteritems():
|
| - if f.startswith(root_path):
|
| - f = variable + f[len(root_path):]
|
| - logging.debug('Converted to %s' % f)
|
| - break
|
| - return f
|
| -
|
| - def fix_all(items):
|
| - """Reduces the items to convert variables, removes unneeded items, apply
|
| - chromium-specific fixes and only return unique items.
|
| - """
|
| - variables_converted = (fix(f.path) for f in items)
|
| - chromium_fixed = (chromium_fix(f, variables) for f in variables_converted)
|
| - return set(f for f in chromium_fixed if f)
|
| -
|
| - tracked = fix_all(tracked)
|
| - untracked = fix_all(untracked)
|
| - touched = fix_all(touched)
|
| - out = classify_files(root_dir, tracked, untracked)
|
| - if touched:
|
| - out[KEY_TOUCHED] = sorted(touched)
|
| - return out
|
| -
|
| -
|
| -def chromium_filter_flags(variables):
|
| - """Filters out build flags used in Chromium that we don't want to treat as
|
| - configuration variables.
|
| - """
|
| - # TODO(benrg): Need a better way to determine this.
|
| - blacklist = set(PATH_VARIABLES + ('EXECUTABLE_SUFFIX', 'FLAG'))
|
| - return dict((k, v) for k, v in variables.iteritems() if k not in blacklist)
|
| -
|
| -
|
| -def generate_isolate(
|
| - tracked, untracked, touched, root_dir, variables, relative_cwd,
|
| - trace_blacklist):
|
| - """Generates a clean and complete .isolate file."""
|
| - dependencies = generate_simplified(
|
| - tracked, untracked, touched, root_dir, variables, relative_cwd,
|
| - trace_blacklist)
|
| - config_variables = chromium_filter_flags(variables)
|
| - config_variable_names, config_values = zip(
|
| - *sorted(config_variables.iteritems()))
|
| - out = Configs(None)
|
| - # The new dependencies apply to just one configuration, namely config_values.
|
| - out.merge_dependencies(dependencies, config_variable_names, [config_values])
|
| - return out.make_isolate_file()
|
| -
|
| -
|
| -def split_touched(files):
|
| - """Splits files that are touched vs files that are read."""
|
| - tracked = []
|
| - touched = []
|
| - for f in files:
|
| - if f.size:
|
| - tracked.append(f)
|
| - else:
|
| - touched.append(f)
|
| - return tracked, touched
|
| -
|
| -
|
| -def pretty_print(variables, stdout):
|
| - """Outputs a gyp compatible list from the decoded variables.
|
| -
|
| - Similar to pprint.print() but with NIH syndrome.
|
| - """
|
| - # Order the dictionary keys by these keys in priority.
|
| - ORDER = (
|
| - 'variables', 'condition', 'command', 'relative_cwd', 'read_only',
|
| - KEY_TRACKED, KEY_UNTRACKED)
|
| -
|
| - def sorting_key(x):
|
| - """Gives priority to 'most important' keys before the others."""
|
| - if x in ORDER:
|
| - return str(ORDER.index(x))
|
| - return x
|
| -
|
| - def loop_list(indent, items):
|
| - for item in items:
|
| - if isinstance(item, basestring):
|
| - stdout.write('%s\'%s\',\n' % (indent, item))
|
| - elif isinstance(item, dict):
|
| - stdout.write('%s{\n' % indent)
|
| - loop_dict(indent + ' ', item)
|
| - stdout.write('%s},\n' % indent)
|
| - elif isinstance(item, list):
|
| - # A list inside a list will write the first item embedded.
|
| - stdout.write('%s[' % indent)
|
| - for index, i in enumerate(item):
|
| - if isinstance(i, basestring):
|
| - stdout.write(
|
| - '\'%s\', ' % i.replace('\\', '\\\\').replace('\'', '\\\''))
|
| - elif isinstance(i, dict):
|
| - stdout.write('{\n')
|
| - loop_dict(indent + ' ', i)
|
| - if index != len(item) - 1:
|
| - x = ', '
|
| - else:
|
| - x = ''
|
| - stdout.write('%s}%s' % (indent, x))
|
| - else:
|
| - assert False
|
| - stdout.write('],\n')
|
| - else:
|
| - assert False
|
| -
|
| - def loop_dict(indent, items):
|
| - for key in sorted(items, key=sorting_key):
|
| - item = items[key]
|
| - stdout.write("%s'%s': " % (indent, key))
|
| - if isinstance(item, dict):
|
| - stdout.write('{\n')
|
| - loop_dict(indent + ' ', item)
|
| - stdout.write(indent + '},\n')
|
| - elif isinstance(item, list):
|
| - stdout.write('[\n')
|
| - loop_list(indent + ' ', item)
|
| - stdout.write(indent + '],\n')
|
| - elif isinstance(item, basestring):
|
| - stdout.write(
|
| - '\'%s\',\n' % item.replace('\\', '\\\\').replace('\'', '\\\''))
|
| - elif item in (True, False, None):
|
| - stdout.write('%s\n' % item)
|
| - else:
|
| - assert False, item
|
| -
|
| - stdout.write('{\n')
|
| - loop_dict(' ', variables)
|
| - stdout.write('}\n')
|
| -
|
| -
|
| -def union(lhs, rhs):
|
| - """Merges two compatible datastructures composed of dict/list/set."""
|
| - assert lhs is not None or rhs is not None
|
| - if lhs is None:
|
| - return copy.deepcopy(rhs)
|
| - if rhs is None:
|
| - return copy.deepcopy(lhs)
|
| - assert type(lhs) == type(rhs), (lhs, rhs)
|
| - if hasattr(lhs, 'union'):
|
| - # Includes set, ConfigSettings and Configs.
|
| - return lhs.union(rhs)
|
| - if isinstance(lhs, dict):
|
| - return dict((k, union(lhs.get(k), rhs.get(k))) for k in set(lhs).union(rhs))
|
| - elif isinstance(lhs, list):
|
| - # Do not go inside the list.
|
| - return lhs + rhs
|
| - assert False, type(lhs)
|
| -
|
| -
|
| -def extract_comment(content):
|
| - """Extracts file level comment."""
|
| - out = []
|
| - for line in content.splitlines(True):
|
| - if line.startswith('#'):
|
| - out.append(line)
|
| - else:
|
| - break
|
| - return ''.join(out)
|
| -
|
| -
|
| -def eval_content(content):
|
| - """Evaluates a python file and return the value defined in it.
|
| -
|
| - Used in practice for .isolate files.
|
| - """
|
| - globs = {'__builtins__': None}
|
| - locs = {}
|
| - try:
|
| - value = eval(content, globs, locs)
|
| - except TypeError as e:
|
| - e.args = list(e.args) + [content]
|
| - raise
|
| - assert locs == {}, locs
|
| - assert globs == {'__builtins__': None}, globs
|
| - return value
|
| -
|
| -
|
| -def match_configs(expr, config_variables, all_configs):
|
| - """Returns the configs from |all_configs| that match the |expr|, where
|
| - the elements of |all_configs| are tuples of values for the |config_variables|.
|
| - Example:
|
| - >>> match_configs(expr = "(foo==1 or foo==2) and bar=='b'",
|
| - config_variables = ["foo", "bar"],
|
| - all_configs = [(1, 'a'), (1, 'b'), (2, 'a'), (2, 'b')])
|
| - [(1, 'b'), (2, 'b')]
|
| - """
|
| - return [
|
| - config for config in all_configs
|
| - if eval(expr, dict(zip(config_variables, config)))
|
| - ]
|
| -
|
| -
|
| -def verify_variables(variables):
|
| - """Verifies the |variables| dictionary is in the expected format."""
|
| - VALID_VARIABLES = [
|
| - KEY_TOUCHED,
|
| - KEY_TRACKED,
|
| - KEY_UNTRACKED,
|
| - 'command',
|
| - 'read_only',
|
| - ]
|
| - assert isinstance(variables, dict), variables
|
| - assert set(VALID_VARIABLES).issuperset(set(variables)), variables.keys()
|
| - for name, value in variables.iteritems():
|
| - if name == 'read_only':
|
| - assert value in (True, False, None), value
|
| - else:
|
| - assert isinstance(value, list), value
|
| - assert all(isinstance(i, basestring) for i in value), value
|
| -
|
| -
|
| -def verify_ast(expr, variables_and_values):
|
| - """Verifies that |expr| is of the form
|
| - expr ::= expr ( "or" | "and" ) expr
|
| - | identifier "==" ( string | int )
|
| - Also collects the variable identifiers and string/int values in the dict
|
| - |variables_and_values|, in the form {'var': set([val1, val2, ...]), ...}.
|
| - """
|
| - assert isinstance(expr, (ast.BoolOp, ast.Compare))
|
| - if isinstance(expr, ast.BoolOp):
|
| - assert isinstance(expr.op, (ast.And, ast.Or))
|
| - for subexpr in expr.values:
|
| - verify_ast(subexpr, variables_and_values)
|
| - else:
|
| - assert isinstance(expr.left.ctx, ast.Load)
|
| - assert len(expr.ops) == 1
|
| - assert isinstance(expr.ops[0], ast.Eq)
|
| - var_values = variables_and_values.setdefault(expr.left.id, set())
|
| - rhs = expr.comparators[0]
|
| - assert isinstance(rhs, (ast.Str, ast.Num))
|
| - var_values.add(rhs.n if isinstance(rhs, ast.Num) else rhs.s)
|
| -
|
| -
|
| -def verify_condition(condition, variables_and_values):
|
| - """Verifies the |condition| dictionary is in the expected format.
|
| - See verify_ast() for the meaning of |variables_and_values|.
|
| - """
|
| - VALID_INSIDE_CONDITION = ['variables']
|
| - assert isinstance(condition, list), condition
|
| - assert len(condition) == 2, condition
|
| - expr, then = condition
|
| -
|
| - test_ast = compile(expr, '<condition>', 'eval', ast.PyCF_ONLY_AST)
|
| - verify_ast(test_ast.body, variables_and_values)
|
| -
|
| - assert isinstance(then, dict), then
|
| - assert set(VALID_INSIDE_CONDITION).issuperset(set(then)), then.keys()
|
| - verify_variables(then['variables'])
|
| -
|
| -
|
| -def verify_root(value, variables_and_values):
|
| - """Verifies that |value| is the parsed form of a valid .isolate file.
|
| - See verify_ast() for the meaning of |variables_and_values|.
|
| - """
|
| - VALID_ROOTS = ['includes', 'conditions']
|
| - assert isinstance(value, dict), value
|
| - assert set(VALID_ROOTS).issuperset(set(value)), value.keys()
|
| -
|
| - includes = value.get('includes', [])
|
| - assert isinstance(includes, list), includes
|
| - for include in includes:
|
| - assert isinstance(include, basestring), include
|
| -
|
| - conditions = value.get('conditions', [])
|
| - assert isinstance(conditions, list), conditions
|
| - for condition in conditions:
|
| - verify_condition(condition, variables_and_values)
|
| -
|
| -
|
| -def remove_weak_dependencies(values, key, item, item_configs):
|
| - """Removes any configs from this key if the item is already under a
|
| - strong key.
|
| - """
|
| - if key == KEY_TOUCHED:
|
| - item_configs = set(item_configs)
|
| - for stronger_key in (KEY_TRACKED, KEY_UNTRACKED):
|
| - try:
|
| - item_configs -= values[stronger_key][item]
|
| - except KeyError:
|
| - pass
|
| -
|
| - return item_configs
|
| -
|
| -
|
| -def remove_repeated_dependencies(folders, key, item, item_configs):
|
| - """Removes any configs from this key if the item is in a folder that is
|
| - already included."""
|
| -
|
| - if key in (KEY_UNTRACKED, KEY_TRACKED, KEY_TOUCHED):
|
| - item_configs = set(item_configs)
|
| - for (folder, configs) in folders.iteritems():
|
| - if folder != item and item.startswith(folder):
|
| - item_configs -= configs
|
| -
|
| - return item_configs
|
| -
|
| -
|
| -def get_folders(values_dict):
|
| - """Returns a dict of all the folders in the given value_dict."""
|
| - return dict(
|
| - (item, configs) for (item, configs) in values_dict.iteritems()
|
| - if item.endswith('/')
|
| - )
|
| -
|
| -
|
| -def invert_map(variables):
|
| - """Converts {config: {deptype: list(depvals)}} to
|
| - {deptype: {depval: set(configs)}}.
|
| - """
|
| - KEYS = (
|
| - KEY_TOUCHED,
|
| - KEY_TRACKED,
|
| - KEY_UNTRACKED,
|
| - 'command',
|
| - 'read_only',
|
| - )
|
| - out = dict((key, {}) for key in KEYS)
|
| - for config, values in variables.iteritems():
|
| - for key in KEYS:
|
| - if key == 'command':
|
| - items = [tuple(values[key])] if key in values else []
|
| - elif key == 'read_only':
|
| - items = [values[key]] if key in values else []
|
| - else:
|
| - assert key in (KEY_TOUCHED, KEY_TRACKED, KEY_UNTRACKED)
|
| - items = values.get(key, [])
|
| - for item in items:
|
| - out[key].setdefault(item, set()).add(config)
|
| - return out
|
| -
|
| -
|
| -def reduce_inputs(values):
|
| - """Reduces the output of invert_map() to the strictest minimum list.
|
| -
|
| - Looks at each individual file and directory, maps where they are used and
|
| - reconstructs the inverse dictionary.
|
| -
|
| - Returns the minimized dictionary.
|
| - """
|
| - KEYS = (
|
| - KEY_TOUCHED,
|
| - KEY_TRACKED,
|
| - KEY_UNTRACKED,
|
| - 'command',
|
| - 'read_only',
|
| - )
|
| -
|
| - # Folders can only live in KEY_UNTRACKED.
|
| - folders = get_folders(values.get(KEY_UNTRACKED, {}))
|
| -
|
| - out = dict((key, {}) for key in KEYS)
|
| - for key in KEYS:
|
| - for item, item_configs in values.get(key, {}).iteritems():
|
| - item_configs = remove_weak_dependencies(values, key, item, item_configs)
|
| - item_configs = remove_repeated_dependencies(
|
| - folders, key, item, item_configs)
|
| - if item_configs:
|
| - out[key][item] = item_configs
|
| - return out
|
| -
|
| -
|
| -def convert_map_to_isolate_dict(values, config_variables):
|
| - """Regenerates back a .isolate configuration dict from files and dirs
|
| - mappings generated from reduce_inputs().
|
| - """
|
| - # Gather a list of configurations for set inversion later.
|
| - all_mentioned_configs = set()
|
| - for configs_by_item in values.itervalues():
|
| - for configs in configs_by_item.itervalues():
|
| - all_mentioned_configs.update(configs)
|
| -
|
| - # Invert the mapping to make it dict first.
|
| - conditions = {}
|
| - for key in values:
|
| - for item, configs in values[key].iteritems():
|
| - then = conditions.setdefault(frozenset(configs), {})
|
| - variables = then.setdefault('variables', {})
|
| -
|
| - if item in (True, False):
|
| - # One-off for read_only.
|
| - variables[key] = item
|
| - else:
|
| - assert item
|
| - if isinstance(item, tuple):
|
| - # One-off for command.
|
| - # Do not merge lists and do not sort!
|
| - # Note that item is a tuple.
|
| - assert key not in variables
|
| - variables[key] = list(item)
|
| - else:
|
| - # The list of items (files or dirs). Append the new item and keep
|
| - # the list sorted.
|
| - l = variables.setdefault(key, [])
|
| - l.append(item)
|
| - l.sort()
|
| -
|
| - if all_mentioned_configs:
|
| - config_values = map(set, zip(*all_mentioned_configs))
|
| - sef = short_expression_finder.ShortExpressionFinder(
|
| - zip(config_variables, config_values))
|
| -
|
| - conditions = sorted(
|
| - [sef.get_expr(configs), then] for configs, then in conditions.iteritems())
|
| - return {'conditions': conditions}
|
| -
|
| -
|
| -### Internal state files.
|
| -
|
| -
|
| -class ConfigSettings(object):
|
| - """Represents the dependency variables for a single build configuration.
|
| - The structure is immutable.
|
| - """
|
| - def __init__(self, config, values):
|
| - self.config = config
|
| - verify_variables(values)
|
| - self.touched = sorted(values.get(KEY_TOUCHED, []))
|
| - self.tracked = sorted(values.get(KEY_TRACKED, []))
|
| - self.untracked = sorted(values.get(KEY_UNTRACKED, []))
|
| - self.command = values.get('command', [])[:]
|
| - self.read_only = values.get('read_only')
|
| -
|
| - def union(self, rhs):
|
| - assert not (self.config and rhs.config) or (self.config == rhs.config)
|
| - assert not (self.command and rhs.command) or (self.command == rhs.command)
|
| - var = {
|
| - KEY_TOUCHED: sorted(self.touched + rhs.touched),
|
| - KEY_TRACKED: sorted(self.tracked + rhs.tracked),
|
| - KEY_UNTRACKED: sorted(self.untracked + rhs.untracked),
|
| - 'command': self.command or rhs.command,
|
| - 'read_only': rhs.read_only if self.read_only is None else self.read_only,
|
| - }
|
| - return ConfigSettings(self.config or rhs.config, var)
|
| -
|
| - def flatten(self):
|
| - out = {}
|
| - if self.command:
|
| - out['command'] = self.command
|
| - if self.touched:
|
| - out[KEY_TOUCHED] = self.touched
|
| - if self.tracked:
|
| - out[KEY_TRACKED] = self.tracked
|
| - if self.untracked:
|
| - out[KEY_UNTRACKED] = self.untracked
|
| - if self.read_only is not None:
|
| - out['read_only'] = self.read_only
|
| - return out
|
| -
|
| -
|
| -class Configs(object):
|
| - """Represents a processed .isolate file.
|
| -
|
| - Stores the file in a processed way, split by configuration.
|
| - """
|
| - def __init__(self, file_comment):
|
| - self.file_comment = file_comment
|
| - # The keys of by_config are tuples of values for the configuration
|
| - # variables. The names of the variables (which must be the same for
|
| - # every by_config key) are kept in config_variables. Initially by_config
|
| - # is empty and we don't know what configuration variables will be used,
|
| - # so config_variables also starts out empty. It will be set by the first
|
| - # call to union() or merge_dependencies().
|
| - self.by_config = {}
|
| - self.config_variables = ()
|
| -
|
| - def union(self, rhs):
|
| - """Adds variables from rhs (a Configs) to the existing variables.
|
| - """
|
| - config_variables = self.config_variables
|
| - if not config_variables:
|
| - config_variables = rhs.config_variables
|
| - else:
|
| - # We can't proceed if this isn't true since we don't know the correct
|
| - # default values for extra variables. The variables are sorted so we
|
| - # don't need to worry about permutations.
|
| - if rhs.config_variables and rhs.config_variables != config_variables:
|
| - raise ExecutionError(
|
| - 'Variables in merged .isolate files do not match: %r and %r' % (
|
| - config_variables, rhs.config_variables))
|
| -
|
| - # Takes the first file comment, prefering lhs.
|
| - out = Configs(self.file_comment or rhs.file_comment)
|
| - out.config_variables = config_variables
|
| - for config in set(self.by_config) | set(rhs.by_config):
|
| - out.by_config[config] = union(
|
| - self.by_config.get(config), rhs.by_config.get(config))
|
| - return out
|
| -
|
| - def merge_dependencies(self, values, config_variables, configs):
|
| - """Adds new dependencies to this object for the given configurations.
|
| - Arguments:
|
| - values: A variables dict as found in a .isolate file, e.g.,
|
| - {KEY_TOUCHED: [...], 'command': ...}.
|
| - config_variables: An ordered list of configuration variables, e.g.,
|
| - ["OS", "chromeos"]. If this object already contains any dependencies,
|
| - the configuration variables must match.
|
| - configs: a list of tuples of values of the configuration variables,
|
| - e.g., [("mac", 0), ("linux", 1)]. The dependencies in |values|
|
| - are added to all of these configurations, and other configurations
|
| - are unchanged.
|
| - """
|
| - if not values:
|
| - return
|
| -
|
| - if not self.config_variables:
|
| - self.config_variables = config_variables
|
| - else:
|
| - # See comment in Configs.union().
|
| - assert self.config_variables == config_variables
|
| -
|
| - for config in configs:
|
| - self.by_config[config] = union(
|
| - self.by_config.get(config), ConfigSettings(config, values))
|
| -
|
| - def flatten(self):
|
| - """Returns a flat dictionary representation of the configuration.
|
| - """
|
| - return dict((k, v.flatten()) for k, v in self.by_config.iteritems())
|
| -
|
| - def make_isolate_file(self):
|
| - """Returns a dictionary suitable for writing to a .isolate file.
|
| - """
|
| - dependencies_by_config = self.flatten()
|
| - configs_by_dependency = reduce_inputs(invert_map(dependencies_by_config))
|
| - return convert_map_to_isolate_dict(configs_by_dependency,
|
| - self.config_variables)
|
| -
|
| -
|
| -# TODO(benrg): Remove this function when no old-format files are left.
|
| -def convert_old_to_new_format(value):
|
| - """Converts from the old .isolate format, which only has one variable (OS),
|
| - always includes 'linux', 'mac' and 'win' in the set of valid values for OS,
|
| - and allows conditions that depend on the set of all OSes, to the new format,
|
| - which allows any set of variables, has no hardcoded values, and only allows
|
| - explicit positive tests of variable values.
|
| - """
|
| - conditions = value.get('conditions', [])
|
| - if 'variables' not in value and all(len(cond) == 2 for cond in conditions):
|
| - return value # Nothing to change
|
| -
|
| - def parse_condition(cond):
|
| - return re.match(r'OS=="(\w+)"\Z', cond[0]).group(1)
|
| -
|
| - oses = set(map(parse_condition, conditions))
|
| - default_oses = set(['linux', 'mac', 'win'])
|
| - oses = sorted(oses | default_oses)
|
| -
|
| - def if_not_os(not_os, then):
|
| - expr = ' or '.join('OS=="%s"' % os for os in oses if os != not_os)
|
| - return [expr, then]
|
| -
|
| - conditions = [
|
| - cond[:2] for cond in conditions if cond[1]
|
| - ] + [
|
| - if_not_os(parse_condition(cond), cond[2])
|
| - for cond in conditions if len(cond) == 3
|
| - ]
|
| -
|
| - if 'variables' in value:
|
| - conditions.append(if_not_os(None, {'variables': value.pop('variables')}))
|
| - conditions.sort()
|
| -
|
| - value = value.copy()
|
| - value['conditions'] = conditions
|
| - return value
|
| -
|
| -
|
| -def load_isolate_as_config(isolate_dir, value, file_comment):
|
| - """Parses one .isolate file and returns a Configs() instance.
|
| -
|
| - |value| is the loaded dictionary that was defined in the gyp file.
|
| -
|
| - The expected format is strict, anything diverting from the format below will
|
| - throw an assert:
|
| - {
|
| - 'includes': [
|
| - 'foo.isolate',
|
| - ],
|
| - 'conditions': [
|
| - ['OS=="vms" and foo=42', {
|
| - 'variables': {
|
| - 'command': [
|
| - ...
|
| - ],
|
| - 'isolate_dependency_tracked': [
|
| - ...
|
| - ],
|
| - 'isolate_dependency_untracked': [
|
| - ...
|
| - ],
|
| - 'read_only': False,
|
| - },
|
| - }],
|
| - ...
|
| - ],
|
| - }
|
| - """
|
| - value = convert_old_to_new_format(value)
|
| -
|
| - variables_and_values = {}
|
| - verify_root(value, variables_and_values)
|
| - if variables_and_values:
|
| - config_variables, config_values = zip(
|
| - *sorted(variables_and_values.iteritems()))
|
| - all_configs = list(itertools.product(*config_values))
|
| - else:
|
| - config_variables = None
|
| - all_configs = []
|
| -
|
| - isolate = Configs(file_comment)
|
| -
|
| - # Add configuration-specific variables.
|
| - for expr, then in value.get('conditions', []):
|
| - configs = match_configs(expr, config_variables, all_configs)
|
| - isolate.merge_dependencies(then['variables'], config_variables, configs)
|
| -
|
| - # Load the includes.
|
| - for include in value.get('includes', []):
|
| - if os.path.isabs(include):
|
| - raise ExecutionError(
|
| - 'Failed to load configuration; absolute include path \'%s\'' %
|
| - include)
|
| - included_isolate = os.path.normpath(os.path.join(isolate_dir, include))
|
| - with open(included_isolate, 'r') as f:
|
| - included_isolate = load_isolate_as_config(
|
| - os.path.dirname(included_isolate),
|
| - eval_content(f.read()),
|
| - None)
|
| - isolate = union(isolate, included_isolate)
|
| -
|
| - return isolate
|
| -
|
| -
|
| -def load_isolate_for_config(isolate_dir, content, variables):
|
| - """Loads the .isolate file and returns the information unprocessed but
|
| - filtered for the specific OS.
|
| -
|
| - Returns the command, dependencies and read_only flag. The dependencies are
|
| - fixed to use os.path.sep.
|
| - """
|
| - # Load the .isolate file, process its conditions, retrieve the command and
|
| - # dependencies.
|
| - isolate = load_isolate_as_config(isolate_dir, eval_content(content), None)
|
| - try:
|
| - config_name = tuple(variables[var] for var in isolate.config_variables)
|
| - except KeyError:
|
| - raise ExecutionError(
|
| - 'These configuration variables were missing from the command line: %s' %
|
| - ', '.join(sorted(set(isolate.config_variables) - set(variables))))
|
| - config = isolate.by_config.get(config_name)
|
| - if not config:
|
| - raise ExecutionError(
|
| - 'Failed to load configuration for variable \'%s\' for config(s) \'%s\''
|
| - '\nAvailable configs: %s' %
|
| - (', '.join(isolate.config_variables),
|
| - ', '.join(config_name),
|
| - ', '.join(str(s) for s in isolate.by_config)))
|
| - # Merge tracked and untracked variables, isolate.py doesn't care about the
|
| - # trackability of the variables, only the build tool does.
|
| - dependencies = [
|
| - f.replace('/', os.path.sep) for f in config.tracked + config.untracked
|
| - ]
|
| - touched = [f.replace('/', os.path.sep) for f in config.touched]
|
| - return config.command, dependencies, touched, config.read_only
|
| -
|
| -
|
| -def save_isolated(isolated, data):
|
| - """Writes one or multiple .isolated files.
|
| -
|
| - Note: this reference implementation does not create child .isolated file so it
|
| - always returns an empty list.
|
| -
|
| - Returns the list of child isolated files that are included by |isolated|.
|
| - """
|
| - trace_inputs.write_json(isolated, data, True)
|
| - return []
|
| -
|
| -
|
| -def chromium_save_isolated(isolated, data, variables, algo):
|
| - """Writes one or many .isolated files.
|
| -
|
| - This slightly increases the cold cache cost but greatly reduce the warm cache
|
| - cost by splitting low-churn files off the master .isolated file. It also
|
| - reduces overall isolateserver memcache consumption.
|
| - """
|
| - slaves = []
|
| -
|
| - def extract_into_included_isolated(prefix):
|
| - new_slave = {
|
| - 'algo': data['algo'],
|
| - 'files': {},
|
| - 'os': data['os'],
|
| - 'version': data['version'],
|
| - }
|
| - for f in data['files'].keys():
|
| - if f.startswith(prefix):
|
| - new_slave['files'][f] = data['files'].pop(f)
|
| - if new_slave['files']:
|
| - slaves.append(new_slave)
|
| -
|
| - # Split test/data/ in its own .isolated file.
|
| - extract_into_included_isolated(os.path.join('test', 'data', ''))
|
| -
|
| - # Split everything out of PRODUCT_DIR in its own .isolated file.
|
| - if variables.get('PRODUCT_DIR'):
|
| - extract_into_included_isolated(variables['PRODUCT_DIR'])
|
| -
|
| - files = []
|
| - for index, f in enumerate(slaves):
|
| - slavepath = isolated[:-len('.isolated')] + '.%d.isolated' % index
|
| - trace_inputs.write_json(slavepath, f, True)
|
| - data.setdefault('includes', []).append(
|
| - isolateserver.hash_file(slavepath, algo))
|
| - files.append(os.path.basename(slavepath))
|
| -
|
| - files.extend(save_isolated(isolated, data))
|
| - return files
|
| -
|
| -
|
| -class Flattenable(object):
|
| - """Represents data that can be represented as a json file."""
|
| - MEMBERS = ()
|
| -
|
| - def flatten(self):
|
| - """Returns a json-serializable version of itself.
|
| -
|
| - Skips None entries.
|
| - """
|
| - items = ((member, getattr(self, member)) for member in self.MEMBERS)
|
| - return dict((member, value) for member, value in items if value is not None)
|
| -
|
| - @classmethod
|
| - def load(cls, data, *args, **kwargs):
|
| - """Loads a flattened version."""
|
| - data = data.copy()
|
| - out = cls(*args, **kwargs)
|
| - for member in out.MEMBERS:
|
| - if member in data:
|
| - # Access to a protected member XXX of a client class
|
| - # pylint: disable=W0212
|
| - out._load_member(member, data.pop(member))
|
| - if data:
|
| - raise ValueError(
|
| - 'Found unexpected entry %s while constructing an object %s' %
|
| - (data, cls.__name__), data, cls.__name__)
|
| - return out
|
| -
|
| - def _load_member(self, member, value):
|
| - """Loads a member into self."""
|
| - setattr(self, member, value)
|
| -
|
| - @classmethod
|
| - def load_file(cls, filename, *args, **kwargs):
|
| - """Loads the data from a file or return an empty instance."""
|
| - try:
|
| - out = cls.load(trace_inputs.read_json(filename), *args, **kwargs)
|
| - logging.debug('Loaded %s(%s)', cls.__name__, filename)
|
| - except (IOError, ValueError) as e:
|
| - # On failure, loads the default instance.
|
| - out = cls(*args, **kwargs)
|
| - logging.warn('Failed to load %s: %s', filename, e)
|
| - return out
|
| -
|
| -
|
| -class SavedState(Flattenable):
|
| - """Describes the content of a .state file.
|
| -
|
| - This file caches the items calculated by this script and is used to increase
|
| - the performance of the script. This file is not loaded by run_isolated.py.
|
| - This file can always be safely removed.
|
| -
|
| - It is important to note that the 'files' dict keys are using native OS path
|
| - separator instead of '/' used in .isolate file.
|
| - """
|
| - MEMBERS = (
|
| - # Algorithm used to generate the hash. The only supported value is at the
|
| - # time of writting 'sha-1'.
|
| - 'algo',
|
| - # Cache of the processed command. This value is saved because .isolated
|
| - # files are never loaded by isolate.py so it's the only way to load the
|
| - # command safely.
|
| - 'command',
|
| - # Cache of the files found so the next run can skip hash calculation.
|
| - 'files',
|
| - # Path of the original .isolate file. Relative path to isolated_basedir.
|
| - 'isolate_file',
|
| - # List of included .isolated files. Used to support/remember 'slave'
|
| - # .isolated files. Relative path to isolated_basedir.
|
| - 'child_isolated_files',
|
| - # If the generated directory tree should be read-only.
|
| - 'read_only',
|
| - # Relative cwd to use to start the command.
|
| - 'relative_cwd',
|
| - # GYP variables used to generate the .isolated file. Variables are saved so
|
| - # a user can use isolate.py after building and the GYP variables are still
|
| - # defined.
|
| - 'variables',
|
| - # Version of the file format in format 'major.minor'. Any non-breaking
|
| - # change must update minor. Any breaking change must update major.
|
| - 'version',
|
| - )
|
| -
|
| - def __init__(self, isolated_basedir):
|
| - """Creates an empty SavedState.
|
| -
|
| - |isolated_basedir| is the directory where the .isolated and .isolated.state
|
| - files are saved.
|
| - """
|
| - super(SavedState, self).__init__()
|
| - assert os.path.isabs(isolated_basedir), isolated_basedir
|
| - assert os.path.isdir(isolated_basedir), isolated_basedir
|
| - self.isolated_basedir = isolated_basedir
|
| -
|
| - # The default algorithm used.
|
| - self.algo = isolateserver.SUPPORTED_ALGOS['sha-1']
|
| - self.command = []
|
| - self.files = {}
|
| - self.isolate_file = None
|
| - self.child_isolated_files = []
|
| - self.read_only = None
|
| - self.relative_cwd = None
|
| - self.variables = {'OS': get_flavor()}
|
| - # The current version.
|
| - self.version = '1.0'
|
| -
|
| - def update(self, isolate_file, variables):
|
| - """Updates the saved state with new data to keep GYP variables and internal
|
| - reference to the original .isolate file.
|
| - """
|
| - assert os.path.isabs(isolate_file)
|
| - # Convert back to a relative path. On Windows, if the isolate and
|
| - # isolated files are on different drives, isolate_file will stay an absolute
|
| - # path.
|
| - isolate_file = safe_relpath(isolate_file, self.isolated_basedir)
|
| -
|
| - # The same .isolate file should always be used to generate the .isolated and
|
| - # .isolated.state.
|
| - assert isolate_file == self.isolate_file or not self.isolate_file, (
|
| - isolate_file, self.isolate_file)
|
| - self.isolate_file = isolate_file
|
| - self.variables.update(variables)
|
| -
|
| - def update_isolated(self, command, infiles, touched, read_only, relative_cwd):
|
| - """Updates the saved state with data necessary to generate a .isolated file.
|
| -
|
| - The new files in |infiles| are added to self.files dict but their hash is
|
| - not calculated here.
|
| - """
|
| - self.command = command
|
| - # Add new files.
|
| - for f in infiles:
|
| - self.files.setdefault(f, {})
|
| - for f in touched:
|
| - self.files.setdefault(f, {})['T'] = True
|
| - # Prune extraneous files that are not a dependency anymore.
|
| - for f in set(self.files).difference(set(infiles).union(touched)):
|
| - del self.files[f]
|
| - if read_only is not None:
|
| - self.read_only = read_only
|
| - self.relative_cwd = relative_cwd
|
| -
|
| - def to_isolated(self):
|
| - """Creates a .isolated dictionary out of the saved state.
|
| -
|
| - https://code.google.com/p/swarming/wiki/IsolatedDesign
|
| - """
|
| - def strip(data):
|
| - """Returns a 'files' entry with only the whitelisted keys."""
|
| - return dict((k, data[k]) for k in ('h', 'l', 'm', 's') if k in data)
|
| -
|
| - out = {
|
| - 'algo': isolateserver.SUPPORTED_ALGOS_REVERSE[self.algo],
|
| - 'files': dict(
|
| - (filepath, strip(data)) for filepath, data in self.files.iteritems()),
|
| - 'os': self.variables['OS'],
|
| - 'version': self.version,
|
| - }
|
| - if self.command:
|
| - out['command'] = self.command
|
| - if self.read_only is not None:
|
| - out['read_only'] = self.read_only
|
| - if self.relative_cwd:
|
| - out['relative_cwd'] = self.relative_cwd
|
| - return out
|
| -
|
| - @property
|
| - def isolate_filepath(self):
|
| - """Returns the absolute path of self.isolate_file."""
|
| - return os.path.normpath(
|
| - os.path.join(self.isolated_basedir, self.isolate_file))
|
| -
|
| - # Arguments number differs from overridden method
|
| - @classmethod
|
| - def load(cls, data, isolated_basedir): # pylint: disable=W0221
|
| - """Special case loading to disallow different OS.
|
| -
|
| - It is not possible to load a .isolated.state files from a different OS, this
|
| - file is saved in OS-specific format.
|
| - """
|
| - out = super(SavedState, cls).load(data, isolated_basedir)
|
| - if 'os' in data:
|
| - out.variables['OS'] = data['os']
|
| -
|
| - # Converts human readable form back into the proper class type.
|
| - algo = data.get('algo', 'sha-1')
|
| - if not algo in isolateserver.SUPPORTED_ALGOS:
|
| - raise isolateserver.ConfigError('Unknown algo \'%s\'' % out.algo)
|
| - out.algo = isolateserver.SUPPORTED_ALGOS[algo]
|
| -
|
| - # For example, 1.1 is guaranteed to be backward compatible with 1.0 code.
|
| - if not re.match(r'^(\d+)\.(\d+)$', out.version):
|
| - raise isolateserver.ConfigError('Unknown version \'%s\'' % out.version)
|
| - if out.version.split('.', 1)[0] != '1':
|
| - raise isolateserver.ConfigError(
|
| - 'Unsupported version \'%s\'' % out.version)
|
| -
|
| - # The .isolate file must be valid. It could be absolute on Windows if the
|
| - # drive containing the .isolate and the drive containing the .isolated files
|
| - # differ.
|
| - assert not os.path.isabs(out.isolate_file) or sys.platform == 'win32'
|
| - assert os.path.isfile(out.isolate_filepath), out.isolate_filepath
|
| - return out
|
| -
|
| - def flatten(self):
|
| - """Makes sure 'algo' is in human readable form."""
|
| - out = super(SavedState, self).flatten()
|
| - out['algo'] = isolateserver.SUPPORTED_ALGOS_REVERSE[out['algo']]
|
| - return out
|
| -
|
| - def __str__(self):
|
| - out = '%s(\n' % self.__class__.__name__
|
| - out += ' command: %s\n' % self.command
|
| - out += ' files: %d\n' % len(self.files)
|
| - out += ' isolate_file: %s\n' % self.isolate_file
|
| - out += ' read_only: %s\n' % self.read_only
|
| - out += ' relative_cwd: %s\n' % self.relative_cwd
|
| - out += ' child_isolated_files: %s\n' % self.child_isolated_files
|
| - out += ' variables: %s' % ''.join(
|
| - '\n %s=%s' % (k, self.variables[k]) for k in sorted(self.variables))
|
| - out += ')'
|
| - return out
|
| -
|
| -
|
| -class CompleteState(object):
|
| - """Contains all the state to run the task at hand."""
|
| - def __init__(self, isolated_filepath, saved_state):
|
| - super(CompleteState, self).__init__()
|
| - assert isolated_filepath is None or os.path.isabs(isolated_filepath)
|
| - self.isolated_filepath = isolated_filepath
|
| - # Contains the data to ease developer's use-case but that is not strictly
|
| - # necessary.
|
| - self.saved_state = saved_state
|
| -
|
| - @classmethod
|
| - def load_files(cls, isolated_filepath):
|
| - """Loads state from disk."""
|
| - assert os.path.isabs(isolated_filepath), isolated_filepath
|
| - isolated_basedir = os.path.dirname(isolated_filepath)
|
| - return cls(
|
| - isolated_filepath,
|
| - SavedState.load_file(
|
| - isolatedfile_to_state(isolated_filepath), isolated_basedir))
|
| -
|
| - def load_isolate(self, cwd, isolate_file, variables, ignore_broken_items):
|
| - """Updates self.isolated and self.saved_state with information loaded from a
|
| - .isolate file.
|
| -
|
| - Processes the loaded data, deduce root_dir, relative_cwd.
|
| - """
|
| - # Make sure to not depend on os.getcwd().
|
| - assert os.path.isabs(isolate_file), isolate_file
|
| - isolate_file = file_path.get_native_path_case(isolate_file)
|
| - logging.info(
|
| - 'CompleteState.load_isolate(%s, %s, %s, %s)',
|
| - cwd, isolate_file, variables, ignore_broken_items)
|
| - relative_base_dir = os.path.dirname(isolate_file)
|
| -
|
| - # Processes the variables and update the saved state.
|
| - variables = process_variables(cwd, variables, relative_base_dir)
|
| - self.saved_state.update(isolate_file, variables)
|
| - variables = self.saved_state.variables
|
| -
|
| - with open(isolate_file, 'r') as f:
|
| - # At that point, variables are not replaced yet in command and infiles.
|
| - # infiles may contain directory entries and is in posix style.
|
| - command, infiles, touched, read_only = load_isolate_for_config(
|
| - os.path.dirname(isolate_file), f.read(), variables)
|
| - command = [eval_variables(i, variables) for i in command]
|
| - infiles = [eval_variables(f, variables) for f in infiles]
|
| - touched = [eval_variables(f, variables) for f in touched]
|
| - # root_dir is automatically determined by the deepest root accessed with the
|
| - # form '../../foo/bar'. Note that path variables must be taken in account
|
| - # too, add them as if they were input files.
|
| - path_variables = [variables[v] for v in PATH_VARIABLES if v in variables]
|
| - root_dir = determine_root_dir(
|
| - relative_base_dir, infiles + touched + path_variables)
|
| - # The relative directory is automatically determined by the relative path
|
| - # between root_dir and the directory containing the .isolate file,
|
| - # isolate_base_dir.
|
| - relative_cwd = os.path.relpath(relative_base_dir, root_dir)
|
| - # Now that we know where the root is, check that the PATH_VARIABLES point
|
| - # inside it.
|
| - for i in PATH_VARIABLES:
|
| - if i in variables:
|
| - if not path_starts_with(
|
| - root_dir, os.path.join(relative_base_dir, variables[i])):
|
| - raise isolateserver.MappingError(
|
| - 'Path variable %s=%r points outside the inferred root directory'
|
| - ' %s' % (i, variables[i], root_dir))
|
| - # Normalize the files based to root_dir. It is important to keep the
|
| - # trailing os.path.sep at that step.
|
| - infiles = [
|
| - relpath(normpath(os.path.join(relative_base_dir, f)), root_dir)
|
| - for f in infiles
|
| - ]
|
| - touched = [
|
| - relpath(normpath(os.path.join(relative_base_dir, f)), root_dir)
|
| - for f in touched
|
| - ]
|
| - follow_symlinks = variables['OS'] != 'win'
|
| - # Expand the directories by listing each file inside. Up to now, trailing
|
| - # os.path.sep must be kept. Do not expand 'touched'.
|
| - infiles = expand_directories_and_symlinks(
|
| - root_dir,
|
| - infiles,
|
| - lambda x: re.match(r'.*\.(git|svn|pyc)$', x),
|
| - follow_symlinks,
|
| - ignore_broken_items)
|
| -
|
| - # If we ignore broken items then remove any missing touched items.
|
| - if ignore_broken_items:
|
| - original_touched_count = len(touched)
|
| - touched = [touch for touch in touched if os.path.exists(touch)]
|
| -
|
| - if len(touched) != original_touched_count:
|
| - logging.info('Removed %d invalid touched entries',
|
| - len(touched) - original_touched_count)
|
| -
|
| - # Finally, update the new data to be able to generate the foo.isolated file,
|
| - # the file that is used by run_isolated.py.
|
| - self.saved_state.update_isolated(
|
| - command, infiles, touched, read_only, relative_cwd)
|
| - logging.debug(self)
|
| -
|
| - def process_inputs(self, subdir):
|
| - """Updates self.saved_state.files with the files' mode and hash.
|
| -
|
| - If |subdir| is specified, filters to a subdirectory. The resulting .isolated
|
| - file is tainted.
|
| -
|
| - See process_input() for more information.
|
| - """
|
| - for infile in sorted(self.saved_state.files):
|
| - if subdir and not infile.startswith(subdir):
|
| - self.saved_state.files.pop(infile)
|
| - else:
|
| - filepath = os.path.join(self.root_dir, infile)
|
| - self.saved_state.files[infile] = process_input(
|
| - filepath,
|
| - self.saved_state.files[infile],
|
| - self.saved_state.read_only,
|
| - self.saved_state.variables['OS'],
|
| - self.saved_state.algo)
|
| -
|
| - def save_files(self):
|
| - """Saves self.saved_state and creates a .isolated file."""
|
| - logging.debug('Dumping to %s' % self.isolated_filepath)
|
| - self.saved_state.child_isolated_files = chromium_save_isolated(
|
| - self.isolated_filepath,
|
| - self.saved_state.to_isolated(),
|
| - self.saved_state.variables,
|
| - self.saved_state.algo)
|
| - total_bytes = sum(
|
| - i.get('s', 0) for i in self.saved_state.files.itervalues())
|
| - if total_bytes:
|
| - # TODO(maruel): Stats are missing the .isolated files.
|
| - logging.debug('Total size: %d bytes' % total_bytes)
|
| - saved_state_file = isolatedfile_to_state(self.isolated_filepath)
|
| - logging.debug('Dumping to %s' % saved_state_file)
|
| - trace_inputs.write_json(saved_state_file, self.saved_state.flatten(), True)
|
| -
|
| - @property
|
| - def root_dir(self):
|
| - """Returns the absolute path of the root_dir to reference the .isolate file
|
| - via relative_cwd.
|
| -
|
| - So that join(root_dir, relative_cwd, basename(isolate_file)) is equivalent
|
| - to isolate_filepath.
|
| - """
|
| - if not self.saved_state.isolate_file:
|
| - raise ExecutionError('Please specify --isolate')
|
| - isolate_dir = os.path.dirname(self.saved_state.isolate_filepath)
|
| - # Special case '.'.
|
| - if self.saved_state.relative_cwd == '.':
|
| - root_dir = isolate_dir
|
| - else:
|
| - if not isolate_dir.endswith(self.saved_state.relative_cwd):
|
| - raise ExecutionError(
|
| - ('Make sure the .isolate file is in the directory that will be '
|
| - 'used as the relative directory. It is currently in %s and should '
|
| - 'be in %s') % (isolate_dir, self.saved_state.relative_cwd))
|
| - # Walk back back to the root directory.
|
| - root_dir = isolate_dir[:-(len(self.saved_state.relative_cwd) + 1)]
|
| - return file_path.get_native_path_case(root_dir)
|
| -
|
| - @property
|
| - def resultdir(self):
|
| - """Returns the absolute path containing the .isolated file.
|
| -
|
| - It is usually equivalent to the variable PRODUCT_DIR. Uses the .isolated
|
| - path as the value.
|
| - """
|
| - return os.path.dirname(self.isolated_filepath)
|
| -
|
| - def __str__(self):
|
| - def indent(data, indent_length):
|
| - """Indents text."""
|
| - spacing = ' ' * indent_length
|
| - return ''.join(spacing + l for l in str(data).splitlines(True))
|
| -
|
| - out = '%s(\n' % self.__class__.__name__
|
| - out += ' root_dir: %s\n' % self.root_dir
|
| - out += ' saved_state: %s)' % indent(self.saved_state, 2)
|
| - return out
|
| -
|
| -
|
| -def load_complete_state(options, cwd, subdir, skip_update):
|
| - """Loads a CompleteState.
|
| -
|
| - This includes data from .isolate and .isolated.state files. Never reads the
|
| - .isolated file.
|
| -
|
| - Arguments:
|
| - options: Options instance generated with OptionParserIsolate. For either
|
| - options.isolate and options.isolated, if the value is set, it is an
|
| - absolute path.
|
| - cwd: base directory to be used when loading the .isolate file.
|
| - subdir: optional argument to only process file in the subdirectory, relative
|
| - to CompleteState.root_dir.
|
| - skip_update: Skip trying to load the .isolate file and processing the
|
| - dependencies. It is useful when not needed, like when tracing.
|
| - """
|
| - assert not options.isolate or os.path.isabs(options.isolate)
|
| - assert not options.isolated or os.path.isabs(options.isolated)
|
| - cwd = file_path.get_native_path_case(unicode(cwd))
|
| - if options.isolated:
|
| - # Load the previous state if it was present. Namely, "foo.isolated.state".
|
| - # Note: this call doesn't load the .isolate file.
|
| - complete_state = CompleteState.load_files(options.isolated)
|
| - else:
|
| - # Constructs a dummy object that cannot be saved. Useful for temporary
|
| - # commands like 'run'.
|
| - complete_state = CompleteState(None, SavedState())
|
| -
|
| - if not options.isolate:
|
| - if not complete_state.saved_state.isolate_file:
|
| - if not skip_update:
|
| - raise ExecutionError('A .isolate file is required.')
|
| - isolate = None
|
| - else:
|
| - isolate = complete_state.saved_state.isolate_filepath
|
| - else:
|
| - isolate = options.isolate
|
| - if complete_state.saved_state.isolate_file:
|
| - rel_isolate = safe_relpath(
|
| - options.isolate, complete_state.saved_state.isolated_basedir)
|
| - if rel_isolate != complete_state.saved_state.isolate_file:
|
| - raise ExecutionError(
|
| - '%s and %s do not match.' % (
|
| - options.isolate, complete_state.saved_state.isolate_file))
|
| -
|
| - if not skip_update:
|
| - # Then load the .isolate and expands directories.
|
| - complete_state.load_isolate(
|
| - cwd, isolate, options.variables, options.ignore_broken_items)
|
| -
|
| - # Regenerate complete_state.saved_state.files.
|
| - if subdir:
|
| - subdir = unicode(subdir)
|
| - subdir = eval_variables(subdir, complete_state.saved_state.variables)
|
| - subdir = subdir.replace('/', os.path.sep)
|
| -
|
| - if not skip_update:
|
| - complete_state.process_inputs(subdir)
|
| - return complete_state
|
| -
|
| -
|
| -def read_trace_as_isolate_dict(complete_state, trace_blacklist):
|
| - """Reads a trace and returns the .isolate dictionary.
|
| -
|
| - Returns exceptions during the log parsing so it can be re-raised.
|
| - """
|
| - api = trace_inputs.get_api()
|
| - logfile = complete_state.isolated_filepath + '.log'
|
| - if not os.path.isfile(logfile):
|
| - raise ExecutionError(
|
| - 'No log file \'%s\' to read, did you forget to \'trace\'?' % logfile)
|
| - try:
|
| - data = api.parse_log(logfile, trace_blacklist, None)
|
| - exceptions = [i['exception'] for i in data if 'exception' in i]
|
| - results = (i['results'] for i in data if 'results' in i)
|
| - results_stripped = (i.strip_root(complete_state.root_dir) for i in results)
|
| - files = set(sum((result.existent for result in results_stripped), []))
|
| - tracked, touched = split_touched(files)
|
| - value = generate_isolate(
|
| - tracked,
|
| - [],
|
| - touched,
|
| - complete_state.root_dir,
|
| - complete_state.saved_state.variables,
|
| - complete_state.saved_state.relative_cwd,
|
| - trace_blacklist)
|
| - return value, exceptions
|
| - except trace_inputs.TracingFailure, e:
|
| - raise ExecutionError(
|
| - 'Reading traces failed for: %s\n%s' %
|
| - (' '.join(complete_state.saved_state.command), str(e)))
|
| -
|
| -
|
| -def print_all(comment, data, stream):
|
| - """Prints a complete .isolate file and its top-level file comment into a
|
| - stream.
|
| - """
|
| - if comment:
|
| - stream.write(comment)
|
| - pretty_print(data, stream)
|
| -
|
| -
|
| -def merge(complete_state, trace_blacklist):
|
| - """Reads a trace and merges it back into the source .isolate file."""
|
| - value, exceptions = read_trace_as_isolate_dict(
|
| - complete_state, trace_blacklist)
|
| -
|
| - # Now take that data and union it into the original .isolate file.
|
| - with open(complete_state.saved_state.isolate_filepath, 'r') as f:
|
| - prev_content = f.read()
|
| - isolate_dir = os.path.dirname(complete_state.saved_state.isolate_filepath)
|
| - prev_config = load_isolate_as_config(
|
| - isolate_dir,
|
| - eval_content(prev_content),
|
| - extract_comment(prev_content))
|
| - new_config = load_isolate_as_config(isolate_dir, value, '')
|
| - config = union(prev_config, new_config)
|
| - data = config.make_isolate_file()
|
| - print('Updating %s' % complete_state.saved_state.isolate_file)
|
| - with open(complete_state.saved_state.isolate_filepath, 'wb') as f:
|
| - print_all(config.file_comment, data, f)
|
| - if exceptions:
|
| - # It got an exception, raise the first one.
|
| - raise \
|
| - exceptions[0][0], \
|
| - exceptions[0][1], \
|
| - exceptions[0][2]
|
| -
|
| -
|
| -### Commands.
|
| -
|
| -
|
| -def CMDarchive(parser, args):
|
| - """Creates a .isolated file and uploads the tree to an isolate server.
|
| -
|
| - All the files listed in the .isolated file are put in the isolate server
|
| - cache via isolateserver.py.
|
| - """
|
| - parser.add_option('--subdir', help='Filters to a subdirectory')
|
| - options, args = parser.parse_args(args)
|
| - if args:
|
| - parser.error('Unsupported argument: %s' % args)
|
| -
|
| - with tools.Profiler('GenerateHashtable'):
|
| - success = False
|
| - try:
|
| - complete_state = load_complete_state(
|
| - options, os.getcwd(), options.subdir, False)
|
| - if not options.outdir:
|
| - options.outdir = os.path.join(
|
| - os.path.dirname(complete_state.isolated_filepath), 'hashtable')
|
| - # Make sure that complete_state isn't modified until save_files() is
|
| - # called, because any changes made to it here will propagate to the files
|
| - # created (which is probably not intended).
|
| - complete_state.save_files()
|
| -
|
| - infiles = complete_state.saved_state.files
|
| - # Add all the .isolated files.
|
| - isolated_hash = []
|
| - isolated_files = [
|
| - options.isolated,
|
| - ] + complete_state.saved_state.child_isolated_files
|
| - for item in isolated_files:
|
| - item_path = os.path.join(
|
| - os.path.dirname(complete_state.isolated_filepath), item)
|
| - # Do not use isolateserver.hash_file() here because the file is
|
| - # likely smallish (under 500kb) and its file size is needed.
|
| - with open(item_path, 'rb') as f:
|
| - content = f.read()
|
| - isolated_hash.append(
|
| - complete_state.saved_state.algo(content).hexdigest())
|
| - isolated_metadata = {
|
| - 'h': isolated_hash[-1],
|
| - 's': len(content),
|
| - 'priority': '0'
|
| - }
|
| - infiles[item_path] = isolated_metadata
|
| -
|
| - logging.info('Creating content addressed object store with %d item',
|
| - len(infiles))
|
| -
|
| - if is_url(options.outdir):
|
| - isolateserver.upload_tree(
|
| - base_url=options.outdir,
|
| - indir=complete_state.root_dir,
|
| - infiles=infiles,
|
| - namespace='default-gzip')
|
| - else:
|
| - recreate_tree(
|
| - outdir=options.outdir,
|
| - indir=complete_state.root_dir,
|
| - infiles=infiles,
|
| - action=run_isolated.HARDLINK_WITH_FALLBACK,
|
| - as_hash=True)
|
| - success = True
|
| - print('%s %s' % (isolated_hash[0], os.path.basename(options.isolated)))
|
| - finally:
|
| - # If the command failed, delete the .isolated file if it exists. This is
|
| - # important so no stale swarm job is executed.
|
| - if not success and os.path.isfile(options.isolated):
|
| - os.remove(options.isolated)
|
| - return not success
|
| -
|
| -
|
| -def CMDcheck(parser, args):
|
| - """Checks that all the inputs are present and generates .isolated."""
|
| - parser.add_option('--subdir', help='Filters to a subdirectory')
|
| - options, args = parser.parse_args(args)
|
| - if args:
|
| - parser.error('Unsupported argument: %s' % args)
|
| -
|
| - complete_state = load_complete_state(
|
| - options, os.getcwd(), options.subdir, False)
|
| -
|
| - # Nothing is done specifically. Just store the result and state.
|
| - complete_state.save_files()
|
| - return 0
|
| -
|
| -
|
| -CMDhashtable = CMDarchive
|
| -
|
| -
|
| -def CMDmerge(parser, args):
|
| - """Reads and merges the data from the trace back into the original .isolate.
|
| -
|
| - Ignores --outdir.
|
| - """
|
| - parser.require_isolated = False
|
| - add_trace_option(parser)
|
| - options, args = parser.parse_args(args)
|
| - if args:
|
| - parser.error('Unsupported argument: %s' % args)
|
| -
|
| - complete_state = load_complete_state(options, os.getcwd(), None, False)
|
| - blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
|
| - merge(complete_state, blacklist)
|
| - return 0
|
| -
|
| -
|
| -def CMDread(parser, args):
|
| - """Reads the trace file generated with command 'trace'.
|
| -
|
| - Ignores --outdir.
|
| - """
|
| - parser.require_isolated = False
|
| - add_trace_option(parser)
|
| - parser.add_option(
|
| - '--skip-refresh', action='store_true',
|
| - help='Skip reading .isolate file and do not refresh the hash of '
|
| - 'dependencies')
|
| - parser.add_option(
|
| - '-m', '--merge', action='store_true',
|
| - help='merge the results back in the .isolate file instead of printing')
|
| - options, args = parser.parse_args(args)
|
| - if args:
|
| - parser.error('Unsupported argument: %s' % args)
|
| -
|
| - complete_state = load_complete_state(
|
| - options, os.getcwd(), None, options.skip_refresh)
|
| - blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
|
| - value, exceptions = read_trace_as_isolate_dict(complete_state, blacklist)
|
| - if options.merge:
|
| - merge(complete_state, blacklist)
|
| - else:
|
| - pretty_print(value, sys.stdout)
|
| -
|
| - if exceptions:
|
| - # It got an exception, raise the first one.
|
| - raise \
|
| - exceptions[0][0], \
|
| - exceptions[0][1], \
|
| - exceptions[0][2]
|
| - return 0
|
| -
|
| -
|
| -def CMDremap(parser, args):
|
| - """Creates a directory with all the dependencies mapped into it.
|
| -
|
| - Useful to test manually why a test is failing. The target executable is not
|
| - run.
|
| - """
|
| - parser.require_isolated = False
|
| - options, args = parser.parse_args(args)
|
| - if args:
|
| - parser.error('Unsupported argument: %s' % args)
|
| - complete_state = load_complete_state(options, os.getcwd(), None, False)
|
| -
|
| - if not options.outdir:
|
| - options.outdir = run_isolated.make_temp_dir(
|
| - 'isolate', complete_state.root_dir)
|
| - else:
|
| - if is_url(options.outdir):
|
| - parser.error('Can\'t use url for --outdir with mode remap.')
|
| - if not os.path.isdir(options.outdir):
|
| - os.makedirs(options.outdir)
|
| - print('Remapping into %s' % options.outdir)
|
| - if len(os.listdir(options.outdir)):
|
| - raise ExecutionError('Can\'t remap in a non-empty directory')
|
| - recreate_tree(
|
| - outdir=options.outdir,
|
| - indir=complete_state.root_dir,
|
| - infiles=complete_state.saved_state.files,
|
| - action=run_isolated.HARDLINK_WITH_FALLBACK,
|
| - as_hash=False)
|
| - if complete_state.saved_state.read_only:
|
| - run_isolated.make_writable(options.outdir, True)
|
| -
|
| - if complete_state.isolated_filepath:
|
| - complete_state.save_files()
|
| - return 0
|
| -
|
| -
|
| -def CMDrewrite(parser, args):
|
| - """Rewrites a .isolate file into the canonical format."""
|
| - parser.require_isolated = False
|
| - options, args = parser.parse_args(args)
|
| - if args:
|
| - parser.error('Unsupported argument: %s' % args)
|
| -
|
| - if options.isolated:
|
| - # Load the previous state if it was present. Namely, "foo.isolated.state".
|
| - complete_state = CompleteState.load_files(options.isolated)
|
| - isolate = options.isolate or complete_state.saved_state.isolate_filepath
|
| - else:
|
| - isolate = options.isolate
|
| - if not isolate:
|
| - parser.error('--isolate is required.')
|
| -
|
| - with open(isolate, 'r') as f:
|
| - content = f.read()
|
| - config = load_isolate_as_config(
|
| - os.path.dirname(os.path.abspath(isolate)),
|
| - eval_content(content),
|
| - extract_comment(content))
|
| - data = config.make_isolate_file()
|
| - print('Updating %s' % isolate)
|
| - with open(isolate, 'wb') as f:
|
| - print_all(config.file_comment, data, f)
|
| - return 0
|
| -
|
| -
|
| -@subcommand.usage('-- [extra arguments]')
|
| -def CMDrun(parser, args):
|
| - """Runs the test executable in an isolated (temporary) directory.
|
| -
|
| - All the dependencies are mapped into the temporary directory and the
|
| - directory is cleaned up after the target exits. Warning: if --outdir is
|
| - specified, it is deleted upon exit.
|
| -
|
| - Argument processing stops at -- and these arguments are appended to the
|
| - command line of the target to run. For example, use:
|
| - isolate.py run --isolated foo.isolated -- --gtest_filter=Foo.Bar
|
| - """
|
| - parser.require_isolated = False
|
| - parser.add_option(
|
| - '--skip-refresh', action='store_true',
|
| - help='Skip reading .isolate file and do not refresh the hash of '
|
| - 'dependencies')
|
| - options, args = parser.parse_args(args)
|
| - if options.outdir and is_url(options.outdir):
|
| - parser.error('Can\'t use url for --outdir with mode run.')
|
| -
|
| - complete_state = load_complete_state(
|
| - options, os.getcwd(), None, options.skip_refresh)
|
| - cmd = complete_state.saved_state.command + args
|
| - if not cmd:
|
| - raise ExecutionError('No command to run.')
|
| -
|
| - cmd = tools.fix_python_path(cmd)
|
| - try:
|
| - root_dir = complete_state.root_dir
|
| - if not options.outdir:
|
| - if not os.path.isabs(root_dir):
|
| - root_dir = os.path.join(os.path.dirname(options.isolated), root_dir)
|
| - options.outdir = run_isolated.make_temp_dir('isolate', root_dir)
|
| - else:
|
| - if not os.path.isdir(options.outdir):
|
| - os.makedirs(options.outdir)
|
| - recreate_tree(
|
| - outdir=options.outdir,
|
| - indir=root_dir,
|
| - infiles=complete_state.saved_state.files,
|
| - action=run_isolated.HARDLINK_WITH_FALLBACK,
|
| - as_hash=False)
|
| - cwd = os.path.normpath(
|
| - os.path.join(options.outdir, complete_state.saved_state.relative_cwd))
|
| - if not os.path.isdir(cwd):
|
| - # It can happen when no files are mapped from the directory containing the
|
| - # .isolate file. But the directory must exist to be the current working
|
| - # directory.
|
| - os.makedirs(cwd)
|
| - if complete_state.saved_state.read_only:
|
| - run_isolated.make_writable(options.outdir, True)
|
| - logging.info('Running %s, cwd=%s' % (cmd, cwd))
|
| - result = subprocess.call(cmd, cwd=cwd)
|
| - finally:
|
| - if options.outdir:
|
| - run_isolated.rmtree(options.outdir)
|
| -
|
| - if complete_state.isolated_filepath:
|
| - complete_state.save_files()
|
| - return result
|
| -
|
| -
|
| -@subcommand.usage('-- [extra arguments]')
|
| -def CMDtrace(parser, args):
|
| - """Traces the target using trace_inputs.py.
|
| -
|
| - It runs the executable without remapping it, and traces all the files it and
|
| - its child processes access. Then the 'merge' command can be used to generate
|
| - an updated .isolate file out of it or the 'read' command to print it out to
|
| - stdout.
|
| -
|
| - Argument processing stops at -- and these arguments are appended to the
|
| - command line of the target to run. For example, use:
|
| - isolate.py trace --isolated foo.isolated -- --gtest_filter=Foo.Bar
|
| - """
|
| - add_trace_option(parser)
|
| - parser.add_option(
|
| - '-m', '--merge', action='store_true',
|
| - help='After tracing, merge the results back in the .isolate file')
|
| - parser.add_option(
|
| - '--skip-refresh', action='store_true',
|
| - help='Skip reading .isolate file and do not refresh the hash of '
|
| - 'dependencies')
|
| - options, args = parser.parse_args(args)
|
| -
|
| - complete_state = load_complete_state(
|
| - options, os.getcwd(), None, options.skip_refresh)
|
| - cmd = complete_state.saved_state.command + args
|
| - if not cmd:
|
| - raise ExecutionError('No command to run.')
|
| - cmd = tools.fix_python_path(cmd)
|
| - cwd = os.path.normpath(os.path.join(
|
| - unicode(complete_state.root_dir),
|
| - complete_state.saved_state.relative_cwd))
|
| - cmd[0] = os.path.normpath(os.path.join(cwd, cmd[0]))
|
| - if not os.path.isfile(cmd[0]):
|
| - raise ExecutionError(
|
| - 'Tracing failed for: %s\nIt doesn\'t exit' % ' '.join(cmd))
|
| - logging.info('Running %s, cwd=%s' % (cmd, cwd))
|
| - api = trace_inputs.get_api()
|
| - logfile = complete_state.isolated_filepath + '.log'
|
| - api.clean_trace(logfile)
|
| - out = None
|
| - try:
|
| - with api.get_tracer(logfile) as tracer:
|
| - result, out = tracer.trace(
|
| - cmd,
|
| - cwd,
|
| - 'default',
|
| - True)
|
| - except trace_inputs.TracingFailure, e:
|
| - raise ExecutionError('Tracing failed for: %s\n%s' % (' '.join(cmd), str(e)))
|
| -
|
| - if result:
|
| - logging.error(
|
| - 'Tracer exited with %d, which means the tests probably failed so the '
|
| - 'trace is probably incomplete.', result)
|
| - logging.info(out)
|
| -
|
| - complete_state.save_files()
|
| -
|
| - if options.merge:
|
| - blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
|
| - merge(complete_state, blacklist)
|
| -
|
| - return result
|
| -
|
| -
|
| -def _process_variable_arg(_option, _opt, _value, parser):
|
| - if not parser.rargs:
|
| - raise optparse.OptionValueError(
|
| - 'Please use --variable FOO=BAR or --variable FOO BAR')
|
| - k = parser.rargs.pop(0)
|
| - if '=' in k:
|
| - parser.values.variables.append(tuple(k.split('=', 1)))
|
| - else:
|
| - if not parser.rargs:
|
| - raise optparse.OptionValueError(
|
| - 'Please use --variable FOO=BAR or --variable FOO BAR')
|
| - v = parser.rargs.pop(0)
|
| - parser.values.variables.append((k, v))
|
| -
|
| -
|
| -def add_variable_option(parser):
|
| - """Adds --isolated and --variable to an OptionParser."""
|
| - parser.add_option(
|
| - '-s', '--isolated',
|
| - metavar='FILE',
|
| - help='.isolated file to generate or read')
|
| - # Keep for compatibility. TODO(maruel): Remove once not used anymore.
|
| - parser.add_option(
|
| - '-r', '--result',
|
| - dest='isolated',
|
| - help=optparse.SUPPRESS_HELP)
|
| - default_variables = [('OS', get_flavor())]
|
| - if sys.platform in ('win32', 'cygwin'):
|
| - default_variables.append(('EXECUTABLE_SUFFIX', '.exe'))
|
| - else:
|
| - default_variables.append(('EXECUTABLE_SUFFIX', ''))
|
| - parser.add_option(
|
| - '-V', '--variable',
|
| - action='callback',
|
| - callback=_process_variable_arg,
|
| - default=default_variables,
|
| - dest='variables',
|
| - metavar='FOO BAR',
|
| - help='Variables to process in the .isolate file, default: %default. '
|
| - 'Variables are persistent accross calls, they are saved inside '
|
| - '<.isolated>.state')
|
| -
|
| -
|
| -def add_trace_option(parser):
|
| - """Adds --trace-blacklist to the parser."""
|
| - parser.add_option(
|
| - '--trace-blacklist',
|
| - action='append', default=list(DEFAULT_BLACKLIST),
|
| - help='List of regexp to use as blacklist filter for files to consider '
|
| - 'important, not to be confused with --blacklist which blacklists '
|
| - 'test case.')
|
| -
|
| -
|
| -def parse_isolated_option(parser, options, cwd, require_isolated):
|
| - """Processes --isolated."""
|
| - if options.isolated:
|
| - options.isolated = os.path.normpath(
|
| - os.path.join(cwd, options.isolated.replace('/', os.path.sep)))
|
| - if require_isolated and not options.isolated:
|
| - parser.error('--isolated is required.')
|
| - if options.isolated and not options.isolated.endswith('.isolated'):
|
| - parser.error('--isolated value must end with \'.isolated\'')
|
| -
|
| -
|
| -def parse_variable_option(options):
|
| - """Processes --variable."""
|
| - # TODO(benrg): Maybe we should use a copy of gyp's NameValueListToDict here,
|
| - # but it wouldn't be backward compatible.
|
| - def try_make_int(s):
|
| - """Converts a value to int if possible, converts to unicode otherwise."""
|
| - try:
|
| - return int(s)
|
| - except ValueError:
|
| - return s.decode('utf-8')
|
| - options.variables = dict((k, try_make_int(v)) for k, v in options.variables)
|
| -
|
| -
|
| -class OptionParserIsolate(tools.OptionParserWithLogging):
|
| - """Adds automatic --isolate, --isolated, --out and --variable handling."""
|
| - # Set it to False if it is not required, e.g. it can be passed on but do not
|
| - # fail if not given.
|
| - require_isolated = True
|
| -
|
| - def __init__(self, **kwargs):
|
| - tools.OptionParserWithLogging.__init__(
|
| - self,
|
| - verbose=int(os.environ.get('ISOLATE_DEBUG', 0)),
|
| - **kwargs)
|
| - group = optparse.OptionGroup(self, "Common options")
|
| - group.add_option(
|
| - '-i', '--isolate',
|
| - metavar='FILE',
|
| - help='.isolate file to load the dependency data from')
|
| - add_variable_option(group)
|
| - group.add_option(
|
| - '-o', '--outdir', metavar='DIR',
|
| - help='Directory used to recreate the tree or store the hash table. '
|
| - 'Defaults: run|remap: a /tmp subdirectory, others: '
|
| - 'defaults to the directory containing --isolated')
|
| - group.add_option(
|
| - '--ignore_broken_items', action='store_true',
|
| - default=bool(os.environ.get('ISOLATE_IGNORE_BROKEN_ITEMS')),
|
| - help='Indicates that invalid entries in the isolated file to be '
|
| - 'only be logged and not stop processing. Defaults to True if '
|
| - 'env var ISOLATE_IGNORE_BROKEN_ITEMS is set')
|
| - self.add_option_group(group)
|
| -
|
| - def parse_args(self, *args, **kwargs):
|
| - """Makes sure the paths make sense.
|
| -
|
| - On Windows, / and \ are often mixed together in a path.
|
| - """
|
| - options, args = tools.OptionParserWithLogging.parse_args(
|
| - self, *args, **kwargs)
|
| - if not self.allow_interspersed_args and args:
|
| - self.error('Unsupported argument: %s' % args)
|
| -
|
| - cwd = file_path.get_native_path_case(unicode(os.getcwd()))
|
| - parse_isolated_option(self, options, cwd, self.require_isolated)
|
| - parse_variable_option(options)
|
| -
|
| - if options.isolate:
|
| - # TODO(maruel): Work with non-ASCII.
|
| - # The path must be in native path case for tracing purposes.
|
| - options.isolate = unicode(options.isolate).replace('/', os.path.sep)
|
| - options.isolate = os.path.normpath(os.path.join(cwd, options.isolate))
|
| - options.isolate = file_path.get_native_path_case(options.isolate)
|
| -
|
| - if options.outdir and not is_url(options.outdir):
|
| - options.outdir = unicode(options.outdir).replace('/', os.path.sep)
|
| - # outdir doesn't need native path case since tracing is never done from
|
| - # there.
|
| - options.outdir = os.path.normpath(os.path.join(cwd, options.outdir))
|
| -
|
| - return options, args
|
| -
|
| -
|
| -def main(argv):
|
| - dispatcher = subcommand.CommandDispatcher(__name__)
|
| - try:
|
| - return dispatcher.execute(OptionParserIsolate(version=__version__), argv)
|
| - except Exception as e:
|
| - tools.report_error(e)
|
| - return 1
|
| -
|
| -
|
| -if __name__ == '__main__':
|
| - fix_encoding.fix_encoding()
|
| - tools.disable_buffering()
|
| - colorama.init()
|
| - sys.exit(main(sys.argv[1:]))
|
|
|
Chromium Code Reviews
