Delete swarm_client.

swarm_client/isolate.py

-#!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Front end tool to operate on .isolate files.
-
-This includes creating, merging or compiling them to generate a .isolated file.
-
-See more information at
-  https://code.google.com/p/swarming/wiki/IsolateDesign
-  https://code.google.com/p/swarming/wiki/IsolateUserGuide
-"""
-# Run ./isolate.py --help for more detailed information.
-
-import ast
-import copy
-import itertools
-import logging
-import optparse
-import os
-import posixpath
-import re
-import stat
-import subprocess
-import sys
-
-import isolateserver
-import run_isolated
-import trace_inputs
-
-# Import here directly so isolate is easier to use as a library.
-from run_isolated import get_flavor
-
-from third_party import colorama
-from third_party.depot_tools import fix_encoding
-from third_party.depot_tools import subcommand
-
-from utils import file_path
-from utils import tools
-from utils import short_expression_finder
-
-
-__version__ = '0.1.1'
-
-
-PATH_VARIABLES = ('DEPTH', 'PRODUCT_DIR')
-
-# Files that should be 0-length when mapped.
-KEY_TOUCHED = 'isolate_dependency_touched'
-# Files that should be tracked by the build tool.
-KEY_TRACKED = 'isolate_dependency_tracked'
-# Files that should not be tracked by the build tool.
-KEY_UNTRACKED = 'isolate_dependency_untracked'
-
-
-class ExecutionError(Exception):
-  """A generic error occurred."""
-  def __str__(self):
-    return self.args[0]
-
-
-### Path handling code.
-
-
-DEFAULT_BLACKLIST = (
-  # Temporary vim or python files.
-  r'^.+\.(?:pyc|swp)$',
-  # .git or .svn directory.
-  r'^(?:.+' + re.escape(os.path.sep) + r'|)\.(?:git|svn)$',
-)
-
-
-# Chromium-specific.
-DEFAULT_BLACKLIST += (
-  r'^.+\.(?:run_test_cases)$',
-  r'^(?:.+' + re.escape(os.path.sep) + r'|)testserver\.log$',
-)
-
-
-def relpath(path, root):
-  """os.path.relpath() that keeps trailing os.path.sep."""
-  out = os.path.relpath(path, root)
-  if path.endswith(os.path.sep):
-    out += os.path.sep
-  return out
-
-
-def safe_relpath(filepath, basepath):
-  """Do not throw on Windows when filepath and basepath are on different drives.
-
-  Different than relpath() above since this one doesn't keep the trailing
-  os.path.sep and it swallows exceptions on Windows and return the original
-  absolute path in the case of different drives.
-  """
-  try:
-    return os.path.relpath(filepath, basepath)
-  except ValueError:
-    assert sys.platform == 'win32'
-    return filepath
-
-
-def normpath(path):
-  """os.path.normpath() that keeps trailing os.path.sep."""
-  out = os.path.normpath(path)
-  if path.endswith(os.path.sep):
-    out += os.path.sep
-  return out
-
-
-def posix_relpath(path, root):
-  """posix.relpath() that keeps trailing slash."""
-  out = posixpath.relpath(path, root)
-  if path.endswith('/'):
-    out += '/'
-  return out
-
-
-def cleanup_path(x):
-  """Cleans up a relative path. Converts any os.path.sep to '/' on Windows."""
-  if x:
-    x = x.rstrip(os.path.sep).replace(os.path.sep, '/')
-  if x == '.':
-    x = ''
-  if x:
-    x += '/'
-  return x
-
-
-def is_url(path):
-  return bool(re.match(r'^https?://.+$', path))
-
-
-def path_starts_with(prefix, path):
-  """Returns true if the components of the path |prefix| are the same as the
-  initial components of |path| (or all of the components of |path|). The paths
-  must be absolute.
-  """
-  assert os.path.isabs(prefix) and os.path.isabs(path)
-  prefix = os.path.normpath(prefix)
-  path = os.path.normpath(path)
-  assert prefix == file_path.get_native_path_case(prefix), prefix
-  assert path == file_path.get_native_path_case(path), path
-  prefix = prefix.rstrip(os.path.sep) + os.path.sep
-  path = path.rstrip(os.path.sep) + os.path.sep
-  return path.startswith(prefix)
-
-
-def fix_native_path_case(root, path):
-  """Ensures that each component of |path| has the proper native case by
-     iterating slowly over the directory elements of |path|."""
-  native_case_path = root
-  for raw_part in path.split(os.sep):
-    if not raw_part or raw_part == '.':
-      break
-
-    part = file_path.find_item_native_case(native_case_path, raw_part)
-    if not part:
-      raise isolateserver.MappingError(
-          'Input file %s doesn\'t exist' %
-          os.path.join(native_case_path, raw_part))
-    native_case_path = os.path.join(native_case_path, part)
-
-  return os.path.normpath(native_case_path)
-
-
-def expand_symlinks(indir, relfile):
-  """Follows symlinks in |relfile|, but treating symlinks that point outside the
-  build tree as if they were ordinary directories/files. Returns the final
-  symlink-free target and a list of paths to symlinks encountered in the
-  process.
-
-  The rule about symlinks outside the build tree is for the benefit of the
-  Chromium OS ebuild, which symlinks the output directory to an unrelated path
-  in the chroot.
-
-  Fails when a directory loop is detected, although in theory we could support
-  that case.
-  """
-  is_directory = relfile.endswith(os.path.sep)
-  done = indir
-  todo = relfile.strip(os.path.sep)
-  symlinks = []
-
-  while todo:
-    pre_symlink, symlink, post_symlink = file_path.split_at_symlink(
-        done, todo)
-    if not symlink:
-      todo = fix_native_path_case(done, todo)
-      done = os.path.join(done, todo)
-      break
-    symlink_path = os.path.join(done, pre_symlink, symlink)
-    post_symlink = post_symlink.lstrip(os.path.sep)
-    # readlink doesn't exist on Windows.
-    # pylint: disable=E1101
-    target = os.path.normpath(os.path.join(done, pre_symlink))
-    symlink_target = os.readlink(symlink_path)
-    if os.path.isabs(symlink_target):
-      # Absolute path are considered a normal directories. The use case is
-      # generally someone who puts the output directory on a separate drive.
-      target = symlink_target
-    else:
-      # The symlink itself could be using the wrong path case.
-      target = fix_native_path_case(target, symlink_target)
-
-    if not os.path.exists(target):
-      raise isolateserver.MappingError(
-          'Symlink target doesn\'t exist: %s -> %s' % (symlink_path, target))
-    target = file_path.get_native_path_case(target)
-    if not path_starts_with(indir, target):
-      done = symlink_path
-      todo = post_symlink
-      continue
-    if path_starts_with(target, symlink_path):
-      raise isolateserver.MappingError(
-          'Can\'t map recursive symlink reference %s -> %s' %
-          (symlink_path, target))
-    logging.info('Found symlink: %s -> %s', symlink_path, target)
-    symlinks.append(os.path.relpath(symlink_path, indir))
-    # Treat the common prefix of the old and new paths as done, and start
-    # scanning again.
-    target = target.split(os.path.sep)
-    symlink_path = symlink_path.split(os.path.sep)
-    prefix_length = 0
-    for target_piece, symlink_path_piece in zip(target, symlink_path):
-      if target_piece == symlink_path_piece:
-        prefix_length += 1
-      else:
-        break
-    done = os.path.sep.join(target[:prefix_length])
-    todo = os.path.join(
-        os.path.sep.join(target[prefix_length:]), post_symlink)
-
-  relfile = os.path.relpath(done, indir)
-  relfile = relfile.rstrip(os.path.sep) + is_directory * os.path.sep
-  return relfile, symlinks
-
-
-def expand_directory_and_symlink(indir, relfile, blacklist, follow_symlinks):
-  """Expands a single input. It can result in multiple outputs.
-
-  This function is recursive when relfile is a directory.
-
-  Note: this code doesn't properly handle recursive symlink like one created
-  with:
-    ln -s .. foo
-  """
-  if os.path.isabs(relfile):
-    raise isolateserver.MappingError(
-        'Can\'t map absolute path %s' % relfile)
-
-  infile = normpath(os.path.join(indir, relfile))
-  if not infile.startswith(indir):
-    raise isolateserver.MappingError(
-        'Can\'t map file %s outside %s' % (infile, indir))
-
-  filepath = os.path.join(indir, relfile)
-  native_filepath = file_path.get_native_path_case(filepath)
-  if filepath != native_filepath:
-    # Special case './'.
-    if filepath != native_filepath + '.' + os.path.sep:
-      # Give up enforcing strict path case on OSX. Really, it's that sad. The
-      # case where it happens is very specific and hard to reproduce:
-      # get_native_path_case(
-      #    u'Foo.framework/Versions/A/Resources/Something.nib') will return
-      # u'Foo.framework/Versions/A/resources/Something.nib', e.g. lowercase 'r'.
-      #
-      # Note that this is really something deep in OSX because running
-      # ls Foo.framework/Versions/A
-      # will print out 'Resources', while file_path.get_native_path_case()
-      # returns a lower case 'r'.
-      #
-      # So *something* is happening under the hood resulting in the command 'ls'
-      # and Carbon.File.FSPathMakeRef('path').FSRefMakePath() to disagree.  We
-      # have no idea why.
-      if sys.platform != 'darwin':
-        raise isolateserver.MappingError(
-            'File path doesn\'t equal native file path\n%s != %s' %
-            (filepath, native_filepath))
-
-  symlinks = []
-  if follow_symlinks:
-    relfile, symlinks = expand_symlinks(indir, relfile)
-
-  if relfile.endswith(os.path.sep):
-    if not os.path.isdir(infile):
-      raise isolateserver.MappingError(
-          '%s is not a directory but ends with "%s"' % (infile, os.path.sep))
-
-    # Special case './'.
-    if relfile.startswith('.' + os.path.sep):
-      relfile = relfile[2:]
-    outfiles = symlinks
-    try:
-      for filename in os.listdir(infile):
-        inner_relfile = os.path.join(relfile, filename)
-        if blacklist(inner_relfile):
-          continue
-        if os.path.isdir(os.path.join(indir, inner_relfile)):
-          inner_relfile += os.path.sep
-        outfiles.extend(
-            expand_directory_and_symlink(indir, inner_relfile, blacklist,
-                                         follow_symlinks))
-      return outfiles
-    except OSError as e:
-      raise isolateserver.MappingError(
-          'Unable to iterate over directory %s.\n%s' % (infile, e))
-  else:
-    # Always add individual files even if they were blacklisted.
-    if os.path.isdir(infile):
-      raise isolateserver.MappingError(
-          'Input directory %s must have a trailing slash' % infile)
-
-    if not os.path.isfile(infile):
-      raise isolateserver.MappingError(
-          'Input file %s doesn\'t exist' % infile)
-
-    return symlinks + [relfile]
-
-
-def expand_directories_and_symlinks(indir, infiles, blacklist,
-                                    follow_symlinks, ignore_broken_items):
-  """Expands the directories and the symlinks, applies the blacklist and
-  verifies files exist.
-
-  Files are specified in os native path separator.
-  """
-  outfiles = []
-  for relfile in infiles:
-    try:
-      outfiles.extend(expand_directory_and_symlink(indir, relfile, blacklist,
-                                                   follow_symlinks))
-    except isolateserver.MappingError as e:
-      if ignore_broken_items:
-        logging.info('warning: %s', e)
-      else:
-        raise
-  return outfiles
-
-
-def recreate_tree(outdir, indir, infiles, action, as_hash):
-  """Creates a new tree with only the input files in it.
-
-  Arguments:
-    outdir:    Output directory to create the files in.
-    indir:     Root directory the infiles are based in.
-    infiles:   dict of files to map from |indir| to |outdir|.
-    action:    One of accepted action of run_isolated.link_file().
-    as_hash:   Output filename is the hash instead of relfile.
-  """
-  logging.info(
-      'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
-      (outdir, indir, len(infiles), action, as_hash))
-
-  assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
-  if not os.path.isdir(outdir):
-    logging.info('Creating %s' % outdir)
-    os.makedirs(outdir)
-
-  for relfile, metadata in infiles.iteritems():
-    infile = os.path.join(indir, relfile)
-    if as_hash:
-      # Do the hashtable specific checks.
-      if 'l' in metadata:
-        # Skip links when storing a hashtable.
-        continue
-      outfile = os.path.join(outdir, metadata['h'])
-      if os.path.isfile(outfile):
-        # Just do a quick check that the file size matches. No need to stat()
-        # again the input file, grab the value from the dict.
-        if not 's' in metadata:
-          raise isolateserver.MappingError(
-              'Misconfigured item %s: %s' % (relfile, metadata))
-        if metadata['s'] == os.stat(outfile).st_size:
-          continue
-        else:
-          logging.warn('Overwritting %s' % metadata['h'])
-          os.remove(outfile)
-    else:
-      outfile = os.path.join(outdir, relfile)
-      outsubdir = os.path.dirname(outfile)
-      if not os.path.isdir(outsubdir):
-        os.makedirs(outsubdir)
-
-    # TODO(csharp): Fix crbug.com/150823 and enable the touched logic again.
-    # if metadata.get('T') == True:
-    #   open(outfile, 'ab').close()
-    if 'l' in metadata:
-      pointed = metadata['l']
-      logging.debug('Symlink: %s -> %s' % (outfile, pointed))
-      # symlink doesn't exist on Windows.
-      os.symlink(pointed, outfile)  # pylint: disable=E1101
-    else:
-      run_isolated.link_file(outfile, infile, action)
-
-
-def process_input(filepath, prevdict, read_only, flavor, algo):
-  """Processes an input file, a dependency, and return meta data about it.
-
-  Arguments:
-  - filepath: File to act on.
-  - prevdict: the previous dictionary. It is used to retrieve the cached sha-1
-              to skip recalculating the hash.
-  - read_only: If True, the file mode is manipulated. In practice, only save
-               one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r). On
-               windows, mode is not set since all files are 'executable' by
-               default.
-  - algo:      Hashing algorithm used.
-
-  Behaviors:
-  - Retrieves the file mode, file size, file timestamp, file link
-    destination if it is a file link and calcultate the SHA-1 of the file's
-    content if the path points to a file and not a symlink.
-  """
-  out = {}
-  # TODO(csharp): Fix crbug.com/150823 and enable the touched logic again.
-  # if prevdict.get('T') == True:
-  #   # The file's content is ignored. Skip the time and hard code mode.
-  #   if get_flavor() != 'win':
-  #     out['m'] = stat.S_IRUSR | stat.S_IRGRP
-  #   out['s'] = 0
-  #   out['h'] = algo().hexdigest()
-  #   out['T'] = True
-  #   return out
-
-  # Always check the file stat and check if it is a link. The timestamp is used
-  # to know if the file's content/symlink destination should be looked into.
-  # E.g. only reuse from prevdict if the timestamp hasn't changed.
-  # There is the risk of the file's timestamp being reset to its last value
-  # manually while its content changed. We don't protect against that use case.
-  try:
-    filestats = os.lstat(filepath)
-  except OSError:
-    # The file is not present.
-    raise isolateserver.MappingError('%s is missing' % filepath)
-  is_link = stat.S_ISLNK(filestats.st_mode)
-
-  if flavor != 'win':
-    # Ignore file mode on Windows since it's not really useful there.
-    filemode = stat.S_IMODE(filestats.st_mode)
-    # Remove write access for group and all access to 'others'.
-    filemode &= ~(stat.S_IWGRP | stat.S_IRWXO)
-    if read_only:
-      filemode &= ~stat.S_IWUSR
-    if filemode & stat.S_IXUSR:
-      filemode |= stat.S_IXGRP
-    else:
-      filemode &= ~stat.S_IXGRP
-    if not is_link:
-      out['m'] = filemode
-
-  # Used to skip recalculating the hash or link destination. Use the most recent
-  # update time.
-  # TODO(maruel): Save it in the .state file instead of .isolated so the
-  # .isolated file is deterministic.
-  out['t'] = int(round(filestats.st_mtime))
-
-  if not is_link:
-    out['s'] = filestats.st_size
-    # If the timestamp wasn't updated and the file size is still the same, carry
-    # on the sha-1.
-    if (prevdict.get('t') == out['t'] and
-        prevdict.get('s') == out['s']):
-      # Reuse the previous hash if available.
-      out['h'] = prevdict.get('h')
-    if not out.get('h'):
-      out['h'] = isolateserver.hash_file(filepath, algo)
-  else:
-    # If the timestamp wasn't updated, carry on the link destination.
-    if prevdict.get('t') == out['t']:
-      # Reuse the previous link destination if available.
-      out['l'] = prevdict.get('l')
-    if out.get('l') is None:
-      # The link could be in an incorrect path case. In practice, this only
-      # happen on OSX on case insensitive HFS.
-      # TODO(maruel): It'd be better if it was only done once, in
-      # expand_directory_and_symlink(), so it would not be necessary to do again
-      # here.
-      symlink_value = os.readlink(filepath)  # pylint: disable=E1101
-      filedir = file_path.get_native_path_case(os.path.dirname(filepath))
-      native_dest = fix_native_path_case(filedir, symlink_value)
-      out['l'] = os.path.relpath(native_dest, filedir)
-  return out
-
-
-### Variable stuff.
-
-
-def isolatedfile_to_state(filename):
-  """Replaces the file's extension."""
-  return filename + '.state'
-
-
-def determine_root_dir(relative_root, infiles):
-  """For a list of infiles, determines the deepest root directory that is
-  referenced indirectly.
-
-  All arguments must be using os.path.sep.
-  """
-  # The trick used to determine the root directory is to look at "how far" back
-  # up it is looking up.
-  deepest_root = relative_root
-  for i in infiles:
-    x = relative_root
-    while i.startswith('..' + os.path.sep):
-      i = i[3:]
-      assert not i.startswith(os.path.sep)
-      x = os.path.dirname(x)
-    if deepest_root.startswith(x):
-      deepest_root = x
-  logging.debug(
-      'determine_root_dir(%s, %d files) -> %s' % (
-          relative_root, len(infiles), deepest_root))
-  return deepest_root
-
-
-def replace_variable(part, variables):
-  m = re.match(r'<\(([A-Z_]+)\)', part)
-  if m:
-    if m.group(1) not in variables:
-      raise ExecutionError(
-        'Variable "%s" was not found in %s.\nDid you forget to specify '
-        '--variable?' % (m.group(1), variables))
-    return variables[m.group(1)]
-  return part
-
-
-def process_variables(cwd, variables, relative_base_dir):
-  """Processes path variables as a special case and returns a copy of the dict.
-
-  For each 'path' variable: first normalizes it based on |cwd|, verifies it
-  exists then sets it as relative to relative_base_dir.
-  """
-  relative_base_dir = file_path.get_native_path_case(relative_base_dir)
-  variables = variables.copy()
-  for i in PATH_VARIABLES:
-    if i not in variables:
-      continue
-    variable = variables[i].strip()
-    # Variables could contain / or \ on windows. Always normalize to
-    # os.path.sep.
-    variable = variable.replace('/', os.path.sep)
-    variable = os.path.join(cwd, variable)
-    variable = os.path.normpath(variable)
-    variable = file_path.get_native_path_case(variable)
-    if not os.path.isdir(variable):
-      raise ExecutionError('%s=%s is not a directory' % (i, variable))
-
-    # All variables are relative to the .isolate file.
-    variable = os.path.relpath(variable, relative_base_dir)
-    logging.debug(
-        'Translated variable %s from %s to %s', i, variables[i], variable)
-    variables[i] = variable
-  return variables
-
-
-def eval_variables(item, variables):
-  """Replaces the .isolate variables in a string item.
-
-  Note that the .isolate format is a subset of the .gyp dialect.
-  """
-  return ''.join(
-      replace_variable(p, variables) for p in re.split(r'(<\([A-Z_]+\))', item))
-
-
-def classify_files(root_dir, tracked, untracked):
-  """Converts the list of files into a .isolate 'variables' dictionary.
-
-  Arguments:
-  - tracked: list of files names to generate a dictionary out of that should
-             probably be tracked.
-  - untracked: list of files names that must not be tracked.
-  """
-  # These directories are not guaranteed to be always present on every builder.
-  OPTIONAL_DIRECTORIES = (
-    'test/data/plugin',
-    'third_party/WebKit/LayoutTests',
-  )
-
-  new_tracked = []
-  new_untracked = list(untracked)
-
-  def should_be_tracked(filepath):
-    """Returns True if it is a file without whitespace in a non-optional
-    directory that has no symlink in its path.
-    """
-    if filepath.endswith('/'):
-      return False
-    if ' ' in filepath:
-      return False
-    if any(i in filepath for i in OPTIONAL_DIRECTORIES):
-      return False
-    # Look if any element in the path is a symlink.
-    split = filepath.split('/')
-    for i in range(len(split)):
-      if os.path.islink(os.path.join(root_dir, '/'.join(split[:i+1]))):
-        return False
-    return True
-
-  for filepath in sorted(tracked):
-    if should_be_tracked(filepath):
-      new_tracked.append(filepath)
-    else:
-      # Anything else.
-      new_untracked.append(filepath)
-
-  variables = {}
-  if new_tracked:
-    variables[KEY_TRACKED] = sorted(new_tracked)
-  if new_untracked:
-    variables[KEY_UNTRACKED] = sorted(new_untracked)
-  return variables
-
-
-def chromium_fix(f, variables):
-  """Fixes an isolate dependnecy with Chromium-specific fixes."""
-  # Skip log in PRODUCT_DIR. Note that these are applied on '/' style path
-  # separator.
-  LOG_FILE = re.compile(r'^\<\(PRODUCT_DIR\)\/[^\/]+\.log$')
-  # Ignored items.
-  IGNORED_ITEMS = (
-      # http://crbug.com/160539, on Windows, it's in chrome/.
-      'Media Cache/',
-      'chrome/Media Cache/',
-      # 'First Run' is not created by the compile, but by the test itself.
-      '<(PRODUCT_DIR)/First Run')
-
-  # Blacklist logs and other unimportant files.
-  if LOG_FILE.match(f) or f in IGNORED_ITEMS:
-    logging.debug('Ignoring %s', f)
-    return None
-
-  EXECUTABLE = re.compile(
-      r'^(\<\(PRODUCT_DIR\)\/[^\/\.]+)' +
-      re.escape(variables.get('EXECUTABLE_SUFFIX', '')) +
-      r'$')
-  match = EXECUTABLE.match(f)
-  if match:
-    return match.group(1) + '<(EXECUTABLE_SUFFIX)'
-
-  if sys.platform == 'darwin':
-    # On OSX, the name of the output is dependent on gyp define, it can be
-    # 'Google Chrome.app' or 'Chromium.app', same for 'XXX
-    # Framework.framework'. Furthermore, they are versioned with a gyp
-    # variable.  To lower the complexity of the .isolate file, remove all the
-    # individual entries that show up under any of the 4 entries and replace
-    # them with the directory itself. Overall, this results in a bit more
-    # files than strictly necessary.
-    OSX_BUNDLES = (
-      '<(PRODUCT_DIR)/Chromium Framework.framework/',
-      '<(PRODUCT_DIR)/Chromium.app/',
-      '<(PRODUCT_DIR)/Google Chrome Framework.framework/',
-      '<(PRODUCT_DIR)/Google Chrome.app/',
-    )
-    for prefix in OSX_BUNDLES:
-      if f.startswith(prefix):
-        # Note this result in duplicate values, so the a set() must be used to
-        # remove duplicates.
-        return prefix
-  return f
-
-
-def generate_simplified(
-    tracked, untracked, touched, root_dir, variables, relative_cwd,
-    trace_blacklist):
-  """Generates a clean and complete .isolate 'variables' dictionary.
-
-  Cleans up and extracts only files from within root_dir then processes
-  variables and relative_cwd.
-  """
-  root_dir = os.path.realpath(root_dir)
-  logging.info(
-      'generate_simplified(%d files, %s, %s, %s)' %
-      (len(tracked) + len(untracked) + len(touched),
-        root_dir, variables, relative_cwd))
-
-  # Preparation work.
-  relative_cwd = cleanup_path(relative_cwd)
-  assert not os.path.isabs(relative_cwd), relative_cwd
-  # Creates the right set of variables here. We only care about PATH_VARIABLES.
-  path_variables = dict(
-      ('<(%s)' % k, variables[k].replace(os.path.sep, '/'))
-      for k in PATH_VARIABLES if k in variables)
-  variables = variables.copy()
-  variables.update(path_variables)
-
-  # Actual work: Process the files.
-  # TODO(maruel): if all the files in a directory are in part tracked and in
-  # part untracked, the directory will not be extracted. Tracked files should be
-  # 'promoted' to be untracked as needed.
-  tracked = trace_inputs.extract_directories(
-      root_dir, tracked, trace_blacklist)
-  untracked = trace_inputs.extract_directories(
-      root_dir, untracked, trace_blacklist)
-  # touched is not compressed, otherwise it would result in files to be archived
-  # that we don't need.
-
-  root_dir_posix = root_dir.replace(os.path.sep, '/')
-  def fix(f):
-    """Bases the file on the most restrictive variable."""
-    # Important, GYP stores the files with / and not \.
-    f = f.replace(os.path.sep, '/')
-    logging.debug('fix(%s)' % f)
-    # If it's not already a variable.
-    if not f.startswith('<'):
-      # relative_cwd is usually the directory containing the gyp file. It may be
-      # empty if the whole directory containing the gyp file is needed.
-      # Use absolute paths in case cwd_dir is outside of root_dir.
-      # Convert the whole thing to / since it's isolate's speak.
-      f = posix_relpath(
-          posixpath.join(root_dir_posix, f),
-          posixpath.join(root_dir_posix, relative_cwd)) or './'
-
-    for variable, root_path in path_variables.iteritems():
-      if f.startswith(root_path):
-        f = variable + f[len(root_path):]
-        logging.debug('Converted to %s' % f)
-        break
-    return f
-
-  def fix_all(items):
-    """Reduces the items to convert variables, removes unneeded items, apply
-    chromium-specific fixes and only return unique items.
-    """
-    variables_converted = (fix(f.path) for f in items)
-    chromium_fixed = (chromium_fix(f, variables) for f in variables_converted)
-    return set(f for f in chromium_fixed if f)
-
-  tracked = fix_all(tracked)
-  untracked = fix_all(untracked)
-  touched = fix_all(touched)
-  out = classify_files(root_dir, tracked, untracked)
-  if touched:
-    out[KEY_TOUCHED] = sorted(touched)
-  return out
-
-
-def chromium_filter_flags(variables):
-  """Filters out build flags used in Chromium that we don't want to treat as
-  configuration variables.
-  """
-  # TODO(benrg): Need a better way to determine this.
-  blacklist = set(PATH_VARIABLES + ('EXECUTABLE_SUFFIX', 'FLAG'))
-  return dict((k, v) for k, v in variables.iteritems() if k not in blacklist)
-
-
-def generate_isolate(
-    tracked, untracked, touched, root_dir, variables, relative_cwd,
-    trace_blacklist):
-  """Generates a clean and complete .isolate file."""
-  dependencies = generate_simplified(
-      tracked, untracked, touched, root_dir, variables, relative_cwd,
-      trace_blacklist)
-  config_variables = chromium_filter_flags(variables)
-  config_variable_names, config_values = zip(
-      *sorted(config_variables.iteritems()))
-  out = Configs(None)
-  # The new dependencies apply to just one configuration, namely config_values.
-  out.merge_dependencies(dependencies, config_variable_names, [config_values])
-  return out.make_isolate_file()
-
-
-def split_touched(files):
-  """Splits files that are touched vs files that are read."""
-  tracked = []
-  touched = []
-  for f in files:
-    if f.size:
-      tracked.append(f)
-    else:
-      touched.append(f)
-  return tracked, touched
-
-
-def pretty_print(variables, stdout):
-  """Outputs a gyp compatible list from the decoded variables.
-
-  Similar to pprint.print() but with NIH syndrome.
-  """
-  # Order the dictionary keys by these keys in priority.
-  ORDER = (
-      'variables', 'condition', 'command', 'relative_cwd', 'read_only',
-      KEY_TRACKED, KEY_UNTRACKED)
-
-  def sorting_key(x):
-    """Gives priority to 'most important' keys before the others."""
-    if x in ORDER:
-      return str(ORDER.index(x))
-    return x
-
-  def loop_list(indent, items):
-    for item in items:
-      if isinstance(item, basestring):
-        stdout.write('%s\'%s\',\n' % (indent, item))
-      elif isinstance(item, dict):
-        stdout.write('%s{\n' % indent)
-        loop_dict(indent + '  ', item)
-        stdout.write('%s},\n' % indent)
-      elif isinstance(item, list):
-        # A list inside a list will write the first item embedded.
-        stdout.write('%s[' % indent)
-        for index, i in enumerate(item):
-          if isinstance(i, basestring):
-            stdout.write(
-                '\'%s\', ' % i.replace('\\', '\\\\').replace('\'', '\\\''))
-          elif isinstance(i, dict):
-            stdout.write('{\n')
-            loop_dict(indent + '  ', i)
-            if index != len(item) - 1:
-              x = ', '
-            else:
-              x = ''
-            stdout.write('%s}%s' % (indent, x))
-          else:
-            assert False
-        stdout.write('],\n')
-      else:
-        assert False
-
-  def loop_dict(indent, items):
-    for key in sorted(items, key=sorting_key):
-      item = items[key]
-      stdout.write("%s'%s': " % (indent, key))
-      if isinstance(item, dict):
-        stdout.write('{\n')
-        loop_dict(indent + '  ', item)
-        stdout.write(indent + '},\n')
-      elif isinstance(item, list):
-        stdout.write('[\n')
-        loop_list(indent + '  ', item)
-        stdout.write(indent + '],\n')
-      elif isinstance(item, basestring):
-        stdout.write(
-            '\'%s\',\n' % item.replace('\\', '\\\\').replace('\'', '\\\''))
-      elif item in (True, False, None):
-        stdout.write('%s\n' % item)
-      else:
-        assert False, item
-
-  stdout.write('{\n')
-  loop_dict('  ', variables)
-  stdout.write('}\n')
-
-
-def union(lhs, rhs):
-  """Merges two compatible datastructures composed of dict/list/set."""
-  assert lhs is not None or rhs is not None
-  if lhs is None:
-    return copy.deepcopy(rhs)
-  if rhs is None:
-    return copy.deepcopy(lhs)
-  assert type(lhs) == type(rhs), (lhs, rhs)
-  if hasattr(lhs, 'union'):
-    # Includes set, ConfigSettings and Configs.
-    return lhs.union(rhs)
-  if isinstance(lhs, dict):
-    return dict((k, union(lhs.get(k), rhs.get(k))) for k in set(lhs).union(rhs))
-  elif isinstance(lhs, list):
-    # Do not go inside the list.
-    return lhs + rhs
-  assert False, type(lhs)
-
-
-def extract_comment(content):
-  """Extracts file level comment."""
-  out = []
-  for line in content.splitlines(True):
-    if line.startswith('#'):
-      out.append(line)
-    else:
-      break
-  return ''.join(out)
-
-
-def eval_content(content):
-  """Evaluates a python file and return the value defined in it.
-
-  Used in practice for .isolate files.
-  """
-  globs = {'__builtins__': None}
-  locs = {}
-  try:
-    value = eval(content, globs, locs)
-  except TypeError as e:
-    e.args = list(e.args) + [content]
-    raise
-  assert locs == {}, locs
-  assert globs == {'__builtins__': None}, globs
-  return value
-
-
-def match_configs(expr, config_variables, all_configs):
-  """Returns the configs from |all_configs| that match the |expr|, where
-  the elements of |all_configs| are tuples of values for the |config_variables|.
-  Example:
-  >>> match_configs(expr = "(foo==1 or foo==2) and bar=='b'",
-                    config_variables = ["foo", "bar"],
-                    all_configs = [(1, 'a'), (1, 'b'), (2, 'a'), (2, 'b')])
-  [(1, 'b'), (2, 'b')]
-  """
-  return [
-    config for config in all_configs
-    if eval(expr, dict(zip(config_variables, config)))
-  ]
-
-
-def verify_variables(variables):
-  """Verifies the |variables| dictionary is in the expected format."""
-  VALID_VARIABLES = [
-    KEY_TOUCHED,
-    KEY_TRACKED,
-    KEY_UNTRACKED,
-    'command',
-    'read_only',
-  ]
-  assert isinstance(variables, dict), variables
-  assert set(VALID_VARIABLES).issuperset(set(variables)), variables.keys()
-  for name, value in variables.iteritems():
-    if name == 'read_only':
-      assert value in (True, False, None), value
-    else:
-      assert isinstance(value, list), value
-      assert all(isinstance(i, basestring) for i in value), value
-
-
-def verify_ast(expr, variables_and_values):
-  """Verifies that |expr| is of the form
-  expr ::= expr ( "or" | "and" ) expr
-         | identifier "==" ( string | int )
-  Also collects the variable identifiers and string/int values in the dict
-  |variables_and_values|, in the form {'var': set([val1, val2, ...]), ...}.
-  """
-  assert isinstance(expr, (ast.BoolOp, ast.Compare))
-  if isinstance(expr, ast.BoolOp):
-    assert isinstance(expr.op, (ast.And, ast.Or))
-    for subexpr in expr.values:
-      verify_ast(subexpr, variables_and_values)
-  else:
-    assert isinstance(expr.left.ctx, ast.Load)
-    assert len(expr.ops) == 1
-    assert isinstance(expr.ops[0], ast.Eq)
-    var_values = variables_and_values.setdefault(expr.left.id, set())
-    rhs = expr.comparators[0]
-    assert isinstance(rhs, (ast.Str, ast.Num))
-    var_values.add(rhs.n if isinstance(rhs, ast.Num) else rhs.s)
-
-
-def verify_condition(condition, variables_and_values):
-  """Verifies the |condition| dictionary is in the expected format.
-  See verify_ast() for the meaning of |variables_and_values|.
-  """
-  VALID_INSIDE_CONDITION = ['variables']
-  assert isinstance(condition, list), condition
-  assert len(condition) == 2, condition
-  expr, then = condition
-
-  test_ast = compile(expr, '<condition>', 'eval', ast.PyCF_ONLY_AST)
-  verify_ast(test_ast.body, variables_and_values)
-
-  assert isinstance(then, dict), then
-  assert set(VALID_INSIDE_CONDITION).issuperset(set(then)), then.keys()
-  verify_variables(then['variables'])
-
-
-def verify_root(value, variables_and_values):
-  """Verifies that |value| is the parsed form of a valid .isolate file.
-  See verify_ast() for the meaning of |variables_and_values|.
-  """
-  VALID_ROOTS = ['includes', 'conditions']
-  assert isinstance(value, dict), value
-  assert set(VALID_ROOTS).issuperset(set(value)), value.keys()
-
-  includes = value.get('includes', [])
-  assert isinstance(includes, list), includes
-  for include in includes:
-    assert isinstance(include, basestring), include
-
-  conditions = value.get('conditions', [])
-  assert isinstance(conditions, list), conditions
-  for condition in conditions:
-    verify_condition(condition, variables_and_values)
-
-
-def remove_weak_dependencies(values, key, item, item_configs):
-  """Removes any configs from this key if the item is already under a
-  strong key.
-  """
-  if key == KEY_TOUCHED:
-    item_configs = set(item_configs)
-    for stronger_key in (KEY_TRACKED, KEY_UNTRACKED):
-      try:
-        item_configs -= values[stronger_key][item]
-      except KeyError:
-        pass
-
-  return item_configs
-
-
-def remove_repeated_dependencies(folders, key, item, item_configs):
-  """Removes any configs from this key if the item is in a folder that is
-  already included."""
-
-  if key in (KEY_UNTRACKED, KEY_TRACKED, KEY_TOUCHED):
-    item_configs = set(item_configs)
-    for (folder, configs) in folders.iteritems():
-      if folder != item and item.startswith(folder):
-        item_configs -= configs
-
-  return item_configs
-
-
-def get_folders(values_dict):
-  """Returns a dict of all the folders in the given value_dict."""
-  return dict(
-    (item, configs) for (item, configs) in values_dict.iteritems()
-    if item.endswith('/')
-  )
-
-
-def invert_map(variables):
-  """Converts {config: {deptype: list(depvals)}} to
-  {deptype: {depval: set(configs)}}.
-  """
-  KEYS = (
-    KEY_TOUCHED,
-    KEY_TRACKED,
-    KEY_UNTRACKED,
-    'command',
-    'read_only',
-  )
-  out = dict((key, {}) for key in KEYS)
-  for config, values in variables.iteritems():
-    for key in KEYS:
-      if key == 'command':
-        items = [tuple(values[key])] if key in values else []
-      elif key == 'read_only':
-        items = [values[key]] if key in values else []
-      else:
-        assert key in (KEY_TOUCHED, KEY_TRACKED, KEY_UNTRACKED)
-        items = values.get(key, [])
-      for item in items:
-        out[key].setdefault(item, set()).add(config)
-  return out
-
-
-def reduce_inputs(values):
-  """Reduces the output of invert_map() to the strictest minimum list.
-
-  Looks at each individual file and directory, maps where they are used and
-  reconstructs the inverse dictionary.
-
-  Returns the minimized dictionary.
-  """
-  KEYS = (
-    KEY_TOUCHED,
-    KEY_TRACKED,
-    KEY_UNTRACKED,
-    'command',
-    'read_only',
-  )
-
-  # Folders can only live in KEY_UNTRACKED.
-  folders = get_folders(values.get(KEY_UNTRACKED, {}))
-
-  out = dict((key, {}) for key in KEYS)
-  for key in KEYS:
-    for item, item_configs in values.get(key, {}).iteritems():
-      item_configs = remove_weak_dependencies(values, key, item, item_configs)
-      item_configs = remove_repeated_dependencies(
-          folders, key, item, item_configs)
-      if item_configs:
-        out[key][item] = item_configs
-  return out
-
-
-def convert_map_to_isolate_dict(values, config_variables):
-  """Regenerates back a .isolate configuration dict from files and dirs
-  mappings generated from reduce_inputs().
-  """
-  # Gather a list of configurations for set inversion later.
-  all_mentioned_configs = set()
-  for configs_by_item in values.itervalues():
-    for configs in configs_by_item.itervalues():
-      all_mentioned_configs.update(configs)
-
-  # Invert the mapping to make it dict first.
-  conditions = {}
-  for key in values:
-    for item, configs in values[key].iteritems():
-      then = conditions.setdefault(frozenset(configs), {})
-      variables = then.setdefault('variables', {})
-
-      if item in (True, False):
-        # One-off for read_only.
-        variables[key] = item
-      else:
-        assert item
-        if isinstance(item, tuple):
-          # One-off for command.
-          # Do not merge lists and do not sort!
-          # Note that item is a tuple.
-          assert key not in variables
-          variables[key] = list(item)
-        else:
-          # The list of items (files or dirs). Append the new item and keep
-          # the list sorted.
-          l = variables.setdefault(key, [])
-          l.append(item)
-          l.sort()
-
-  if all_mentioned_configs:
-    config_values = map(set, zip(*all_mentioned_configs))
-    sef = short_expression_finder.ShortExpressionFinder(
-        zip(config_variables, config_values))
-
-  conditions = sorted(
-      [sef.get_expr(configs), then] for configs, then in conditions.iteritems())
-  return {'conditions': conditions}
-
-
-### Internal state files.
-
-
-class ConfigSettings(object):
-  """Represents the dependency variables for a single build configuration.
-  The structure is immutable.
-  """
-  def __init__(self, config, values):
-    self.config = config
-    verify_variables(values)
-    self.touched = sorted(values.get(KEY_TOUCHED, []))
-    self.tracked = sorted(values.get(KEY_TRACKED, []))
-    self.untracked = sorted(values.get(KEY_UNTRACKED, []))
-    self.command = values.get('command', [])[:]
-    self.read_only = values.get('read_only')
-
-  def union(self, rhs):
-    assert not (self.config and rhs.config) or (self.config == rhs.config)
-    assert not (self.command and rhs.command) or (self.command == rhs.command)
-    var = {
-      KEY_TOUCHED: sorted(self.touched + rhs.touched),
-      KEY_TRACKED: sorted(self.tracked + rhs.tracked),
-      KEY_UNTRACKED: sorted(self.untracked + rhs.untracked),
-      'command': self.command or rhs.command,
-      'read_only': rhs.read_only if self.read_only is None else self.read_only,
-    }
-    return ConfigSettings(self.config or rhs.config, var)
-
-  def flatten(self):
-    out = {}
-    if self.command:
-      out['command'] = self.command
-    if self.touched:
-      out[KEY_TOUCHED] = self.touched
-    if self.tracked:
-      out[KEY_TRACKED] = self.tracked
-    if self.untracked:
-      out[KEY_UNTRACKED] = self.untracked
-    if self.read_only is not None:
-      out['read_only'] = self.read_only
-    return out
-
-
-class Configs(object):
-  """Represents a processed .isolate file.
-
-  Stores the file in a processed way, split by configuration.
-  """
-  def __init__(self, file_comment):
-    self.file_comment = file_comment
-    # The keys of by_config are tuples of values for the configuration
-    # variables. The names of the variables (which must be the same for
-    # every by_config key) are kept in config_variables. Initially by_config
-    # is empty and we don't know what configuration variables will be used,
-    # so config_variables also starts out empty. It will be set by the first
-    # call to union() or merge_dependencies().
-    self.by_config = {}
-    self.config_variables = ()
-
-  def union(self, rhs):
-    """Adds variables from rhs (a Configs) to the existing variables.
-    """
-    config_variables = self.config_variables
-    if not config_variables:
-      config_variables = rhs.config_variables
-    else:
-      # We can't proceed if this isn't true since we don't know the correct
-      # default values for extra variables. The variables are sorted so we
-      # don't need to worry about permutations.
-      if rhs.config_variables and rhs.config_variables != config_variables:
-        raise ExecutionError(
-            'Variables in merged .isolate files do not match: %r and %r' % (
-                config_variables, rhs.config_variables))
-
-    # Takes the first file comment, prefering lhs.
-    out = Configs(self.file_comment or rhs.file_comment)
-    out.config_variables = config_variables
-    for config in set(self.by_config) | set(rhs.by_config):
-      out.by_config[config] = union(
-          self.by_config.get(config), rhs.by_config.get(config))
-    return out
-
-  def merge_dependencies(self, values, config_variables, configs):
-    """Adds new dependencies to this object for the given configurations.
-    Arguments:
-      values: A variables dict as found in a .isolate file, e.g.,
-          {KEY_TOUCHED: [...], 'command': ...}.
-      config_variables: An ordered list of configuration variables, e.g.,
-          ["OS", "chromeos"]. If this object already contains any dependencies,
-          the configuration variables must match.
-      configs: a list of tuples of values of the configuration variables,
-          e.g., [("mac", 0), ("linux", 1)]. The dependencies in |values|
-          are added to all of these configurations, and other configurations
-          are unchanged.
-    """
-    if not values:
-      return
-
-    if not self.config_variables:
-      self.config_variables = config_variables
-    else:
-      # See comment in Configs.union().
-      assert self.config_variables == config_variables
-
-    for config in configs:
-      self.by_config[config] = union(
-          self.by_config.get(config), ConfigSettings(config, values))
-
-  def flatten(self):
-    """Returns a flat dictionary representation of the configuration.
-    """
-    return dict((k, v.flatten()) for k, v in self.by_config.iteritems())
-
-  def make_isolate_file(self):
-    """Returns a dictionary suitable for writing to a .isolate file.
-    """
-    dependencies_by_config = self.flatten()
-    configs_by_dependency = reduce_inputs(invert_map(dependencies_by_config))
-    return convert_map_to_isolate_dict(configs_by_dependency,
-                                       self.config_variables)
-
-
-# TODO(benrg): Remove this function when no old-format files are left.
-def convert_old_to_new_format(value):
-  """Converts from the old .isolate format, which only has one variable (OS),
-  always includes 'linux', 'mac' and 'win' in the set of valid values for OS,
-  and allows conditions that depend on the set of all OSes, to the new format,
-  which allows any set of variables, has no hardcoded values, and only allows
-  explicit positive tests of variable values.
-  """
-  conditions = value.get('conditions', [])
-  if 'variables' not in value and all(len(cond) == 2 for cond in conditions):
-    return value  # Nothing to change
-
-  def parse_condition(cond):
-    return re.match(r'OS=="(\w+)"\Z', cond[0]).group(1)
-
-  oses = set(map(parse_condition, conditions))
-  default_oses = set(['linux', 'mac', 'win'])
-  oses = sorted(oses | default_oses)
-
-  def if_not_os(not_os, then):
-    expr = ' or '.join('OS=="%s"' % os for os in oses if os != not_os)
-    return [expr, then]
-
-  conditions = [
-    cond[:2] for cond in conditions if cond[1]
-  ] + [
-    if_not_os(parse_condition(cond), cond[2])
-    for cond in conditions if len(cond) == 3
-  ]
-
-  if 'variables' in value:
-    conditions.append(if_not_os(None, {'variables': value.pop('variables')}))
-  conditions.sort()
-
-  value = value.copy()
-  value['conditions'] = conditions
-  return value
-
-
-def load_isolate_as_config(isolate_dir, value, file_comment):
-  """Parses one .isolate file and returns a Configs() instance.
-
-  |value| is the loaded dictionary that was defined in the gyp file.
-
-  The expected format is strict, anything diverting from the format below will
-  throw an assert:
-  {
-    'includes': [
-      'foo.isolate',
-    ],
-    'conditions': [
-      ['OS=="vms" and foo=42', {
-        'variables': {
-          'command': [
-            ...
-          ],
-          'isolate_dependency_tracked': [
-            ...
-          ],
-          'isolate_dependency_untracked': [
-            ...
-          ],
-          'read_only': False,
-        },
-      }],
-      ...
-    ],
-  }
-  """
-  value = convert_old_to_new_format(value)
-
-  variables_and_values = {}
-  verify_root(value, variables_and_values)
-  if variables_and_values:
-    config_variables, config_values = zip(
-        *sorted(variables_and_values.iteritems()))
-    all_configs = list(itertools.product(*config_values))
-  else:
-    config_variables = None
-    all_configs = []
-
-  isolate = Configs(file_comment)
-
-  # Add configuration-specific variables.
-  for expr, then in value.get('conditions', []):
-    configs = match_configs(expr, config_variables, all_configs)
-    isolate.merge_dependencies(then['variables'], config_variables, configs)
-
-  # Load the includes.
-  for include in value.get('includes', []):
-    if os.path.isabs(include):
-      raise ExecutionError(
-          'Failed to load configuration; absolute include path \'%s\'' %
-          include)
-    included_isolate = os.path.normpath(os.path.join(isolate_dir, include))
-    with open(included_isolate, 'r') as f:
-      included_isolate = load_isolate_as_config(
-          os.path.dirname(included_isolate),
-          eval_content(f.read()),
-          None)
-    isolate = union(isolate, included_isolate)
-
-  return isolate
-
-
-def load_isolate_for_config(isolate_dir, content, variables):
-  """Loads the .isolate file and returns the information unprocessed but
-  filtered for the specific OS.
-
-  Returns the command, dependencies and read_only flag. The dependencies are
-  fixed to use os.path.sep.
-  """
-  # Load the .isolate file, process its conditions, retrieve the command and
-  # dependencies.
-  isolate = load_isolate_as_config(isolate_dir, eval_content(content), None)
-  try:
-    config_name = tuple(variables[var] for var in isolate.config_variables)
-  except KeyError:
-    raise ExecutionError(
-        'These configuration variables were missing from the command line: %s' %
-        ', '.join(sorted(set(isolate.config_variables) - set(variables))))
-  config = isolate.by_config.get(config_name)
-  if not config:
-    raise ExecutionError(
-        'Failed to load configuration for variable \'%s\' for config(s) \'%s\''
-        '\nAvailable configs: %s' %
-        (', '.join(isolate.config_variables),
-        ', '.join(config_name),
-        ', '.join(str(s) for s in isolate.by_config)))
-  # Merge tracked and untracked variables, isolate.py doesn't care about the
-  # trackability of the variables, only the build tool does.
-  dependencies = [
-    f.replace('/', os.path.sep) for f in config.tracked + config.untracked
-  ]
-  touched = [f.replace('/', os.path.sep) for f in config.touched]
-  return config.command, dependencies, touched, config.read_only
-
-
-def save_isolated(isolated, data):
-  """Writes one or multiple .isolated files.
-
-  Note: this reference implementation does not create child .isolated file so it
-  always returns an empty list.
-
-  Returns the list of child isolated files that are included by |isolated|.
-  """
-  trace_inputs.write_json(isolated, data, True)
-  return []
-
-
-def chromium_save_isolated(isolated, data, variables, algo):
-  """Writes one or many .isolated files.
-
-  This slightly increases the cold cache cost but greatly reduce the warm cache
-  cost by splitting low-churn files off the master .isolated file. It also
-  reduces overall isolateserver memcache consumption.
-  """
-  slaves = []
-
-  def extract_into_included_isolated(prefix):
-    new_slave = {
-      'algo': data['algo'],
-      'files': {},
-      'os': data['os'],
-      'version': data['version'],
-    }
-    for f in data['files'].keys():
-      if f.startswith(prefix):
-        new_slave['files'][f] = data['files'].pop(f)
-    if new_slave['files']:
-      slaves.append(new_slave)
-
-  # Split test/data/ in its own .isolated file.
-  extract_into_included_isolated(os.path.join('test', 'data', ''))
-
-  # Split everything out of PRODUCT_DIR in its own .isolated file.
-  if variables.get('PRODUCT_DIR'):
-    extract_into_included_isolated(variables['PRODUCT_DIR'])
-
-  files = []
-  for index, f in enumerate(slaves):
-    slavepath = isolated[:-len('.isolated')] + '.%d.isolated' % index
-    trace_inputs.write_json(slavepath, f, True)
-    data.setdefault('includes', []).append(
-        isolateserver.hash_file(slavepath, algo))
-    files.append(os.path.basename(slavepath))
-
-  files.extend(save_isolated(isolated, data))
-  return files
-
-
-class Flattenable(object):
-  """Represents data that can be represented as a json file."""
-  MEMBERS = ()
-
-  def flatten(self):
-    """Returns a json-serializable version of itself.
-
-    Skips None entries.
-    """
-    items = ((member, getattr(self, member)) for member in self.MEMBERS)
-    return dict((member, value) for member, value in items if value is not None)
-
-  @classmethod
-  def load(cls, data, *args, **kwargs):
-    """Loads a flattened version."""
-    data = data.copy()
-    out = cls(*args, **kwargs)
-    for member in out.MEMBERS:
-      if member in data:
-        # Access to a protected member XXX of a client class
-        # pylint: disable=W0212
-        out._load_member(member, data.pop(member))
-    if data:
-      raise ValueError(
-          'Found unexpected entry %s while constructing an object %s' %
-            (data, cls.__name__), data, cls.__name__)
-    return out
-
-  def _load_member(self, member, value):
-    """Loads a member into self."""
-    setattr(self, member, value)
-
-  @classmethod
-  def load_file(cls, filename, *args, **kwargs):
-    """Loads the data from a file or return an empty instance."""
-    try:
-      out = cls.load(trace_inputs.read_json(filename), *args, **kwargs)
-      logging.debug('Loaded %s(%s)', cls.__name__, filename)
-    except (IOError, ValueError) as e:
-      # On failure, loads the default instance.
-      out = cls(*args, **kwargs)
-      logging.warn('Failed to load %s: %s', filename, e)
-    return out
-
-
-class SavedState(Flattenable):
-  """Describes the content of a .state file.
-
-  This file caches the items calculated by this script and is used to increase
-  the performance of the script. This file is not loaded by run_isolated.py.
-  This file can always be safely removed.
-
-  It is important to note that the 'files' dict keys are using native OS path
-  separator instead of '/' used in .isolate file.
-  """
-  MEMBERS = (
-    # Algorithm used to generate the hash. The only supported value is at the
-    # time of writting 'sha-1'.
-    'algo',
-    # Cache of the processed command. This value is saved because .isolated
-    # files are never loaded by isolate.py so it's the only way to load the
-    # command safely.
-    'command',
-    # Cache of the files found so the next run can skip hash calculation.
-    'files',
-    # Path of the original .isolate file. Relative path to isolated_basedir.
-    'isolate_file',
-    # List of included .isolated files. Used to support/remember 'slave'
-    # .isolated files. Relative path to isolated_basedir.
-    'child_isolated_files',
-    # If the generated directory tree should be read-only.
-    'read_only',
-    # Relative cwd to use to start the command.
-    'relative_cwd',
-    # GYP variables used to generate the .isolated file. Variables are saved so
-    # a user can use isolate.py after building and the GYP variables are still
-    # defined.
-    'variables',
-    # Version of the file format in format 'major.minor'. Any non-breaking
-    # change must update minor. Any breaking change must update major.
-    'version',
-  )
-
-  def __init__(self, isolated_basedir):
-    """Creates an empty SavedState.
-
-    |isolated_basedir| is the directory where the .isolated and .isolated.state
-    files are saved.
-    """
-    super(SavedState, self).__init__()
-    assert os.path.isabs(isolated_basedir), isolated_basedir
-    assert os.path.isdir(isolated_basedir), isolated_basedir
-    self.isolated_basedir = isolated_basedir
-
-    # The default algorithm used.
-    self.algo = isolateserver.SUPPORTED_ALGOS['sha-1']
-    self.command = []
-    self.files = {}
-    self.isolate_file = None
-    self.child_isolated_files = []
-    self.read_only = None
-    self.relative_cwd = None
-    self.variables = {'OS': get_flavor()}
-    # The current version.
-    self.version = '1.0'
-
-  def update(self, isolate_file, variables):
-    """Updates the saved state with new data to keep GYP variables and internal
-    reference to the original .isolate file.
-    """
-    assert os.path.isabs(isolate_file)
-    # Convert back to a relative path. On Windows, if the isolate and
-    # isolated files are on different drives, isolate_file will stay an absolute
-    # path.
-    isolate_file = safe_relpath(isolate_file, self.isolated_basedir)
-
-    # The same .isolate file should always be used to generate the .isolated and
-    # .isolated.state.
-    assert isolate_file == self.isolate_file or not self.isolate_file, (
-        isolate_file, self.isolate_file)
-    self.isolate_file = isolate_file
-    self.variables.update(variables)
-
-  def update_isolated(self, command, infiles, touched, read_only, relative_cwd):
-    """Updates the saved state with data necessary to generate a .isolated file.
-
-    The new files in |infiles| are added to self.files dict but their hash is
-    not calculated here.
-    """
-    self.command = command
-    # Add new files.
-    for f in infiles:
-      self.files.setdefault(f, {})
-    for f in touched:
-      self.files.setdefault(f, {})['T'] = True
-    # Prune extraneous files that are not a dependency anymore.
-    for f in set(self.files).difference(set(infiles).union(touched)):
-      del self.files[f]
-    if read_only is not None:
-      self.read_only = read_only
-    self.relative_cwd = relative_cwd
-
-  def to_isolated(self):
-    """Creates a .isolated dictionary out of the saved state.
-
-    https://code.google.com/p/swarming/wiki/IsolatedDesign
-    """
-    def strip(data):
-      """Returns a 'files' entry with only the whitelisted keys."""
-      return dict((k, data[k]) for k in ('h', 'l', 'm', 's') if k in data)
-
-    out = {
-      'algo': isolateserver.SUPPORTED_ALGOS_REVERSE[self.algo],
-      'files': dict(
-          (filepath, strip(data)) for filepath, data in self.files.iteritems()),
-      'os': self.variables['OS'],
-      'version': self.version,
-    }
-    if self.command:
-      out['command'] = self.command
-    if self.read_only is not None:
-      out['read_only'] = self.read_only
-    if self.relative_cwd:
-      out['relative_cwd'] = self.relative_cwd
-    return out
-
-  @property
-  def isolate_filepath(self):
-    """Returns the absolute path of self.isolate_file."""
-    return os.path.normpath(
-        os.path.join(self.isolated_basedir, self.isolate_file))
-
-  # Arguments number differs from overridden method
-  @classmethod
-  def load(cls, data, isolated_basedir):  # pylint: disable=W0221
-    """Special case loading to disallow different OS.
-
-    It is not possible to load a .isolated.state files from a different OS, this
-    file is saved in OS-specific format.
-    """
-    out = super(SavedState, cls).load(data, isolated_basedir)
-    if 'os' in data:
-      out.variables['OS'] = data['os']
-
-    # Converts human readable form back into the proper class type.
-    algo = data.get('algo', 'sha-1')
-    if not algo in isolateserver.SUPPORTED_ALGOS:
-      raise isolateserver.ConfigError('Unknown algo \'%s\'' % out.algo)
-    out.algo = isolateserver.SUPPORTED_ALGOS[algo]
-
-    # For example, 1.1 is guaranteed to be backward compatible with 1.0 code.
-    if not re.match(r'^(\d+)\.(\d+)$', out.version):
-      raise isolateserver.ConfigError('Unknown version \'%s\'' % out.version)
-    if out.version.split('.', 1)[0] != '1':
-      raise isolateserver.ConfigError(
-          'Unsupported version \'%s\'' % out.version)
-
-    # The .isolate file must be valid. It could be absolute on Windows if the
-    # drive containing the .isolate and the drive containing the .isolated files
-    # differ.
-    assert not os.path.isabs(out.isolate_file) or sys.platform == 'win32'
-    assert os.path.isfile(out.isolate_filepath), out.isolate_filepath
-    return out
-
-  def flatten(self):
-    """Makes sure 'algo' is in human readable form."""
-    out = super(SavedState, self).flatten()
-    out['algo'] = isolateserver.SUPPORTED_ALGOS_REVERSE[out['algo']]
-    return out
-
-  def __str__(self):
-    out = '%s(\n' % self.__class__.__name__
-    out += '  command: %s\n' % self.command
-    out += '  files: %d\n' % len(self.files)
-    out += '  isolate_file: %s\n' % self.isolate_file
-    out += '  read_only: %s\n' % self.read_only
-    out += '  relative_cwd: %s\n' % self.relative_cwd
-    out += '  child_isolated_files: %s\n' % self.child_isolated_files
-    out += '  variables: %s' % ''.join(
-        '\n    %s=%s' % (k, self.variables[k]) for k in sorted(self.variables))
-    out += ')'
-    return out
-
-
-class CompleteState(object):
-  """Contains all the state to run the task at hand."""
-  def __init__(self, isolated_filepath, saved_state):
-    super(CompleteState, self).__init__()
-    assert isolated_filepath is None or os.path.isabs(isolated_filepath)
-    self.isolated_filepath = isolated_filepath
-    # Contains the data to ease developer's use-case but that is not strictly
-    # necessary.
-    self.saved_state = saved_state
-
-  @classmethod
-  def load_files(cls, isolated_filepath):
-    """Loads state from disk."""
-    assert os.path.isabs(isolated_filepath), isolated_filepath
-    isolated_basedir = os.path.dirname(isolated_filepath)
-    return cls(
-        isolated_filepath,
-        SavedState.load_file(
-            isolatedfile_to_state(isolated_filepath), isolated_basedir))
-
-  def load_isolate(self, cwd, isolate_file, variables, ignore_broken_items):
-    """Updates self.isolated and self.saved_state with information loaded from a
-    .isolate file.
-
-    Processes the loaded data, deduce root_dir, relative_cwd.
-    """
-    # Make sure to not depend on os.getcwd().
-    assert os.path.isabs(isolate_file), isolate_file
-    isolate_file = file_path.get_native_path_case(isolate_file)
-    logging.info(
-        'CompleteState.load_isolate(%s, %s, %s, %s)',
-        cwd, isolate_file, variables, ignore_broken_items)
-    relative_base_dir = os.path.dirname(isolate_file)
-
-    # Processes the variables and update the saved state.
-    variables = process_variables(cwd, variables, relative_base_dir)
-    self.saved_state.update(isolate_file, variables)
-    variables = self.saved_state.variables
-
-    with open(isolate_file, 'r') as f:
-      # At that point, variables are not replaced yet in command and infiles.
-      # infiles may contain directory entries and is in posix style.
-      command, infiles, touched, read_only = load_isolate_for_config(
-          os.path.dirname(isolate_file), f.read(), variables)
-    command = [eval_variables(i, variables) for i in command]
-    infiles = [eval_variables(f, variables) for f in infiles]
-    touched = [eval_variables(f, variables) for f in touched]
-    # root_dir is automatically determined by the deepest root accessed with the
-    # form '../../foo/bar'. Note that path variables must be taken in account
-    # too, add them as if they were input files.
-    path_variables = [variables[v] for v in PATH_VARIABLES if v in variables]
-    root_dir = determine_root_dir(
-        relative_base_dir, infiles + touched + path_variables)
-    # The relative directory is automatically determined by the relative path
-    # between root_dir and the directory containing the .isolate file,
-    # isolate_base_dir.
-    relative_cwd = os.path.relpath(relative_base_dir, root_dir)
-    # Now that we know where the root is, check that the PATH_VARIABLES point
-    # inside it.
-    for i in PATH_VARIABLES:
-      if i in variables:
-        if not path_starts_with(
-            root_dir, os.path.join(relative_base_dir, variables[i])):
-          raise isolateserver.MappingError(
-              'Path variable %s=%r points outside the inferred root directory'
-              ' %s' % (i, variables[i], root_dir))
-    # Normalize the files based to root_dir. It is important to keep the
-    # trailing os.path.sep at that step.
-    infiles = [
-      relpath(normpath(os.path.join(relative_base_dir, f)), root_dir)
-      for f in infiles
-    ]
-    touched = [
-      relpath(normpath(os.path.join(relative_base_dir, f)), root_dir)
-      for f in touched
-    ]
-    follow_symlinks = variables['OS'] != 'win'
-    # Expand the directories by listing each file inside. Up to now, trailing
-    # os.path.sep must be kept. Do not expand 'touched'.
-    infiles = expand_directories_and_symlinks(
-        root_dir,
-        infiles,
-        lambda x: re.match(r'.*\.(git|svn|pyc)$', x),
-        follow_symlinks,
-        ignore_broken_items)
-
-    # If we ignore broken items then remove any missing touched items.
-    if ignore_broken_items:
-      original_touched_count = len(touched)
-      touched = [touch for touch in touched if os.path.exists(touch)]
-
-      if len(touched) != original_touched_count:
-        logging.info('Removed %d invalid touched entries',
-                     len(touched) - original_touched_count)
-
-    # Finally, update the new data to be able to generate the foo.isolated file,
-    # the file that is used by run_isolated.py.
-    self.saved_state.update_isolated(
-        command, infiles, touched, read_only, relative_cwd)
-    logging.debug(self)
-
-  def process_inputs(self, subdir):
-    """Updates self.saved_state.files with the files' mode and hash.
-
-    If |subdir| is specified, filters to a subdirectory. The resulting .isolated
-    file is tainted.
-
-    See process_input() for more information.
-    """
-    for infile in sorted(self.saved_state.files):
-      if subdir and not infile.startswith(subdir):
-        self.saved_state.files.pop(infile)
-      else:
-        filepath = os.path.join(self.root_dir, infile)
-        self.saved_state.files[infile] = process_input(
-            filepath,
-            self.saved_state.files[infile],
-            self.saved_state.read_only,
-            self.saved_state.variables['OS'],
-            self.saved_state.algo)
-
-  def save_files(self):
-    """Saves self.saved_state and creates a .isolated file."""
-    logging.debug('Dumping to %s' % self.isolated_filepath)
-    self.saved_state.child_isolated_files = chromium_save_isolated(
-        self.isolated_filepath,
-        self.saved_state.to_isolated(),
-        self.saved_state.variables,
-        self.saved_state.algo)
-    total_bytes = sum(
-        i.get('s', 0) for i in self.saved_state.files.itervalues())
-    if total_bytes:
-      # TODO(maruel): Stats are missing the .isolated files.
-      logging.debug('Total size: %d bytes' % total_bytes)
-    saved_state_file = isolatedfile_to_state(self.isolated_filepath)
-    logging.debug('Dumping to %s' % saved_state_file)
-    trace_inputs.write_json(saved_state_file, self.saved_state.flatten(), True)
-
-  @property
-  def root_dir(self):
-    """Returns the absolute path of the root_dir to reference the .isolate file
-    via relative_cwd.
-
-    So that join(root_dir, relative_cwd, basename(isolate_file)) is equivalent
-    to isolate_filepath.
-    """
-    if not self.saved_state.isolate_file:
-      raise ExecutionError('Please specify --isolate')
-    isolate_dir = os.path.dirname(self.saved_state.isolate_filepath)
-    # Special case '.'.
-    if self.saved_state.relative_cwd == '.':
-      root_dir = isolate_dir
-    else:
-      if not isolate_dir.endswith(self.saved_state.relative_cwd):
-        raise ExecutionError(
-            ('Make sure the .isolate file is in the directory that will be '
-             'used as the relative directory. It is currently in %s and should '
-             'be in %s') % (isolate_dir, self.saved_state.relative_cwd))
-      # Walk back back to the root directory.
-      root_dir = isolate_dir[:-(len(self.saved_state.relative_cwd) + 1)]
-    return file_path.get_native_path_case(root_dir)
-
-  @property
-  def resultdir(self):
-    """Returns the absolute path containing the .isolated file.
-
-    It is usually equivalent to the variable PRODUCT_DIR. Uses the .isolated
-    path as the value.
-    """
-    return os.path.dirname(self.isolated_filepath)
-
-  def __str__(self):
-    def indent(data, indent_length):
-      """Indents text."""
-      spacing = ' ' * indent_length
-      return ''.join(spacing + l for l in str(data).splitlines(True))
-
-    out = '%s(\n' % self.__class__.__name__
-    out += '  root_dir: %s\n' % self.root_dir
-    out += '  saved_state: %s)' % indent(self.saved_state, 2)
-    return out
-
-
-def load_complete_state(options, cwd, subdir, skip_update):
-  """Loads a CompleteState.
-
-  This includes data from .isolate and .isolated.state files. Never reads the
-  .isolated file.
-
-  Arguments:
-    options: Options instance generated with OptionParserIsolate. For either
-             options.isolate and options.isolated, if the value is set, it is an
-             absolute path.
-    cwd: base directory to be used when loading the .isolate file.
-    subdir: optional argument to only process file in the subdirectory, relative
-            to CompleteState.root_dir.
-    skip_update: Skip trying to load the .isolate file and processing the
-                 dependencies. It is useful when not needed, like when tracing.
-  """
-  assert not options.isolate or os.path.isabs(options.isolate)
-  assert not options.isolated or os.path.isabs(options.isolated)
-  cwd = file_path.get_native_path_case(unicode(cwd))
-  if options.isolated:
-    # Load the previous state if it was present. Namely, "foo.isolated.state".
-    # Note: this call doesn't load the .isolate file.
-    complete_state = CompleteState.load_files(options.isolated)
-  else:
-    # Constructs a dummy object that cannot be saved. Useful for temporary
-    # commands like 'run'.
-    complete_state = CompleteState(None, SavedState())
-
-  if not options.isolate:
-    if not complete_state.saved_state.isolate_file:
-      if not skip_update:
-        raise ExecutionError('A .isolate file is required.')
-      isolate = None
-    else:
-      isolate = complete_state.saved_state.isolate_filepath
-  else:
-    isolate = options.isolate
-    if complete_state.saved_state.isolate_file:
-      rel_isolate = safe_relpath(
-          options.isolate, complete_state.saved_state.isolated_basedir)
-      if rel_isolate != complete_state.saved_state.isolate_file:
-        raise ExecutionError(
-            '%s and %s do not match.' % (
-              options.isolate, complete_state.saved_state.isolate_file))
-
-  if not skip_update:
-    # Then load the .isolate and expands directories.
-    complete_state.load_isolate(
-        cwd, isolate, options.variables, options.ignore_broken_items)
-
-  # Regenerate complete_state.saved_state.files.
-  if subdir:
-    subdir = unicode(subdir)
-    subdir = eval_variables(subdir, complete_state.saved_state.variables)
-    subdir = subdir.replace('/', os.path.sep)
-
-  if not skip_update:
-    complete_state.process_inputs(subdir)
-  return complete_state
-
-
-def read_trace_as_isolate_dict(complete_state, trace_blacklist):
-  """Reads a trace and returns the .isolate dictionary.
-
-  Returns exceptions during the log parsing so it can be re-raised.
-  """
-  api = trace_inputs.get_api()
-  logfile = complete_state.isolated_filepath + '.log'
-  if not os.path.isfile(logfile):
-    raise ExecutionError(
-        'No log file \'%s\' to read, did you forget to \'trace\'?' % logfile)
-  try:
-    data = api.parse_log(logfile, trace_blacklist, None)
-    exceptions = [i['exception'] for i in data if 'exception' in i]
-    results = (i['results'] for i in data if 'results' in i)
-    results_stripped = (i.strip_root(complete_state.root_dir) for i in results)
-    files = set(sum((result.existent for result in results_stripped), []))
-    tracked, touched = split_touched(files)
-    value = generate_isolate(
-        tracked,
-        [],
-        touched,
-        complete_state.root_dir,
-        complete_state.saved_state.variables,
-        complete_state.saved_state.relative_cwd,
-        trace_blacklist)
-    return value, exceptions
-  except trace_inputs.TracingFailure, e:
-    raise ExecutionError(
-        'Reading traces failed for: %s\n%s' %
-          (' '.join(complete_state.saved_state.command), str(e)))
-
-
-def print_all(comment, data, stream):
-  """Prints a complete .isolate file and its top-level file comment into a
-  stream.
-  """
-  if comment:
-    stream.write(comment)
-  pretty_print(data, stream)
-
-
-def merge(complete_state, trace_blacklist):
-  """Reads a trace and merges it back into the source .isolate file."""
-  value, exceptions = read_trace_as_isolate_dict(
-      complete_state, trace_blacklist)
-
-  # Now take that data and union it into the original .isolate file.
-  with open(complete_state.saved_state.isolate_filepath, 'r') as f:
-    prev_content = f.read()
-  isolate_dir = os.path.dirname(complete_state.saved_state.isolate_filepath)
-  prev_config = load_isolate_as_config(
-      isolate_dir,
-      eval_content(prev_content),
-      extract_comment(prev_content))
-  new_config = load_isolate_as_config(isolate_dir, value, '')
-  config = union(prev_config, new_config)
-  data = config.make_isolate_file()
-  print('Updating %s' % complete_state.saved_state.isolate_file)
-  with open(complete_state.saved_state.isolate_filepath, 'wb') as f:
-    print_all(config.file_comment, data, f)
-  if exceptions:
-    # It got an exception, raise the first one.
-    raise \
-        exceptions[0][0], \
-        exceptions[0][1], \
-        exceptions[0][2]
-
-
-### Commands.
-
-
-def CMDarchive(parser, args):
-  """Creates a .isolated file and uploads the tree to an isolate server.
-
-  All the files listed in the .isolated file are put in the isolate server
-  cache via isolateserver.py.
-  """
-  parser.add_option('--subdir', help='Filters to a subdirectory')
-  options, args = parser.parse_args(args)
-  if args:
-    parser.error('Unsupported argument: %s' % args)
-
-  with tools.Profiler('GenerateHashtable'):
-    success = False
-    try:
-      complete_state = load_complete_state(
-          options, os.getcwd(), options.subdir, False)
-      if not options.outdir:
-        options.outdir = os.path.join(
-            os.path.dirname(complete_state.isolated_filepath), 'hashtable')
-      # Make sure that complete_state isn't modified until save_files() is
-      # called, because any changes made to it here will propagate to the files
-      # created (which is probably not intended).
-      complete_state.save_files()
-
-      infiles = complete_state.saved_state.files
-      # Add all the .isolated files.
-      isolated_hash = []
-      isolated_files = [
-        options.isolated,
-      ] + complete_state.saved_state.child_isolated_files
-      for item in isolated_files:
-        item_path = os.path.join(
-            os.path.dirname(complete_state.isolated_filepath), item)
-        # Do not use isolateserver.hash_file() here because the file is
-        # likely smallish (under 500kb) and its file size is needed.
-        with open(item_path, 'rb') as f:
-          content = f.read()
-        isolated_hash.append(
-            complete_state.saved_state.algo(content).hexdigest())
-        isolated_metadata = {
-          'h': isolated_hash[-1],
-          's': len(content),
-          'priority': '0'
-        }
-        infiles[item_path] = isolated_metadata
-
-      logging.info('Creating content addressed object store with %d item',
-                   len(infiles))
-
-      if is_url(options.outdir):
-        isolateserver.upload_tree(
-            base_url=options.outdir,
-            indir=complete_state.root_dir,
-            infiles=infiles,
-            namespace='default-gzip')
-      else:
-        recreate_tree(
-            outdir=options.outdir,
-            indir=complete_state.root_dir,
-            infiles=infiles,
-            action=run_isolated.HARDLINK_WITH_FALLBACK,
-            as_hash=True)
-      success = True
-      print('%s  %s' % (isolated_hash[0], os.path.basename(options.isolated)))
-    finally:
-      # If the command failed, delete the .isolated file if it exists. This is
-      # important so no stale swarm job is executed.
-      if not success and os.path.isfile(options.isolated):
-        os.remove(options.isolated)
-  return not success
-
-
-def CMDcheck(parser, args):
-  """Checks that all the inputs are present and generates .isolated."""
-  parser.add_option('--subdir', help='Filters to a subdirectory')
-  options, args = parser.parse_args(args)
-  if args:
-    parser.error('Unsupported argument: %s' % args)
-
-  complete_state = load_complete_state(
-      options, os.getcwd(), options.subdir, False)
-
-  # Nothing is done specifically. Just store the result and state.
-  complete_state.save_files()
-  return 0
-
-
-CMDhashtable = CMDarchive
-
-
-def CMDmerge(parser, args):
-  """Reads and merges the data from the trace back into the original .isolate.
-
-  Ignores --outdir.
-  """
-  parser.require_isolated = False
-  add_trace_option(parser)
-  options, args = parser.parse_args(args)
-  if args:
-    parser.error('Unsupported argument: %s' % args)
-
-  complete_state = load_complete_state(options, os.getcwd(), None, False)
-  blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
-  merge(complete_state, blacklist)
-  return 0
-
-
-def CMDread(parser, args):
-  """Reads the trace file generated with command 'trace'.
-
-  Ignores --outdir.
-  """
-  parser.require_isolated = False
-  add_trace_option(parser)
-  parser.add_option(
-      '--skip-refresh', action='store_true',
-      help='Skip reading .isolate file and do not refresh the hash of '
-           'dependencies')
-  parser.add_option(
-      '-m', '--merge', action='store_true',
-      help='merge the results back in the .isolate file instead of printing')
-  options, args = parser.parse_args(args)
-  if args:
-    parser.error('Unsupported argument: %s' % args)
-
-  complete_state = load_complete_state(
-      options, os.getcwd(), None, options.skip_refresh)
-  blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
-  value, exceptions = read_trace_as_isolate_dict(complete_state, blacklist)
-  if options.merge:
-    merge(complete_state, blacklist)
-  else:
-    pretty_print(value, sys.stdout)
-
-  if exceptions:
-    # It got an exception, raise the first one.
-    raise \
-        exceptions[0][0], \
-        exceptions[0][1], \
-        exceptions[0][2]
-  return 0
-
-
-def CMDremap(parser, args):
-  """Creates a directory with all the dependencies mapped into it.
-
-  Useful to test manually why a test is failing. The target executable is not
-  run.
-  """
-  parser.require_isolated = False
-  options, args = parser.parse_args(args)
-  if args:
-    parser.error('Unsupported argument: %s' % args)
-  complete_state = load_complete_state(options, os.getcwd(), None, False)
-
-  if not options.outdir:
-    options.outdir = run_isolated.make_temp_dir(
-        'isolate', complete_state.root_dir)
-  else:
-    if is_url(options.outdir):
-      parser.error('Can\'t use url for --outdir with mode remap.')
-    if not os.path.isdir(options.outdir):
-      os.makedirs(options.outdir)
-  print('Remapping into %s' % options.outdir)
-  if len(os.listdir(options.outdir)):
-    raise ExecutionError('Can\'t remap in a non-empty directory')
-  recreate_tree(
-      outdir=options.outdir,
-      indir=complete_state.root_dir,
-      infiles=complete_state.saved_state.files,
-      action=run_isolated.HARDLINK_WITH_FALLBACK,
-      as_hash=False)
-  if complete_state.saved_state.read_only:
-    run_isolated.make_writable(options.outdir, True)
-
-  if complete_state.isolated_filepath:
-    complete_state.save_files()
-  return 0
-
-
-def CMDrewrite(parser, args):
-  """Rewrites a .isolate file into the canonical format."""
-  parser.require_isolated = False
-  options, args = parser.parse_args(args)
-  if args:
-    parser.error('Unsupported argument: %s' % args)
-
-  if options.isolated:
-    # Load the previous state if it was present. Namely, "foo.isolated.state".
-    complete_state = CompleteState.load_files(options.isolated)
-    isolate = options.isolate or complete_state.saved_state.isolate_filepath
-  else:
-    isolate = options.isolate
-  if not isolate:
-    parser.error('--isolate is required.')
-
-  with open(isolate, 'r') as f:
-    content = f.read()
-  config = load_isolate_as_config(
-      os.path.dirname(os.path.abspath(isolate)),
-      eval_content(content),
-      extract_comment(content))
-  data = config.make_isolate_file()
-  print('Updating %s' % isolate)
-  with open(isolate, 'wb') as f:
-    print_all(config.file_comment, data, f)
-  return 0
-
-
-@subcommand.usage('-- [extra arguments]')
-def CMDrun(parser, args):
-  """Runs the test executable in an isolated (temporary) directory.
-
-  All the dependencies are mapped into the temporary directory and the
-  directory is cleaned up after the target exits. Warning: if --outdir is
-  specified, it is deleted upon exit.
-
-  Argument processing stops at -- and these arguments are appended to the
-  command line of the target to run. For example, use:
-    isolate.py run --isolated foo.isolated -- --gtest_filter=Foo.Bar
-  """
-  parser.require_isolated = False
-  parser.add_option(
-      '--skip-refresh', action='store_true',
-      help='Skip reading .isolate file and do not refresh the hash of '
-           'dependencies')
-  options, args = parser.parse_args(args)
-  if options.outdir and is_url(options.outdir):
-    parser.error('Can\'t use url for --outdir with mode run.')
-
-  complete_state = load_complete_state(
-      options, os.getcwd(), None, options.skip_refresh)
-  cmd = complete_state.saved_state.command + args
-  if not cmd:
-    raise ExecutionError('No command to run.')
-
-  cmd = tools.fix_python_path(cmd)
-  try:
-    root_dir = complete_state.root_dir
-    if not options.outdir:
-      if not os.path.isabs(root_dir):
-        root_dir = os.path.join(os.path.dirname(options.isolated), root_dir)
-      options.outdir = run_isolated.make_temp_dir('isolate', root_dir)
-    else:
-      if not os.path.isdir(options.outdir):
-        os.makedirs(options.outdir)
-    recreate_tree(
-        outdir=options.outdir,
-        indir=root_dir,
-        infiles=complete_state.saved_state.files,
-        action=run_isolated.HARDLINK_WITH_FALLBACK,
-        as_hash=False)
-    cwd = os.path.normpath(
-        os.path.join(options.outdir, complete_state.saved_state.relative_cwd))
-    if not os.path.isdir(cwd):
-      # It can happen when no files are mapped from the directory containing the
-      # .isolate file. But the directory must exist to be the current working
-      # directory.
-      os.makedirs(cwd)
-    if complete_state.saved_state.read_only:
-      run_isolated.make_writable(options.outdir, True)
-    logging.info('Running %s, cwd=%s' % (cmd, cwd))
-    result = subprocess.call(cmd, cwd=cwd)
-  finally:
-    if options.outdir:
-      run_isolated.rmtree(options.outdir)
-
-  if complete_state.isolated_filepath:
-    complete_state.save_files()
-  return result
-
-
-@subcommand.usage('-- [extra arguments]')
-def CMDtrace(parser, args):
-  """Traces the target using trace_inputs.py.
-
-  It runs the executable without remapping it, and traces all the files it and
-  its child processes access. Then the 'merge' command can be used to generate
-  an updated .isolate file out of it or the 'read' command to print it out to
-  stdout.
-
-  Argument processing stops at -- and these arguments are appended to the
-  command line of the target to run. For example, use:
-    isolate.py trace --isolated foo.isolated -- --gtest_filter=Foo.Bar
-  """
-  add_trace_option(parser)
-  parser.add_option(
-      '-m', '--merge', action='store_true',
-      help='After tracing, merge the results back in the .isolate file')
-  parser.add_option(
-      '--skip-refresh', action='store_true',
-      help='Skip reading .isolate file and do not refresh the hash of '
-           'dependencies')
-  options, args = parser.parse_args(args)
-
-  complete_state = load_complete_state(
-      options, os.getcwd(), None, options.skip_refresh)
-  cmd = complete_state.saved_state.command + args
-  if not cmd:
-    raise ExecutionError('No command to run.')
-  cmd = tools.fix_python_path(cmd)
-  cwd = os.path.normpath(os.path.join(
-      unicode(complete_state.root_dir),
-      complete_state.saved_state.relative_cwd))
-  cmd[0] = os.path.normpath(os.path.join(cwd, cmd[0]))
-  if not os.path.isfile(cmd[0]):
-    raise ExecutionError(
-        'Tracing failed for: %s\nIt doesn\'t exit' % ' '.join(cmd))
-  logging.info('Running %s, cwd=%s' % (cmd, cwd))
-  api = trace_inputs.get_api()
-  logfile = complete_state.isolated_filepath + '.log'
-  api.clean_trace(logfile)
-  out = None
-  try:
-    with api.get_tracer(logfile) as tracer:
-      result, out = tracer.trace(
-          cmd,
-          cwd,
-          'default',
-          True)
-  except trace_inputs.TracingFailure, e:
-    raise ExecutionError('Tracing failed for: %s\n%s' % (' '.join(cmd), str(e)))
-
-  if result:
-    logging.error(
-        'Tracer exited with %d, which means the tests probably failed so the '
-        'trace is probably incomplete.', result)
-    logging.info(out)
-
-  complete_state.save_files()
-
-  if options.merge:
-    blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
-    merge(complete_state, blacklist)
-
-  return result
-
-
-def _process_variable_arg(_option, _opt, _value, parser):
-  if not parser.rargs:
-    raise optparse.OptionValueError(
-        'Please use --variable FOO=BAR or --variable FOO BAR')
-  k = parser.rargs.pop(0)
-  if '=' in k:
-    parser.values.variables.append(tuple(k.split('=', 1)))
-  else:
-    if not parser.rargs:
-      raise optparse.OptionValueError(
-          'Please use --variable FOO=BAR or --variable FOO BAR')
-    v = parser.rargs.pop(0)
-    parser.values.variables.append((k, v))
-
-
-def add_variable_option(parser):
-  """Adds --isolated and --variable to an OptionParser."""
-  parser.add_option(
-      '-s', '--isolated',
-      metavar='FILE',
-      help='.isolated file to generate or read')
-  # Keep for compatibility. TODO(maruel): Remove once not used anymore.
-  parser.add_option(
-      '-r', '--result',
-      dest='isolated',
-      help=optparse.SUPPRESS_HELP)
-  default_variables = [('OS', get_flavor())]
-  if sys.platform in ('win32', 'cygwin'):
-    default_variables.append(('EXECUTABLE_SUFFIX', '.exe'))
-  else:
-    default_variables.append(('EXECUTABLE_SUFFIX', ''))
-  parser.add_option(
-      '-V', '--variable',
-      action='callback',
-      callback=_process_variable_arg,
-      default=default_variables,
-      dest='variables',
-      metavar='FOO BAR',
-      help='Variables to process in the .isolate file, default: %default. '
-            'Variables are persistent accross calls, they are saved inside '
-            '<.isolated>.state')
-
-
-def add_trace_option(parser):
-  """Adds --trace-blacklist to the parser."""
-  parser.add_option(
-      '--trace-blacklist',
-      action='append', default=list(DEFAULT_BLACKLIST),
-      help='List of regexp to use as blacklist filter for files to consider '
-           'important, not to be confused with --blacklist which blacklists '
-           'test case.')
-
-
-def parse_isolated_option(parser, options, cwd, require_isolated):
-  """Processes --isolated."""
-  if options.isolated:
-    options.isolated = os.path.normpath(
-        os.path.join(cwd, options.isolated.replace('/', os.path.sep)))
-  if require_isolated and not options.isolated:
-    parser.error('--isolated is required.')
-  if options.isolated and not options.isolated.endswith('.isolated'):
-    parser.error('--isolated value must end with \'.isolated\'')
-
-
-def parse_variable_option(options):
-  """Processes --variable."""
-  # TODO(benrg): Maybe we should use a copy of gyp's NameValueListToDict here,
-  # but it wouldn't be backward compatible.
-  def try_make_int(s):
-    """Converts a value to int if possible, converts to unicode otherwise."""
-    try:
-      return int(s)
-    except ValueError:
-      return s.decode('utf-8')
-  options.variables = dict((k, try_make_int(v)) for k, v in options.variables)
-
-
-class OptionParserIsolate(tools.OptionParserWithLogging):
-  """Adds automatic --isolate, --isolated, --out and --variable handling."""
-  # Set it to False if it is not required, e.g. it can be passed on but do not
-  # fail if not given.
-  require_isolated = True
-
-  def __init__(self, **kwargs):
-    tools.OptionParserWithLogging.__init__(
-        self,
-        verbose=int(os.environ.get('ISOLATE_DEBUG', 0)),
-        **kwargs)
-    group = optparse.OptionGroup(self, "Common options")
-    group.add_option(
-        '-i', '--isolate',
-        metavar='FILE',
-        help='.isolate file to load the dependency data from')
-    add_variable_option(group)
-    group.add_option(
-        '-o', '--outdir', metavar='DIR',
-        help='Directory used to recreate the tree or store the hash table. '
-             'Defaults: run|remap: a /tmp subdirectory, others: '
-             'defaults to the directory containing --isolated')
-    group.add_option(
-        '--ignore_broken_items', action='store_true',
-        default=bool(os.environ.get('ISOLATE_IGNORE_BROKEN_ITEMS')),
-        help='Indicates that invalid entries in the isolated file to be '
-             'only be logged and not stop processing. Defaults to True if '
-             'env var ISOLATE_IGNORE_BROKEN_ITEMS is set')
-    self.add_option_group(group)
-
-  def parse_args(self, *args, **kwargs):
-    """Makes sure the paths make sense.
-
-    On Windows, / and \ are often mixed together in a path.
-    """
-    options, args = tools.OptionParserWithLogging.parse_args(
-        self, *args, **kwargs)
-    if not self.allow_interspersed_args and args:
-      self.error('Unsupported argument: %s' % args)
-
-    cwd = file_path.get_native_path_case(unicode(os.getcwd()))
-    parse_isolated_option(self, options, cwd, self.require_isolated)
-    parse_variable_option(options)
-
-    if options.isolate:
-      # TODO(maruel): Work with non-ASCII.
-      # The path must be in native path case for tracing purposes.
-      options.isolate = unicode(options.isolate).replace('/', os.path.sep)
-      options.isolate = os.path.normpath(os.path.join(cwd, options.isolate))
-      options.isolate = file_path.get_native_path_case(options.isolate)
-
-    if options.outdir and not is_url(options.outdir):
-      options.outdir = unicode(options.outdir).replace('/', os.path.sep)
-      # outdir doesn't need native path case since tracing is never done from
-      # there.
-      options.outdir = os.path.normpath(os.path.join(cwd, options.outdir))
-
-    return options, args
-
-
-def main(argv):
-  dispatcher = subcommand.CommandDispatcher(__name__)
-  try:
-    return dispatcher.execute(OptionParserIsolate(version=__version__), argv)
-  except Exception as e:
-    tools.report_error(e)
-    return 1
-
-
-if __name__ == '__main__':
-  fix_encoding.fix_encoding()
-  tools.disable_buffering()
-  colorama.init()
-  sys.exit(main(sys.argv[1:]))