Switch trace_inputs.py and isolate.py to subcommand.py.

isolate.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Front end tool to manage .isolate files and corresponding tests.
+"""Front end tool to operate on .isolate files.
 
-Run ./isolate.py --help for more detailed information.
+This includes creating, merging or compiling them to generate a .isolated file.
 
 See more information at
-https://code.google.com/p/swarming/wiki/IsolateDesign
-https://code.google.com/p/swarming/wiki/IsolateUserGuide
+  https://code.google.com/p/swarming/wiki/IsolateDesign
+  https://code.google.com/p/swarming/wiki/IsolateUserGuide
 """
+# Run ./isolate.py --help for more detailed information.
 
 import ast
 import copy
 import hashlib
 import itertools
 import logging
 import optparse
 import os
 import posixpath
 import re
 import stat
 import subprocess
 import sys
 
 import isolateserver_archive
 import run_isolated
 import short_expression_finder
 import trace_inputs
 
 # Import here directly so isolate is easier to use as a library.
 from run_isolated import get_flavor
 
+from third_party import colorama
+from third_party.depot_tools import fix_encoding
+from third_party.depot_tools import subcommand
+
 
 PATH_VARIABLES = ('DEPTH', 'PRODUCT_DIR')
 
 # Files that should be 0-length when mapped.
 KEY_TOUCHED = 'isolate_dependency_touched'
 # Files that should be tracked by the build tool.
 KEY_TRACKED = 'isolate_dependency_tracked'
 # Files that should not be tracked by the build tool.
 KEY_UNTRACKED = 'isolate_dependency_untracked'
 
@@ skipping 1859 matching lines @@
   with open(complete_state.saved_state.isolate_filepath, 'wb') as f:
     print_all(config.file_comment, data, f)
   if exceptions:
     # It got an exception, raise the first one.
     raise \
         exceptions[0][0], \
         exceptions[0][1], \
         exceptions[0][2]
 
 
-def CMDcheck(args):
+def CMDcheck(parser, args):
   """Checks that all the inputs are present and generates .isolated."""
-  parser = OptionParserIsolate(command='check')
   parser.add_option('--subdir', help='Filters to a subdirectory')
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
   complete_state = load_complete_state(
       options, os.getcwd(), options.subdir, False)
 
   # Nothing is done specifically. Just store the result and state.
   complete_state.save_files()
   return 0
 
 
-def CMDhashtable(args):
+def CMDhashtable(parser, args):
   """Creates a hash table content addressed object store.
 
   All the files listed in the .isolated file are put in the output directory
   with the file name being the sha-1 of the file's content.
   """
-  parser = OptionParserIsolate(command='hashtable')
   parser.add_option('--subdir', help='Filters to a subdirectory')
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
 
   with run_isolated.Profiler('GenerateHashtable'):
     success = False
     try:
       complete_state = load_complete_state(
           options, os.getcwd(), options.subdir, False)
@@ skipping 45 matching lines @@
       success = True
       print('%s  %s' % (isolated_hash[0], os.path.basename(options.isolated)))
     finally:
       # If the command failed, delete the .isolated file if it exists. This is
       # important so no stale swarm job is executed.
       if not success and os.path.isfile(options.isolated):
         os.remove(options.isolated)
   return not success
 
 
-def CMDmerge(args):
+def CMDmerge(parser, args):
   """Reads and merges the data from the trace back into the original .isolate.
 
   Ignores --outdir.
   """
-  parser = OptionParserIsolate(command='merge', require_isolated=False)
+  parser.require_isolated = False
   add_trace_option(parser)
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
   complete_state = load_complete_state(options, os.getcwd(), None, False)
   blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
   merge(complete_state, blacklist)
   return 0
 
 
-def CMDread(args):
+def CMDread(parser, args):
   """Reads the trace file generated with command 'trace'.
 
   Ignores --outdir.
   """
-  parser = OptionParserIsolate(command='read', require_isolated=False)
+  parser.require_isolated = False
   add_trace_option(parser)
   parser.add_option(
       '--skip-refresh', action='store_true',
       help='Skip reading .isolate file and do not refresh the sha1 of '
            'dependencies')
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
   complete_state = load_complete_state(
       options, os.getcwd(), None, options.skip_refresh)
   blacklist = trace_inputs.gen_blacklist(options.trace_blacklist)
   value, exceptions = read_trace_as_isolate_dict(complete_state, blacklist)
   pretty_print(value, sys.stdout)
   if exceptions:
     # It got an exception, raise the first one.
     raise \
         exceptions[0][0], \
         exceptions[0][1], \
         exceptions[0][2]
   return 0
 
 
-def CMDremap(args):
+def CMDremap(parser, args):
   """Creates a directory with all the dependencies mapped into it.
 
   Useful to test manually why a test is failing. The target executable is not
   run.
   """
-  parser = OptionParserIsolate(command='remap', require_isolated=False)
+  parser.require_isolated = False
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
   complete_state = load_complete_state(options, os.getcwd(), None, False)
 
   if not options.outdir:
     options.outdir = run_isolated.make_temp_dir(
         'isolate', complete_state.root_dir)
   else:
     if is_url(options.outdir):
@@ skipping 10 matching lines @@
       action=run_isolated.HARDLINK_WITH_FALLBACK,
       as_sha1=False)
   if complete_state.saved_state.read_only:
     run_isolated.make_writable(options.outdir, True)
 
   if complete_state.isolated_filepath:
     complete_state.save_files()
   return 0
 
 
-def CMDrewrite(args):
+def CMDrewrite(parser, args):
   """Rewrites a .isolate file into the canonical format."""
-  parser = OptionParserIsolate(command='rewrite', require_isolated=False)
+  parser.require_isolated = False
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
 
   if options.isolated:
     # Load the previous state if it was present. Namely, "foo.isolated.state".
     complete_state = CompleteState.load_files(options.isolated)
     isolate = options.isolate or complete_state.saved_state.isolate_filepath
   else:
     isolate = options.isolate
   if not isolate:
     raise ExecutionError('A .isolate file is required.')
   with open(isolate, 'r') as f:
     content = f.read()
   config = load_isolate_as_config(
       os.path.dirname(os.path.abspath(isolate)),
       eval_content(content),
       extract_comment(content))
   data = config.make_isolate_file()
   print('Updating %s' % isolate)
   with open(isolate, 'wb') as f:
     print_all(config.file_comment, data, f)
   return 0
 
 
-def CMDrun(args):
+def CMDrun(parser, args):
   """Runs the test executable in an isolated (temporary) directory.
 
   All the dependencies are mapped into the temporary directory and the
   directory is cleaned up after the target exits. Warning: if --outdir is
   specified, it is deleted upon exit.
 
   Argument processing stops at the first non-recognized argument and these
   arguments are appended to the command line of the target to run. For example,
   use: isolate.py --isolated foo.isolated -- --gtest_filter=Foo.Bar
   """
-  parser = OptionParserIsolate(command='run', require_isolated=False)
+  parser.require_isolated = False
   parser.add_option(
       '--skip-refresh', action='store_true',
       help='Skip reading .isolate file and do not refresh the sha1 of '
            'dependencies')
   parser.enable_interspersed_args()
   options, args = parser.parse_args(args)
   complete_state = load_complete_state(
       options, os.getcwd(), None, options.skip_refresh)
   cmd = complete_state.saved_state.command + args
   if not cmd:
@@ skipping 30 matching lines @@
     result = subprocess.call(cmd, cwd=cwd)
   finally:
     if options.outdir:
       run_isolated.rmtree(options.outdir)
 
   if complete_state.isolated_filepath:
     complete_state.save_files()
   return result
 
 
-def CMDtrace(args):
+def CMDtrace(parser, args):
   """Traces the target using trace_inputs.py.
 
   It runs the executable without remapping it, and traces all the files it and
   its child processes access. Then the 'merge' command can be used to generate
   an updated .isolate file out of it or the 'read' command to print it out to
   stdout.
 
   Argument processing stops at the first non-recognized argument and these
   arguments are appended to the command line of the target to run. For example,
   use: isolate.py --isolated foo.isolated -- --gtest_filter=Foo.Bar
   """
-  parser = OptionParserIsolate(command='trace')
   add_trace_option(parser)
   parser.enable_interspersed_args()
   parser.add_option(
       '-m', '--merge', action='store_true',
       help='After tracing, merge the results back in the .isolate file')
   parser.add_option(
       '--skip-refresh', action='store_true',
       help='Skip reading .isolate file and do not refresh the sha1 of '
            'dependencies')
   options, args = parser.parse_args(args)
@@ skipping 110 matching lines @@
   # but it wouldn't be backward compatible.
   def try_make_int(s):
     """Converts a value to int if possible, converts to unicode otherwise."""
     try:
       return int(s)
     except ValueError:
       return s.decode('utf-8')
   options.variables = dict((k, try_make_int(v)) for k, v in options.variables)
 
 
-class OptionParserIsolate(trace_inputs.OptionParserWithNiceDescription):
+class OptionParserIsolate(trace_inputs.OptionParserWithLogging):
   """Adds automatic --isolate, --isolated, --out and --variable handling."""
-  def __init__(self, require_isolated=True, **kwargs):
-    trace_inputs.OptionParserWithNiceDescription.__init__(
+  # Set it to False if it is not required, e.g. it can be passed on but do not
+  # fail if not given.
+  require_isolated = True
+
+  def __init__(self, **kwargs):
+    trace_inputs.OptionParserWithLogging.__init__(
         self,
         verbose=int(os.environ.get('ISOLATE_DEBUG', 0)),
         **kwargs)
     group = optparse.OptionGroup(self, "Common options")
     group.add_option(
         '-i', '--isolate',
         metavar='FILE',
         help='.isolate file to load the dependency data from')
     add_variable_option(group)
     group.add_option(
         '-o', '--outdir', metavar='DIR',
         help='Directory used to recreate the tree or store the hash table. '
              'Defaults: run|remap: a /tmp subdirectory, others: '
              'defaults to the directory containing --isolated')
     group.add_option(
         '--ignore_broken_items', action='store_true',
         default=bool(os.environ.get('ISOLATE_IGNORE_BROKEN_ITEMS')),
         help='Indicates that invalid entries in the isolated file to be '
              'only be logged and not stop processing. Defaults to True if '
              'env var ISOLATE_IGNORE_BROKEN_ITEMS is set')
     self.add_option_group(group)
-    self.require_isolated = require_isolated
 
   def parse_args(self, *args, **kwargs):
     """Makes sure the paths make sense.
 
     On Windows, / and \ are often mixed together in a path.
     """
-    options, args = trace_inputs.OptionParserWithNiceDescription.parse_args(
+    options, args = trace_inputs.OptionParserWithLogging.parse_args(
         self, *args, **kwargs)
     if not self.allow_interspersed_args and args:
       self.error('Unsupported argument: %s' % args)
 
     cwd = trace_inputs.get_native_path_case(unicode(os.getcwd()))
     parse_isolated_option(self, options, cwd, self.require_isolated)
     parse_variable_option(options)
 
     if options.isolate:
       # TODO(maruel): Work with non-ASCII.
       # The path must be in native path case for tracing purposes.
       options.isolate = unicode(options.isolate).replace('/', os.path.sep)
       options.isolate = os.path.normpath(os.path.join(cwd, options.isolate))
       options.isolate = trace_inputs.get_native_path_case(options.isolate)
 
     if options.outdir and not is_url(options.outdir):
       options.outdir = unicode(options.outdir).replace('/', os.path.sep)
       # outdir doesn't need native path case since tracing is never done from
       # there.
       options.outdir = os.path.normpath(os.path.join(cwd, options.outdir))
 
     return options, args
 
 
-### Glue code to make all the commands works magically.
-
-
-CMDhelp = trace_inputs.CMDhelp
-
-
 def main(argv):
+  dispatcher = subcommand.CommandDispatcher(__name__)
   try:
-    return trace_inputs.main_impl(argv)
+    return dispatcher.execute(OptionParserIsolate(), argv)
   except (
       ExecutionError,
       run_isolated.MappingError,
       run_isolated.ConfigError) as e:
     sys.stderr.write('\nError: ')
     sys.stderr.write(str(e))
     sys.stderr.write('\n')
     return 1
 
 
 if __name__ == '__main__':
+  fix_encoding.fix_encoding()
+  trace_inputs.disable_buffering()
+  colorama.init()
   sys.exit(main(sys.argv[1:]))