Move grit from DEPS into src.

tools/grit/grit/tool/build.py

Index: tools/grit/grit/tool/build.py
diff --git a/tools/grit/grit/tool/build.py b/tools/grit/grit/tool/build.py
new file mode 100755
index 0000000000000000000000000000000000000000..65a966f5ac041b30a72a202a856c05f0dc025765
--- /dev/null
+++ b/tools/grit/grit/tool/build.py
@@ -0,0 +1,499 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+'''The 'grit build' tool along with integration for this tool with the
+SCons build system.
+'''
+
+import codecs
+import filecmp
+import getopt
+import os
+import shutil
+import sys
+
+from grit import grd_reader
+from grit import util
+from grit.tool import interface
+from grit import shortcuts
+
+
+# It would be cleaner to have each module register itself, but that would
+# require importing all of them on every run of GRIT.
+'''Map from <output> node types to modules under grit.format.'''
+_format_modules = {
+  'android':                  'android_xml',
+  'c_format':                 'c_format',
+  'chrome_messages_json':     'chrome_messages_json',
+  'data_package':             'data_pack',
+  'js_map_format':            'js_map_format',
+  'rc_all':                   'rc',
+  'rc_translateable':         'rc',
+  'rc_nontranslateable':      'rc',
+  'rc_header':                'rc_header',
+  'resource_map_header':      'resource_map',
+  'resource_map_source':      'resource_map',
+  'resource_file_map_source': 'resource_map',
+}
+_format_modules.update(
+    (type, 'policy_templates.template_formatter') for type in
+        [ 'adm', 'admx', 'adml', 'reg', 'doc', 'json',
+          'plist', 'plist_strings', 'ios_plist', 'android_policy' ])
+
+
+def GetFormatter(type):
+  modulename = 'grit.format.' + _format_modules[type]
+  __import__(modulename)
+  module = sys.modules[modulename]
+  try:
+    return module.Format
+  except AttributeError:
+    return module.GetFormatter(type)
+
+
+class RcBuilder(interface.Tool):
+  '''A tool that builds RC files and resource header files for compilation.
+
+Usage:  grit build [-o OUTPUTDIR] [-D NAME[=VAL]]*
+
+All output options for this tool are specified in the input file (see
+'grit help' for details on how to specify the input file - it is a global
+option).
+
+Options:
+
+  -a FILE           Assert that the given file is an output. There can be
+                    multiple "-a" flags listed for multiple outputs. If a "-a"
+                    or "--assert-file-list" argument is present, then the list
+                    of asserted files must match the output files or the tool
+                    will fail. The use-case is for the build system to maintain
+                    separate lists of output files and to catch errors if the
+                    build system's list and the grit list are out-of-sync.
+
+  --assert-file-list  Provide a file listing multiple asserted output files.
+                    There is one file name per line. This acts like specifying
+                    each file with "-a" on the command line, but without the
+                    possibility of running into OS line-length limits for very
+                    long lists.
+
+  -o OUTPUTDIR      Specify what directory output paths are relative to.
+                    Defaults to the current directory.
+
+  -D NAME[=VAL]     Specify a C-preprocessor-like define NAME with optional
+                    value VAL (defaults to 1) which will be used to control
+                    conditional inclusion of resources.
+
+  -E NAME=VALUE     Set environment variable NAME to VALUE (within grit).
+
+  -f FIRSTIDSFILE   Path to a python file that specifies the first id of
+                    value to use for resources.  A non-empty value here will
+                    override the value specified in the <grit> node's
+                    first_ids_file.
+
+  -w WHITELISTFILE  Path to a file containing the string names of the
+                    resources to include.  Anything not listed is dropped.
+
+  -t PLATFORM       Specifies the platform the build is targeting; defaults
+                    to the value of sys.platform. The value provided via this
+                    flag should match what sys.platform would report for your
+                    target platform; see grit.node.base.EvaluateCondition.
+
+  -h HEADERFORMAT   Custom format string to use for generating rc header files.
+                    The string should have two placeholders: {textual_id}
+                    and {numeric_id}. E.g. "#define {textual_id} {numeric_id}"
+                    Otherwise it will use the default "#define SYMBOL 1234"
+
+  --output-all-resource-defines
+  --no-output-all-resource-defines  If specified, overrides the value of the
+                    output_all_resource_defines attribute of the root <grit>
+                    element of the input .grd file.
+
+  --write-only-new flag
+                    If flag is non-0, write output files to a temporary file
+                    first, and copy it to the real output only if the new file
+                    is different from the old file.  This allows some build
+                    systems to realize that dependent build steps might be
+                    unnecessary, at the cost of comparing the output data at
+                    grit time.
+
+  --depend-on-stamp
+                    If specified along with --depfile and --depdir, the depfile
+                    generated will depend on a stampfile instead of the first
+                    output in the input .grd file.
+
+Conditional inclusion of resources only affects the output of files which
+control which resources get linked into a binary, e.g. it affects .rc files
+meant for compilation but it does not affect resource header files (that define
+IDs).  This helps ensure that values of IDs stay the same, that all messages
+are exported to translation interchange files (e.g. XMB files), etc.
+'''
+
+  def ShortDescription(self):
+    return 'A tool that builds RC files for compilation.'
+
+  def Run(self, opts, args):
+    self.output_directory = '.'
+    first_ids_file = None
+    whitelist_filenames = []
+    assert_output_files = []
+    target_platform = None
+    depfile = None
+    depdir = None
+    rc_header_format = None
+    output_all_resource_defines = None
+    write_only_new = False
+    depend_on_stamp = False
+    (own_opts, args) = getopt.getopt(args, 'a:o:D:E:f:w:t:h:',
+        ('depdir=','depfile=','assert-file-list=',
+         'output-all-resource-defines',
+         'no-output-all-resource-defines',
+         'depend-on-stamp',
+         'write-only-new='))
+    for (key, val) in own_opts:
+      if key == '-a':
+        assert_output_files.append(val)
+      elif key == '--assert-file-list':
+        with open(val) as f:
+          assert_output_files += f.read().splitlines()
+      elif key == '-o':
+        self.output_directory = val
+      elif key == '-D':
+        name, val = util.ParseDefine(val)
+        self.defines[name] = val
+      elif key == '-E':
+        (env_name, env_value) = val.split('=', 1)
+        os.environ[env_name] = env_value
+      elif key == '-f':
+        # TODO(joi@chromium.org): Remove this override once change
+        # lands in WebKit.grd to specify the first_ids_file in the
+        # .grd itself.
+        first_ids_file = val
+      elif key == '-w':
+        whitelist_filenames.append(val)
+      elif key == '--output-all-resource-defines':
+        output_all_resource_defines = True
+      elif key == '--no-output-all-resource-defines':
+        output_all_resource_defines = False
+      elif key == '-t':
+        target_platform = val
+      elif key == '-h':
+        rc_header_format = val
+      elif key == '--depdir':
+        depdir = val
+      elif key == '--depfile':
+        depfile = val
+      elif key == '--write-only-new':
+        write_only_new = val != '0'
+      elif key == '--depend-on-stamp':
+        depend_on_stamp = True
+
+    if len(args):
+      print 'This tool takes no tool-specific arguments.'
+      return 2
+    self.SetOptions(opts)
+    if self.scons_targets:
+      self.VerboseOut('Using SCons targets to identify files to output.\n')
+    else:
+      self.VerboseOut('Output directory: %s (absolute path: %s)\n' %
+                      (self.output_directory,
+                       os.path.abspath(self.output_directory)))
+
+    if whitelist_filenames:
+      self.whitelist_names = set()
+      for whitelist_filename in whitelist_filenames:
+        self.VerboseOut('Using whitelist: %s\n' % whitelist_filename);
+        whitelist_contents = util.ReadFile(whitelist_filename, util.RAW_TEXT)
+        self.whitelist_names.update(whitelist_contents.strip().split('\n'))
+
+    self.write_only_new = write_only_new
+
+    self.res = grd_reader.Parse(opts.input,
+                                debug=opts.extra_verbose,
+                                first_ids_file=first_ids_file,
+                                defines=self.defines,
+                                target_platform=target_platform)
+
+    # If the output_all_resource_defines option is specified, override the value
+    # found in the grd file.
+    if output_all_resource_defines is not None:
+      self.res.SetShouldOutputAllResourceDefines(output_all_resource_defines)
+
+    # Set an output context so that conditionals can use defines during the
+    # gathering stage; we use a dummy language here since we are not outputting
+    # a specific language.
+    self.res.SetOutputLanguage('en')
+    if rc_header_format:
+      self.res.AssignRcHeaderFormat(rc_header_format)
+    self.res.RunGatherers()
+    self.Process()
+
+    if assert_output_files:
+      if not self.CheckAssertedOutputFiles(assert_output_files):
+        return 2
+
+    if depfile and depdir:
+      self.GenerateDepfile(depfile, depdir, first_ids_file, depend_on_stamp)
+
+    return 0
+
+  def __init__(self, defines=None):
+    # Default file-creation function is codecs.open().  Only done to allow
+    # overriding by unit test.
+    self.fo_create = codecs.open
+
+    # key/value pairs of C-preprocessor like defines that are used for
+    # conditional output of resources
+    self.defines = defines or {}
+
+    # self.res is a fully-populated resource tree if Run()
+    # has been called, otherwise None.
+    self.res = None
+
+    # Set to a list of filenames for the output nodes that are relative
+    # to the current working directory.  They are in the same order as the
+    # output nodes in the file.
+    self.scons_targets = None
+
+    # The set of names that are whitelisted to actually be included in the
+    # output.
+    self.whitelist_names = None
+
+    # Whether to compare outputs to their old contents before writing.
+    self.write_only_new = False
+
+  @staticmethod
+  def AddWhitelistTags(start_node, whitelist_names):
+    # Walk the tree of nodes added attributes for the nodes that shouldn't
+    # be written into the target files (skip markers).
+    from grit.node import include
+    from grit.node import message
+    from grit.node import structure
+    for node in start_node:
+      # Same trick data_pack.py uses to see what nodes actually result in
+      # real items.
+      if (isinstance(node, include.IncludeNode) or
+          isinstance(node, message.MessageNode) or
+          isinstance(node, structure.StructureNode)):
+        text_ids = node.GetTextualIds()
+        # Mark the item to be skipped if it wasn't in the whitelist.
+        if text_ids and text_ids[0] not in whitelist_names:
+          node.SetWhitelistMarkedAsSkip(True)
+
+  @staticmethod
+  def ProcessNode(node, output_node, outfile):
+    '''Processes a node in-order, calling its formatter before and after
+    recursing to its children.
+
+    Args:
+      node: grit.node.base.Node subclass
+      output_node: grit.node.io.OutputNode
+      outfile: open filehandle
+    '''
+    base_dir = util.dirname(output_node.GetOutputFilename())
+
+    formatter = GetFormatter(output_node.GetType())
+    formatted = formatter(node, output_node.GetLanguage(), output_dir=base_dir)
+    outfile.writelines(formatted)
+
+
+  def Process(self):
+    # Update filenames with those provided by SCons if we're being invoked
+    # from SCons.  The list of SCons targets also includes all <structure>
+    # node outputs, but it starts with our output files, in the order they
+    # occur in the .grd
+    if self.scons_targets:
+      assert len(self.scons_targets) >= len(self.res.GetOutputFiles())
+      outfiles = self.res.GetOutputFiles()
+      for ix in range(len(outfiles)):
+        outfiles[ix].output_filename = os.path.abspath(
+          self.scons_targets[ix])
+    else:
+      for output in self.res.GetOutputFiles():
+        output.output_filename = os.path.abspath(os.path.join(
+          self.output_directory, output.GetFilename()))
+
+    # If there are whitelisted names, tag the tree once up front, this way
+    # while looping through the actual output, it is just an attribute check.
+    if self.whitelist_names:
+      self.AddWhitelistTags(self.res, self.whitelist_names)
+
+    for output in self.res.GetOutputFiles():
+      self.VerboseOut('Creating %s...' % output.GetFilename())
+
+      # Microsoft's RC compiler can only deal with single-byte or double-byte
+      # files (no UTF-8), so we make all RC files UTF-16 to support all
+      # character sets.
+      if output.GetType() in ('rc_header', 'resource_map_header',
+          'resource_map_source', 'resource_file_map_source'):
+        encoding = 'cp1252'
+      elif output.GetType() in ('android', 'c_format', 'js_map_format', 'plist',
+                                'plist_strings', 'doc', 'json', 'android_policy'):
+        encoding = 'utf_8'
+      elif output.GetType() in ('chrome_messages_json'):
+        # Chrome Web Store currently expects BOM for UTF-8 files :-(
+        encoding = 'utf-8-sig'
+      else:
+        # TODO(gfeher) modify here to set utf-8 encoding for admx/adml
+        encoding = 'utf_16'
+
+      # Set the context, for conditional inclusion of resources
+      self.res.SetOutputLanguage(output.GetLanguage())
+      self.res.SetOutputContext(output.GetContext())
+      self.res.SetFallbackToDefaultLayout(output.GetFallbackToDefaultLayout())
+      self.res.SetDefines(self.defines)
+
+      # Make the output directory if it doesn't exist.
+      self.MakeDirectoriesTo(output.GetOutputFilename())
+
+      # Write the results to a temporary file and only overwrite the original
+      # if the file changed.  This avoids unnecessary rebuilds.
+      outfile = self.fo_create(output.GetOutputFilename() + '.tmp', 'wb')
+
+      if output.GetType() != 'data_package':
+        outfile = util.WrapOutputStream(outfile, encoding)
+
+      # Iterate in-order through entire resource tree, calling formatters on
+      # the entry into a node and on exit out of it.
+      with outfile:
+        self.ProcessNode(self.res, output, outfile)
+
+      # Now copy from the temp file back to the real output, but on Windows,
+      # only if the real output doesn't exist or the contents of the file
+      # changed.  This prevents identical headers from being written and .cc
+      # files from recompiling (which is painful on Windows).
+      if not os.path.exists(output.GetOutputFilename()):
+        os.rename(output.GetOutputFilename() + '.tmp',
+                  output.GetOutputFilename())
+      else:
+        # CHROMIUM SPECIFIC CHANGE.
+        # This clashes with gyp + vstudio, which expect the output timestamp
+        # to change on a rebuild, even if nothing has changed, so only do
+        # it when opted in.
+        if not self.write_only_new:
+          write_file = True
+        else:
+          files_match = filecmp.cmp(output.GetOutputFilename(),
+              output.GetOutputFilename() + '.tmp')
+          write_file = not files_match
+        if write_file:
+          shutil.copy2(output.GetOutputFilename() + '.tmp',
+                       output.GetOutputFilename())
+        os.remove(output.GetOutputFilename() + '.tmp')
+
+      self.VerboseOut(' done.\n')
+
+    # Print warnings if there are any duplicate shortcuts.
+    warnings = shortcuts.GenerateDuplicateShortcutsWarnings(
+        self.res.UberClique(), self.res.GetTcProject())
+    if warnings:
+      print '\n'.join(warnings)
+
+    # Print out any fallback warnings, and missing translation errors, and
+    # exit with an error code if there are missing translations in a non-pseudo
+    # and non-official build.
+    warnings = (self.res.UberClique().MissingTranslationsReport().
+        encode('ascii', 'replace'))
+    if warnings:
+      self.VerboseOut(warnings)
+    if self.res.UberClique().HasMissingTranslations():
+      print self.res.UberClique().missing_translations_
+      sys.exit(-1)
+
+
+  def CheckAssertedOutputFiles(self, assert_output_files):
+    '''Checks that the asserted output files are specified in the given list.
+
+    Returns true if the asserted files are present. If they are not, returns
+    False and prints the failure.
+    '''
+    # Compare the absolute path names, sorted.
+    asserted = sorted([os.path.abspath(i) for i in assert_output_files])
+    actual = sorted([
+        os.path.abspath(os.path.join(self.output_directory, i.GetFilename()))
+        for i in self.res.GetOutputFiles()])
+
+    if asserted != actual:
+      missing = list(set(actual) - set(asserted))
+      extra = list(set(asserted) - set(actual))
+      error = '''Asserted file list does not match.
+
+Expected output files:
+%s
+Actual output files:
+%s
+Missing output files:
+%s
+Extra output files:
+%s
+'''
+      print error % ('\n'.join(asserted), '\n'.join(actual), '\n'.join(missing),
+          '\n'.join(extra))
+      return False
+    return True
+
+
+  def GenerateDepfile(self, depfile, depdir, first_ids_file, depend_on_stamp):
+    '''Generate a depfile that contains the imlicit dependencies of the input
+    grd. The depfile will be in the same format as a makefile, and will contain
+    references to files relative to |depdir|. It will be put in |depfile|.
+
+    For example, supposing we have three files in a directory src/
+
+    src/
+      blah.grd    <- depends on input{1,2}.xtb
+      input1.xtb
+      input2.xtb
+
+    and we run
+
+      grit -i blah.grd -o ../out/gen --depdir ../out --depfile ../out/gen/blah.rd.d
+
+    from the directory src/ we will generate a depfile ../out/gen/blah.grd.d
+    that has the contents
+
+      gen/blah.h: ../src/input1.xtb ../src/input2.xtb
+
+    Where "gen/blah.h" is the first output (Ninja expects the .d file to list
+    the first output in cases where there is more than one). If the flag
+    --depend-on-stamp is specified, "gen/blah.rd.d.stamp" will be used that is
+    'touched' whenever a new depfile is generated.
+
+    Note that all paths in the depfile are relative to ../out, the depdir.
+    '''
+    depfile = os.path.abspath(depfile)
+    depdir = os.path.abspath(depdir)
+    infiles = self.res.GetInputFiles()
+
+    # We want to trigger a rebuild if the first ids change.
+    if first_ids_file is not None:
+      infiles.append(first_ids_file)
+
+    if (depend_on_stamp):
+      output_file = depfile + ".stamp"
+      # Touch the stamp file before generating the depfile.
+      with open(output_file, 'a'):
+        os.utime(output_file, None)
+    else:
+      # Get the first output file relative to the depdir.
+      outputs = self.res.GetOutputFiles()
+      output_file = os.path.join(self.output_directory,
+                                 outputs[0].GetFilename())
+
+    output_file = os.path.relpath(output_file, depdir)
+    # The path prefix to prepend to dependencies in the depfile.
+    prefix = os.path.relpath(os.getcwd(), depdir)
+    deps_text = ' '.join([os.path.join(prefix, i) for i in infiles])
+
+    depfile_contents = output_file + ': ' + deps_text
+    self.MakeDirectoriesTo(depfile)
+    outfile = self.fo_create(depfile, 'w', encoding='utf-8')
+    outfile.writelines(depfile_contents)
+
+  @staticmethod
+  def MakeDirectoriesTo(file):
+    '''Creates directories necessary to contain |file|.'''
+    dir = os.path.split(file)[0]
+    if not os.path.exists(dir):
+      os.makedirs(dir)