Move grit from DEPS into src.

tools/grit/grit/node/base.py

Index: tools/grit/grit/node/base.py
diff --git a/tools/grit/grit/node/base.py b/tools/grit/grit/node/base.py
new file mode 100755
index 0000000000000000000000000000000000000000..a40794b159dd0104e0f1f32daa09da01b4b4bc8b
--- /dev/null
+++ b/tools/grit/grit/node/base.py
@@ -0,0 +1,613 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+'''Base types for nodes in a GRIT resource tree.
+'''
+
+import ast
+import os
+import types
+from xml.sax import saxutils
+
+from grit import clique
+from grit import exception
+from grit import util
+
+
+class Node(object):
+  '''An item in the tree that has children.'''
+
+  # Valid content types that can be returned by _ContentType()
+  _CONTENT_TYPE_NONE = 0   # No CDATA content but may have children
+  _CONTENT_TYPE_CDATA = 1  # Only CDATA, no children.
+  _CONTENT_TYPE_MIXED = 2  # CDATA and children, possibly intermingled
+
+  # Default nodes to not whitelist skipped
+  _whitelist_marked_as_skip = False
+
+  # A class-static cache to speed up EvaluateExpression().
+  # Keys are expressions (e.g. 'is_ios and lang == "fr"'). Values are tuples
+  # (code, variables_in_expr) where code is the compiled expression and can be
+  # directly eval'd, and variables_in_expr is the list of variable and method
+  # names used in the expression (e.g. ['is_ios', 'lang']).
+  eval_expr_cache = {}
+
+  def __init__(self):
+    self.children = []        # A list of child elements
+    self.mixed_content = []   # A list of u'' and/or child elements (this
+                              # duplicates 'children' but
+                              # is needed to preserve markup-type content).
+    self.name = u''           # The name of this element
+    self.attrs = {}           # The set of attributes (keys to values)
+    self.parent = None        # Our parent unless we are the root element.
+    self.uberclique = None    # Allows overriding uberclique for parts of tree
+
+  # This context handler allows you to write "with node:" and get a
+  # line identifying the offending node if an exception escapes from the body
+  # of the with statement.
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    if exc_type is not None:
+      print u'Error processing node %s' % unicode(self)
+
+  def __iter__(self):
+    '''A preorder iteration through the tree that this node is the root of.'''
+    return self.Preorder()
+
+  def Preorder(self):
+    '''Generator that generates first this node, then the same generator for
+    any child nodes.'''
+    yield self
+    for child in self.children:
+      for iterchild in child.Preorder():
+        yield iterchild
+
+  def ActiveChildren(self):
+    '''Returns the children of this node that should be included in the current
+    configuration. Overridden by <if>.'''
+    return [node for node in self.children if not node.WhitelistMarkedAsSkip()]
+
+  def ActiveDescendants(self):
+    '''Yields the current node and all descendants that should be included in
+    the current configuration, in preorder.'''
+    yield self
+    for child in self.ActiveChildren():
+      for descendant in child.ActiveDescendants():
+        yield descendant
+
+  def GetRoot(self):
+    '''Returns the root Node in the tree this Node belongs to.'''
+    curr = self
+    while curr.parent:
+      curr = curr.parent
+    return curr
+
+    # TODO(joi) Use this (currently untested) optimization?:
+    #if hasattr(self, '_root'):
+    #  return self._root
+    #curr = self
+    #while curr.parent and not hasattr(curr, '_root'):
+    #  curr = curr.parent
+    #if curr.parent:
+    #  self._root = curr._root
+    #else:
+    #  self._root = curr
+    #return self._root
+
+  def StartParsing(self, name, parent):
+    '''Called at the start of parsing.
+
+    Args:
+      name: u'elementname'
+      parent: grit.node.base.Node or subclass or None
+    '''
+    assert isinstance(name, types.StringTypes)
+    assert not parent or isinstance(parent, Node)
+    self.name = name
+    self.parent = parent
+
+  def AddChild(self, child):
+    '''Adds a child to the list of children of this node, if it is a valid
+    child for the node.'''
+    assert isinstance(child, Node)
+    if (not self._IsValidChild(child) or
+        self._ContentType() == self._CONTENT_TYPE_CDATA):
+      explanation = 'invalid child %s for parent %s' % (str(child), self.name)
+      raise exception.UnexpectedChild(explanation)
+    self.children.append(child)
+    self.mixed_content.append(child)
+
+  def RemoveChild(self, child_id):
+    '''Removes the first node that has a "name" attribute which
+    matches "child_id" in the list of immediate children of
+    this node.
+
+    Args:
+      child_id: String identifying the child to be removed
+    '''
+    index = 0
+    # Safe not to copy since we only remove the first element found
+    for child in self.children:
+      name_attr = child.attrs['name']
+      if name_attr == child_id:
+        self.children.pop(index)
+        self.mixed_content.pop(index)
+        break
+      index += 1
+
+  def AppendContent(self, content):
+    '''Appends a chunk of text as content of this node.
+
+    Args:
+      content: u'hello'
+
+    Return:
+      None
+    '''
+    assert isinstance(content, types.StringTypes)
+    if self._ContentType() != self._CONTENT_TYPE_NONE:
+      self.mixed_content.append(content)
+    elif content.strip() != '':
+      raise exception.UnexpectedContent()
+
+  def HandleAttribute(self, attrib, value):
+    '''Informs the node of an attribute that was parsed out of the GRD file
+    for it.
+
+    Args:
+      attrib: 'name'
+      value: 'fooblat'
+
+    Return:
+      None
+    '''
+    assert isinstance(attrib, types.StringTypes)
+    assert isinstance(value, types.StringTypes)
+    if self._IsValidAttribute(attrib, value):
+      self.attrs[attrib] = value
+    else:
+      raise exception.UnexpectedAttribute(attrib)
+
+  def EndParsing(self):
+    '''Called at the end of parsing.'''
+
+    # TODO(joi) Rewrite this, it's extremely ugly!
+    if len(self.mixed_content):
+      if isinstance(self.mixed_content[0], types.StringTypes):
+        # Remove leading and trailing chunks of pure whitespace.
+        while (len(self.mixed_content) and
+               isinstance(self.mixed_content[0], types.StringTypes) and
+               self.mixed_content[0].strip() == ''):
+          self.mixed_content = self.mixed_content[1:]
+        # Strip leading and trailing whitespace from mixed content chunks
+        # at front and back.
+        if (len(self.mixed_content) and
+            isinstance(self.mixed_content[0], types.StringTypes)):
+          self.mixed_content[0] = self.mixed_content[0].lstrip()
+        # Remove leading and trailing ''' (used to demarcate whitespace)
+        if (len(self.mixed_content) and
+            isinstance(self.mixed_content[0], types.StringTypes)):
+          if self.mixed_content[0].startswith("'''"):
+            self.mixed_content[0] = self.mixed_content[0][3:]
+    if len(self.mixed_content):
+      if isinstance(self.mixed_content[-1], types.StringTypes):
+        # Same stuff all over again for the tail end.
+        while (len(self.mixed_content) and
+               isinstance(self.mixed_content[-1], types.StringTypes) and
+               self.mixed_content[-1].strip() == ''):
+          self.mixed_content = self.mixed_content[:-1]
+        if (len(self.mixed_content) and
+            isinstance(self.mixed_content[-1], types.StringTypes)):
+          self.mixed_content[-1] = self.mixed_content[-1].rstrip()
+        if (len(self.mixed_content) and
+            isinstance(self.mixed_content[-1], types.StringTypes)):
+          if self.mixed_content[-1].endswith("'''"):
+            self.mixed_content[-1] = self.mixed_content[-1][:-3]
+
+    # Check that all mandatory attributes are there.
+    for node_mandatt in self.MandatoryAttributes():
+      mandatt_list = []
+      if node_mandatt.find('|') >= 0:
+        mandatt_list = node_mandatt.split('|')
+      else:
+        mandatt_list.append(node_mandatt)
+
+      mandatt_option_found = False
+      for mandatt in mandatt_list:
+        assert mandatt not in self.DefaultAttributes().keys()
+        if mandatt in self.attrs:
+          if not mandatt_option_found:
+            mandatt_option_found = True
+          else:
+            raise exception.MutuallyExclusiveMandatoryAttribute(mandatt)
+
+      if not mandatt_option_found:
+        raise exception.MissingMandatoryAttribute(mandatt)
+
+    # Add default attributes if not specified in input file.
+    for defattr in self.DefaultAttributes():
+      if not defattr in self.attrs:
+        self.attrs[defattr] = self.DefaultAttributes()[defattr]
+
+  def GetCdata(self):
+    '''Returns all CDATA of this element, concatenated into a single
+    string.  Note that this ignores any elements embedded in CDATA.'''
+    return ''.join([c for c in self.mixed_content
+                    if isinstance(c, types.StringTypes)])
+
+  def __unicode__(self):
+    '''Returns this node and all nodes below it as an XML document in a Unicode
+    string.'''
+    header = u'<?xml version="1.0" encoding="UTF-8"?>\n'
+    return header + self.FormatXml()
+
+  def FormatXml(self, indent = u'', one_line = False):
+    '''Returns this node and all nodes below it as an XML
+    element in a Unicode string.  This differs from __unicode__ in that it does
+    not include the <?xml> stuff at the top of the string.  If one_line is true,
+    children and CDATA are layed out in a way that preserves internal
+    whitespace.
+    '''
+    assert isinstance(indent, types.StringTypes)
+
+    content_one_line = (one_line or
+                        self._ContentType() == self._CONTENT_TYPE_MIXED)
+    inside_content = self.ContentsAsXml(indent, content_one_line)
+
+    # Then the attributes for this node.
+    attribs = u''
+    default_attribs = self.DefaultAttributes()
+    for attrib, value in sorted(self.attrs.items()):
+      # Only print an attribute if it is other than the default value.
+      if attrib not in default_attribs or value != default_attribs[attrib]:
+        attribs += u' %s=%s' % (attrib, saxutils.quoteattr(value))
+
+    # Finally build the XML for our node and return it
+    if len(inside_content) > 0:
+      if one_line:
+        return u'<%s%s>%s</%s>' % (self.name, attribs, inside_content, self.name)
+      elif content_one_line:
+        return u'%s<%s%s>\n%s  %s\n%s</%s>' % (
+          indent, self.name, attribs,
+          indent, inside_content,
+          indent, self.name)
+      else:
+        return u'%s<%s%s>\n%s\n%s</%s>' % (
+          indent, self.name, attribs,
+          inside_content,
+          indent, self.name)
+    else:
+      return u'%s<%s%s />' % (indent, self.name, attribs)
+
+  def ContentsAsXml(self, indent, one_line):
+    '''Returns the contents of this node (CDATA and child elements) in XML
+    format.  If 'one_line' is true, the content will be laid out on one line.'''
+    assert isinstance(indent, types.StringTypes)
+
+    # Build the contents of the element.
+    inside_parts = []
+    last_item = None
+    for mixed_item in self.mixed_content:
+      if isinstance(mixed_item, Node):
+        inside_parts.append(mixed_item.FormatXml(indent + u'  ', one_line))
+        if not one_line:
+          inside_parts.append(u'\n')
+      else:
+        message = mixed_item
+        # If this is the first item and it starts with whitespace, we add
+        # the ''' delimiter.
+        if not last_item and message.lstrip() != message:
+          message = u"'''" + message
+        inside_parts.append(util.EncodeCdata(message))
+      last_item = mixed_item
+
+    # If there are only child nodes and no cdata, there will be a spurious
+    # trailing \n
+    if len(inside_parts) and inside_parts[-1] == '\n':
+      inside_parts = inside_parts[:-1]
+
+    # If the last item is a string (not a node) and ends with whitespace,
+    # we need to add the ''' delimiter.
+    if (isinstance(last_item, types.StringTypes) and
+        last_item.rstrip() != last_item):
+      inside_parts[-1] = inside_parts[-1] + u"'''"
+
+    return u''.join(inside_parts)
+
+  def SubstituteMessages(self, substituter):
+    '''Applies substitutions to all messages in the tree.
+
+    Called as a final step of RunGatherers.
+
+    Args:
+      substituter: a grit.util.Substituter object.
+    '''
+    for child in self.children:
+      child.SubstituteMessages(substituter)
+
+  def _IsValidChild(self, child):
+    '''Returns true if 'child' is a valid child of this node.
+    Overridden by subclasses.'''
+    return False
+
+  def _IsValidAttribute(self, name, value):
+    '''Returns true if 'name' is the name of a valid attribute of this element
+    and 'value' is a valid value for that attribute.  Overriden by
+    subclasses unless they have only mandatory attributes.'''
+    return (name in self.MandatoryAttributes() or
+            name in self.DefaultAttributes())
+
+  def _ContentType(self):
+    '''Returns the type of content this element can have.  Overridden by
+    subclasses.  The content type can be one of the _CONTENT_TYPE_XXX constants
+    above.'''
+    return self._CONTENT_TYPE_NONE
+
+  def MandatoryAttributes(self):
+    '''Returns a list of attribute names that are mandatory (non-optional)
+    on the current element. One can specify a list of
+    "mutually exclusive mandatory" attributes by specifying them as one
+    element in the list, separated by a "|" character.
+    '''
+    return []
+
+  def DefaultAttributes(self):
+    '''Returns a dictionary of attribute names that have defaults, mapped to
+    the default value.  Overridden by subclasses.'''
+    return {}
+
+  def GetCliques(self):
+    '''Returns all MessageClique objects belonging to this node.  Overridden
+    by subclasses.
+
+    Return:
+      [clique1, clique2] or []
+    '''
+    return []
+
+  def ToRealPath(self, path_from_basedir):
+    '''Returns a real path (which can be absolute or relative to the current
+    working directory), given a path that is relative to the base directory
+    set for the GRIT input file.
+
+    Args:
+      path_from_basedir: '..'
+
+    Return:
+      'resource'
+    '''
+    return util.normpath(os.path.join(self.GetRoot().GetBaseDir(),
+                                      os.path.expandvars(path_from_basedir)))
+
+  def GetInputPath(self):
+    '''Returns a path, relative to the base directory set for the grd file,
+    that points to the file the node refers to.
+    '''
+    # This implementation works for most nodes that have an input file.
+    return self.attrs['file']
+
+  def UberClique(self):
+    '''Returns the uberclique that should be used for messages originating in
+    a given node.  If the node itself has its uberclique set, that is what we
+    use, otherwise we search upwards until we find one.  If we do not find one
+    even at the root node, we set the root node's uberclique to a new
+    uberclique instance.
+    '''
+    node = self
+    while not node.uberclique and node.parent:
+      node = node.parent
+    if not node.uberclique:
+      node.uberclique = clique.UberClique()
+    return node.uberclique
+
+  def IsTranslateable(self):
+    '''Returns false if the node has contents that should not be translated,
+    otherwise returns false (even if the node has no contents).
+    '''
+    if not 'translateable' in self.attrs:
+      return True
+    else:
+      return self.attrs['translateable'] == 'true'
+
+  def GetNodeById(self, id):
+    '''Returns the node in the subtree parented by this node that has a 'name'
+    attribute matching 'id'.  Returns None if no such node is found.
+    '''
+    for node in self:
+      if 'name' in node.attrs and node.attrs['name'] == id:
+        return node
+    return None
+
+  def GetChildrenOfType(self, type):
+    '''Returns a list of all subnodes (recursing to all leaves) of this node
+    that are of the indicated type (or tuple of types).
+
+    Args:
+      type: A type you could use with isinstance().
+
+    Return:
+      A list, possibly empty.
+    '''
+    return [child for child in self if isinstance(child, type)]
+
+  def GetTextualIds(self):
+    '''Returns a list of the textual ids of this node.
+    '''
+    if 'name' in self.attrs:
+      return [self.attrs['name']]
+    return []
+
+  @classmethod
+  def EvaluateExpression(cls, expr, defs, target_platform, extra_variables={}):
+    '''Worker for EvaluateCondition (below) and conditions in XTB files.'''
+    if expr in cls.eval_expr_cache:
+      code, variables_in_expr = cls.eval_expr_cache[expr]
+    else:
+      # Get a list of all variable and method names used in the expression.
+      syntax_tree = ast.parse(expr, mode='eval')
+      variables_in_expr = [node.id for node in ast.walk(syntax_tree) if
+          isinstance(node, ast.Name) and node.id not in ('True', 'False')]
+      code = compile(syntax_tree, filename='<string>', mode='eval')
+      cls.eval_expr_cache[expr] = code, variables_in_expr
+
+    # Set values only for variables that are needed to eval the expression.
+    variable_map = {}
+    for name in variables_in_expr:
+      if name == 'os':
+        value = target_platform
+      elif name == 'defs':
+        value = defs
+
+      elif name == 'is_linux':
+        value = target_platform.startswith('linux')
+      elif name == 'is_macosx':
+        value = target_platform == 'darwin'
+      elif name == 'is_win':
+        value = target_platform in ('cygwin', 'win32')
+      elif name == 'is_android':
+        value = target_platform == 'android'
+      elif name == 'is_ios':
+        value = target_platform == 'ios'
+      elif name == 'is_bsd':
+        value = 'bsd' in target_platform
+      elif name == 'is_posix':
+        value = (target_platform in ('darwin', 'linux2', 'linux3', 'sunos5',
+                                     'android', 'ios')
+                 or 'bsd' in target_platform)
+
+      elif name == 'pp_ifdef':
+        def pp_ifdef(symbol):
+          return symbol in defs
+        value = pp_ifdef
+      elif name == 'pp_if':
+        def pp_if(symbol):
+          return defs.get(symbol, False)
+        value = pp_if
+
+      elif name in defs:
+        value = defs[name]
+      elif name in extra_variables:
+        value = extra_variables[name]
+      else:
+        # Undefined variables default to False.
+        value = False
+
+      variable_map[name] = value
+
+    eval_result = eval(code, {}, variable_map)
+    assert isinstance(eval_result, bool)
+    return eval_result
+
+  def EvaluateCondition(self, expr):
+    '''Returns true if and only if the Python expression 'expr' evaluates
+    to true.
+
+    The expression is given a few local variables:
+      - 'lang' is the language currently being output
+           (the 'lang' attribute of the <output> element).
+      - 'context' is the current output context
+           (the 'context' attribute of the <output> element).
+      - 'defs' is a map of C preprocessor-style symbol names to their values.
+      - 'os' is the current platform (likely 'linux2', 'win32' or 'darwin').
+      - 'pp_ifdef(symbol)' is a shorthand for "symbol in defs".
+      - 'pp_if(symbol)' is a shorthand for "symbol in defs and defs[symbol]".
+      - 'is_linux', 'is_macosx', 'is_win', 'is_posix' are true if 'os'
+           matches the given platform.
+    '''
+    root = self.GetRoot()
+    lang = getattr(root, 'output_language', '')
+    context = getattr(root, 'output_context', '')
+    defs = getattr(root, 'defines', {})
+    target_platform = getattr(root, 'target_platform', '')
+    extra_variables = {
+        'lang': lang,
+        'context': context,
+    }
+    return Node.EvaluateExpression(
+        expr, defs, target_platform, extra_variables)
+
+  def OnlyTheseTranslations(self, languages):
+    '''Turns off loading of translations for languages not in the provided list.
+
+    Attrs:
+      languages: ['fr', 'zh_cn']
+    '''
+    for node in self:
+      if (hasattr(node, 'IsTranslation') and
+          node.IsTranslation() and
+          node.GetLang() not in languages):
+        node.DisableLoading()
+
+  def FindBooleanAttribute(self, attr, default, skip_self):
+    '''Searches all ancestors of the current node for the nearest enclosing
+    definition of the given boolean attribute.
+
+    Args:
+      attr: 'fallback_to_english'
+      default: What to return if no node defines the attribute.
+      skip_self: Don't check the current node, only its parents.
+    '''
+    p = self.parent if skip_self else self
+    while p:
+      value = p.attrs.get(attr, 'default').lower()
+      if value != 'default':
+        return (value == 'true')
+      p = p.parent
+    return default
+
+  def PseudoIsAllowed(self):
+    '''Returns true if this node is allowed to use pseudo-translations.  This
+    is true by default, unless this node is within a <release> node that has
+    the allow_pseudo attribute set to false.
+    '''
+    return self.FindBooleanAttribute('allow_pseudo',
+                                     default=True, skip_self=True)
+
+  def ShouldFallbackToEnglish(self):
+    '''Returns true iff this node should fall back to English when
+    pseudotranslations are disabled and no translation is available for a
+    given message.
+    '''
+    return self.FindBooleanAttribute('fallback_to_english',
+                                     default=False, skip_self=True)
+
+  def WhitelistMarkedAsSkip(self):
+    '''Returns true if the node is marked to be skipped in the output by a
+    whitelist.
+    '''
+    return self._whitelist_marked_as_skip
+
+  def SetWhitelistMarkedAsSkip(self, mark_skipped):
+    '''Sets WhitelistMarkedAsSkip.
+    '''
+    self._whitelist_marked_as_skip = mark_skipped
+
+  def ExpandVariables(self):
+    '''Whether we need to expand variables on a given node.'''
+    return False
+
+  def IsResourceMapSource(self):
+    '''Whether this node is a resource map source.'''
+    return False
+
+  def GeneratesResourceMapEntry(self, output_all_resource_defines,
+                                is_active_descendant):
+    '''Whether this node should output a resource map entry.
+
+    Args:
+      output_all_resource_defines: The value of output_all_resource_defines for
+                                   the root node.
+      is_active_descendant: Whether the current node is an active descendant
+                            from the root node.'''
+    return False
+
+
+class ContentNode(Node):
+  '''Convenience baseclass for nodes that can have content.'''
+  def _ContentType(self):
+    return self._CONTENT_TYPE_MIXED
+