Integrate user strings into the JSON policy template file

tools/grit/grit/gather/policy_json.py

+#!/usr/bin/python2.4
+# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+'''Support for "policy_templates.json" format used by the policy template
+generator as a source for generating ADM,ADMX,etc files.'''
+
+import types
+import pprint
+import re
+
+from grit.gather import regexp
+from grit import util
+from grit import tclib
+from xml.dom import minidom
+
+
+class PolicyJson(regexp.RegexpGatherer):
+  '''Collects and translates the following strings from policy_templates.json:
+    - captions,descriptions and labels of policies
+    - captions of enumeration items
+    - misc strings from the 'messages' section
+     Translatable strings may have untranslateable placeholders with the same
+     format that is used in .grd files.
+  '''
+
+  def __init__(self, text):
+    if util.IsExtraVerbose():
+      print text
+    regexp.RegexpGatherer.__init__(self, text)
+
+  def _ParsePlaceholder(self, placeholder, msg):
+    '''Extracts a placeholder from a DOM node and adds it to a tclib Message.
+
+    Args:
+      placeholder: A DOM node of the form:
+        <ph name="PLACEHOLDER_NAME">Placeholder text<ex>Example value</ex></ph>
+      msg: The placeholder is added to this message.
+    '''
+    text = []
+    example_text = []
+    for node1 in placeholder.childNodes:
+      if (node1.nodeType == minidom.Node.TEXT_NODE):
+        text.append(node1.data)
+      elif (node1.nodeType == minidom.Node.ELEMENT_NODE and
+            node1.tagName == 'ex'):
+        for node2 in node1.childNodes:
+          example_text.append(node2.toxml())
+      else:
+         raise Exception('Unexpected element inside a placeholder: ' +
+                         node2.toxml())
+    if example_text == []:
+      # In such cases the original text is okay for an example.
+      example_text = text
+    msg.AppendPlaceholder(tclib.Placeholder(
+        placeholder.attributes['name'].value,
+        ''.join(text).strip(),
+        ''.join(example_text).strip()))
+
+  def _ParseMessage(self, string, desc):
+    '''Parses a given string and adds it to the output as a translatable chunk
+    with a given description.
+
+    Args:
+      string: The message string to parse.
+      desc: The description of the message (for the translators).
+    '''
+    msg = tclib.Message(description=desc)
+    xml = '<msg>' + string + '</msg>'
+    node = minidom.parseString(xml).childNodes[0]
+    for child in node.childNodes:
+      if child.nodeType == minidom.Node.TEXT_NODE:
+        msg.AppendText(child.data)
+      elif child.nodeType == minidom.Node.ELEMENT_NODE:
+        if child.tagName == 'ph':
+          self._ParsePlaceholder(child, msg)
+        else:
+          raise Exception("Not implemented.")
+      else:
+        raise Exception("Not implemented.")
+    self.skeleton_.append(self.uberclique.MakeClique(msg))

[Jói, 2011/01/10 21:16:50]

Don't you get into problems with JSON escaping when you output the file? Seems
you would at least need to override the Escape method from RegexpGatherer.

I don't mind which approach you take to this, but did you think about using an
existing JSON writer implementation and not using the
translatable/non-translatable skeleton idiom that RegexpGatherer uses (instead
you would just keep a list of messages and other metadata and then output it as
a JSON file)?

Do you use the .json file that calls to the Translate function will generate at
all? Perhaps this is the case since you have all those other writer classes in
the policy_templates folder that get invoked as output formats via <output>
nodes, but if you don't use it, then maybe you should provide an overridden
Translate method that just returns the empty string, to save some time, and
again if you don't use the output .json, there is no need for you to keep all
those non-translatable chunks.

[gfeher, 2011/01/11 20:09:11]

The output seems fine. I've diffed all the outputted templates against their
versions generated on trunk. However, I will override Escape once we had agreed
on the general approach.
If you mean policy_templates/json_writer.py, then that is a different thing. In
short, it outputs a very-very stripped down version of policy_templates.json.
Otherwise please clarify.
Yes, I do use the output of Translate. It goes through template_formatter.py and
goes to the writers. So it is not the structure node that writes the output
files, but the output nodes. I wanted to utilize the structure node to create
these files (as you had recommended earlier), but it seemed like that there is
no such possibility in open source GRIT.

[Jói, 2011/01/11 20:24:57]

OK.

[gfeher, 2011/01/12 12:18:33]

I probably won't need to override Escape.

+
+  def _ParseNode(self, node):
+    '''Traverses the subtree of a DOM node, and register a tclib message for
+    all the <message> nodes.
+    '''
+    att_text = []
+    if node.attributes:
+      items = node.attributes.items()
+      items.sort()
+      for key, value in items:
+        att_text.append(' %s=\"%s\"' % (key, value))
+    self._AddNontranslateableChunk("<%s%s>" %
+                                   (node.tagName, ''.join(att_text)))
+    if node.tagName == 'message':
+      msg = tclib.Message(description=node.attributes['desc'])
+      for child in node.childNodes:
+        if child.nodeType == minidom.Node.TEXT_NODE:
+          if msg == None:
+            self._AddNontranslateableChunk(child.data)
+          else:
+            msg.AppendText(child.data)
+        elif child.nodeType == minidom.Node.ELEMENT_NODE:
+          if child.tagName == 'ph':
+            self._ParsePlaceholder(child, msg)
+        else:
+          assert False
+      self.skeleton_.append(self.uberclique.MakeClique(msg))
+    else:
+      for child in node.childNodes:
+        if child.nodeType == minidom.Node.TEXT_NODE:
+          self._AddNontranslateableChunk(child.data)
+        elif node.nodeType == minidom.Node.ELEMENT_NODE:
+          self._ParseNode(child)
+
+    self._AddNontranslateableChunk("</%s>" % node.tagName)
+
+  def _AddIndentedNontranslateableChunk(self, depth, string):
+    '''Adds a nontranslateable chunk of text to the internally stored output.
+
+    Args:
+      depth: The number of double spaces to prepend to the next argument string.
+      string: The chunk of text to add.
+    '''
+    result = []
+    while depth > 0:
+      result.append('  ')
+      depth = depth - 1
+    result.append(string)
+    self._AddNontranslateableChunk(''.join(result))
+
+  def _GetDescription(self, item, item_type, parent_item, key):
+    '''Creates a description for a translatable message. The description gives
+    some context for the person who will translate this message.
+
+    Args:
+      item: A policy or an enumeration item.
+      item_type: 'enum_item' | 'policy'
+      parent_item: The owner of item. (A policy of type group or enum.)
+      key: The name of the key to parse.
+      depth: The level of indentation.
+    '''
+    key_map = {
+      'desc': 'Description',
+      'caption': 'Caption',
+      'label': 'Label',
+    }
+    if type == 'policy':
+      return '%s of the policy named %s' % (key_map[key], item['name'])
+    elif type == 'enum_item':
+      return ('%s of the option named %s in policy %s' %
+              (key_map[key], item['name'], parent_item['name']))
+
+  def _AddPolicyKey(self, item, item_type, parent_item, key, depth):
+    '''Given a policy/enumeration item and a key, adds that key and its value
+    into the output.
+    E.g.:
+       'example_value': 123
+    If key indicates that the value is a translatable string, then it is parsed
+    as a translatable string.
+
+    Args:
+      item: A policy or an enumeration item.
+      item_type: 'enum_item' | 'policy'
+      parent_item: The owner of item. (A policy of type group or enum.)
+      key: The name of the key to parse.
+      depth: The level of indentation.
+    '''
+    self._AddIndentedNontranslateableChunk(depth, "'%s': " % key)
+    if key in ('desc', 'caption', 'label'):
+      self._AddNontranslateableChunk("'''")
+      self._ParseMessage(
+          item[key],
+          self._GetDescription(item, item_type, parent_item, key))
+      self._AddNontranslateableChunk("''',\n")
+    else:
+      str_val = item[key]
+      if type(str_val) == types.StringType:
+        str_val = "'%s'" % str_val
+      else:
+        str_val = str(str_val)
+      self._AddNontranslateableChunk(str_val + ',\n')
+
+  def _AddItems(self, items, item_type, parent_item, depth):
+    '''Parses and adds a list of items from the JSON file. Items can be policies
+    or parts of an enum policy.
+
+    Args:
+      items: Either a list of policies or a list of dictionaries.
+      item_type: 'enum_item' | 'policy'
+      parent_item: If items contains a list of policies, then this is the policy
+        group that owns them. If items contains a list of enumeration items,
+        then this is the enum policy that holds them.
+      depth: Indicates the depth of our position in the JSON hierarchy. Used to
+        add nice line-indent to the output.
+    '''
+    for item1 in items:
+      self._AddIndentedNontranslateableChunk(depth, "{\n")
+      for key in item1.keys():
+        if key == 'items':
+          self._AddIndentedNontranslateableChunk(depth + 1, "'items': [\n")
+          self._AddItems(item1['items'], 'enum_item', item1, depth + 2)
+          self._AddIndentedNontranslateableChunk(depth + 1, "],\n")
+        if key == 'policies':
+          self._AddIndentedNontranslateableChunk(depth + 1, "'policies': [\n")
+          self._AddItems(item1['policies'], 'policy', item1, depth + 2)
+          self._AddIndentedNontranslateableChunk(depth + 1, "],\n")
+        else:
+          self._AddPolicyKey(item1, item_type, parent_item, key, depth + 1)
+      self._AddIndentedNontranslateableChunk(depth, "},\n")
+
+  def _AddMessages(self):
+    '''Processed and adds the 'messages' section to the output.'''
+    self._AddNontranslateableChunk("  'messages': {\n")
+    for name, message in self.data['messages'].iteritems():
+        self._AddNontranslateableChunk("      '%s': {\n" % name)

[Jói, 2011/01/10 21:16:50]

use two space indent

[gfeher, 2011/01/11 20:09:11]

Done.

+        self._AddNontranslateableChunk("        'text': '''")
+        self._ParseMessage(message['text'], message['desc'])
+        self._AddNontranslateableChunk("'''\n")
+        self._AddNontranslateableChunk("      },\n")
+    self._AddNontranslateableChunk("  },\n")
+
+  def _AddPlaceholders(self):
+    '''Adds the 'placeholders' section to the output.'''
+    self._AddNontranslateableChunk("  'placeholders':\n")
+    self._AddNontranslateableChunk(
+        pprint.pformat(self.data['placeholders'], indent=2, depth=2))
+
+  # Although we use the RegexpGatherer base class, we do not use the

[Jói, 2011/01/10 21:16:50]

This is confusing, since the inheritance implies an is-a relationship.

It would probably be better to split out a common base class from RegexpGatherer
(and have RegexpGatherer and your class inherit from it), named
SkeletonGatherer, that leaves the Parse() method unimplemented but takes the
rest of the stuff that's not really regular expression stuff and moves it to the
base.

[gfeher, 2011/01/11 20:09:11]

I agree.
Would it also make sense to reimplement that stuff here? (Please see my e-mail.)

[Jói, 2011/01/11 20:24:57]

I would recommend extracting the base class as discussed.

[gfeher, 2011/01/12 12:18:33]

Done.

+  # _RegExpParse method of that class to implement Parse().  Instead, we
+  # parse using a DOM parser.
+  def Parse(self):
+    if (self.have_parsed_):

[Jói, 2011/01/10 21:16:50]

looks like you are forgetting to set self.have_parsed_ below

[gfeher, 2011/01/11 20:09:11]

Done. This was copied from muppet_strings.py, should I fix that as well?

[Jói, 2011/01/11 20:24:57]

Yes please.

[gfeher, 2011/01/12 12:18:33]

Done.

+      return
+
+    self.data = eval(self.text_)
+
+    self._AddNontranslateableChunk('{\n')
+    self._AddNontranslateableChunk("  'policy_definitions': [\n")
+    self._AddItems(self.data['policy_definitions'], 'policy', None, 2)
+    self._AddNontranslateableChunk("  ],\n")
+    self._AddMessages()
+    self._AddPlaceholders()
+    self._AddNontranslateableChunk('\n}')
+
+
+  def FromFile(filename_or_stream, extkey=None, encoding='cp1252'):
+    if isinstance(filename_or_stream, types.StringTypes):
+      if util.IsVerbose():
+        print "PolicyJson reading file %s, encoding %s" % (
+          filename_or_stream, encoding)
+      filename_or_stream = \
+          util.WrapInputStream(file(filename_or_stream, 'r'), encoding)
+    return PolicyJson(filename_or_stream.read())
+  FromFile = staticmethod(FromFile)