Moving prepopulated search engines to a JSON file.

tools/json_to_struct/element_generator.py

Index: tools/json_to_struct/element_generator.py
diff --git a/tools/json_to_struct/element_generator.py b/tools/json_to_struct/element_generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..a77f75bbeed91affb620d9f162f722818fda3289
--- /dev/null
+++ b/tools/json_to_struct/element_generator.py
@@ -0,0 +1,139 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import struct_generator
+
+def JSONToCString16(json_string_literal):

[not at google - send to devlin, 2012/11/12 18:38:13]

Private functions prefix with _.

[beaudoin, 2012/11/13 18:44:26]

I've also elected to make some other function private. Not sure if I should try
to lock this even more?

Done.

+  """Converts a JSON string literal to a C++ UTF-16 string literal. This is
+  done by converting \\u#### to \\x####.

[not at google - send to devlin, 2012/11/12 18:38:13]

Can you use a regex?

[beaudoin, 2012/11/13 18:44:26]

I thought about it and could not find a way that would guarantee I catch \u but
not \\u. Negative lookbehind does not work as it would fail catching \\\u. I
think this is the best solution.

+  """
+  c_string_literal = json_string_literal
+  i = 0
+  while 1:
+    i = json_string_literal.find('\\', i)
+    if i == -1: break
+    if json_string_literal[i + 1] == 'u':
+      c_string_literal = (json_string_literal[0:i + 1] + 'x' +
+                          json_string_literal[i + 2:])
+    i += 2
+  return c_string_literal
+
+def GenerateInt(field_info, content, lines):

[not at google - send to devlin, 2012/11/12 18:38:13]

likewise, consider generating and returning Code objects, then composing them
from the callers.

[beaudoin, 2012/11/13 18:44:26]

Thought about this too, but didn't want to reorganize the tools/ directory when
what I needed was readily obtained via '\n'.join()

[not at google - send to devlin, 2012/11/13 20:28:07]

Why do you need to reorganize the tools directory? Can't you just include it the
way you include the other tools, by augmenting sys.path?

The Code object really is useful, mostly just because it's composable and leads
the code generators toward a nice structure. And as I mentioned, it's much less
error prone given it does automatic indentation. Plus has nice features like
block-level % substitution.

+  """Generates an int to be included in a static structure initializer. If
+  content is not specified, generates the default int for this field.
+  """
+  if content is None:
+    content = field_info['default']
+  lines.append('  %s,' % content)
+
+def GenerateString(field_info, content, lines):
+  """Generates an UTF-8 string to be included in a static structure initializer.
+  If content is not specified, uses this field's default string or NULL if it
+  doesn't have any default.
+  """
+  try:
+    if content is None:
+      content = field_info['default']
+    # json.dumps quotes the string and escape characters as required.
+    lines.append('  %s,' % json.dumps(content))
+  except KeyError:
+    lines.append('  NULL,')
+
+def GenerateString16(field_info, content, lines):
+  """Generates an UTF-16 string to be included in a static structure
+  initializer. If content is not specified, uses this field's default string or
+  NULL if it doesn't have any default.
+  """
+  try:
+    if content is None:
+      content = field_info['default']
+    # json.dumps quotes the string and escape characters as required.
+    lines.append('  L%s,' % JSONToCString16(json.dumps(content)))
+  except KeyError:
+    lines.append('  NULL,')
+
+def GenerateEnum(field_info, content, lines):
+  """Generates an enum to be included in a static structure initializer. If
+  content is not specified, generates the default enum for this field.
+  """
+  if content is None:
+    content = field_info['default']
+  lines.append('  %s,' % content)
+
+# A counter for the temporary variables containing generated static arrays.
+var_counter = 0

[not at google - send to devlin, 2012/11/12 18:38:13]

If this is necessary, prefer to make the element generator a class which
maintains its own state.

[beaudoin, 2012/11/13 18:44:26]

No longer necessary.

+
+def GenerateArray(field_info, content, lines):
+  """Generates an array to be included in a static structure initializer. If
+  content is not specified, uses NULL. The array is assigned to a temporary
+  variable which is initilized before the structure.
+  """
+  global var_counter
+  if content is None:
+    lines.append('  NULL,')
+  else:
+    # Create a new array variable and use it in the structure initializer.
+    var = 'temp_%s' % var_counter

[not at google - send to devlin, 2012/11/12 18:38:13]

Can you use the name of the fields as a variable name rather than temp values?
It would make the generated code easier to read.

[beaudoin, 2012/11/13 18:44:26]

Excellent idea. It forced me to disallow nested arrays (otherwise you would get
the same variable multiple times) but we don't need them so I think it's OK.

There is still the risk that a duplicate variable name exists, but the mangling
should reduce that.

Done.

+    lines.append('  %s,' % var)
+    var_counter += 1
+    # Generate the array content.
+    array_lines = []
+    field_info['contents']['field'] = var;
+    array_lines.append(struct_generator.GenerateField(
+                       field_info['contents']) + '[] = {')
+    for subcontent in content:
+      GenerateFieldContent(field_info['contents'], subcontent, array_lines)
+    array_lines.append('};')
+    # Prepend the generated array so it is initialized before the structure.
+    lines.reverse()
+    array_lines.reverse()
+    lines.extend(array_lines)
+    lines.reverse()
+
+# Map of supported types to generator functions.
+content_generators = {
+  "int": GenerateInt,
+  "string": GenerateString,
+  "string16": GenerateString16,
+  "enum": GenerateEnum,
+  "array": GenerateArray,
+}
+
+def GenerateFieldContent(field_info, content, lines):
+  """Generate the content of a field to be included in the static structure
+  initializer.
+  """
+  return content_generators[field_info['type']](field_info, content, lines)
+
+def GenerateElement(type_name, schema, element_name, element):
+  """Generate the static structure initializer for one element.
+  """
+  lines = [];
+  lines.append('const %s %s = {' % (type_name, element_name));
+  for field_info in schema:
+    try:
+      content = element[field_info['field']]

[not at google - send to devlin, 2012/11/12 18:38:13]

content = elemtent.get(field_info['field'], None) rather than the KeyError stuff

[beaudoin, 2012/11/13 18:44:26]

Done.

+    except KeyError:
+      content = None
+    GenerateFieldContent(field_info, content, lines)
+  lines.append('};')
+  return '\n'.join(lines)
+
+def GenerateElements(type_name, schema, description):
+  """Generate the static structure initializer for all the elements in the
+  description['elements'] dictionary, as well as for any variables in
+  description['int_variables'].
+  """
+  result = [];
+  try:
+    for var_name, value in description['int_variables'].items():
+      result.append('const int %s = %s;' % (var_name, value))
+    result.append('')
+  except KeyError:
+    pass
+  for element_name, element in description['elements'].items():
+    result.append(GenerateElement(type_name, schema, element_name, element))
+    result.append('')
+  return '\n'.join(result)