C++->Java string constants auto-generator

build/android/gyp/java_cpp_string.py

 #!/usr/bin/env python
 #
-# Copyright 2014 The Chromium Authors. All rights reserved.
+# Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 from datetime import date
 import re
 import optparse
 import os
 from string import Template
 import sys
 import zipfile
 
 from util import build_utils
 
-# List of C++ types that are compatible with the Java code generated by this
-# script.
-#
-# This script can parse .idl files however, at present it ignores special
-# rules such as [cpp_enum_prefix_override="ax_attr"].
-ENUM_FIXED_TYPE_WHITELIST = ['char', 'unsigned char',
-  'short', 'unsigned short',
-  'int', 'int8_t', 'int16_t', 'int32_t', 'uint8_t', 'uint16_t']
-
-class EnumDefinition(object):
-  def __init__(self, original_enum_name=None, class_name_override=None,
-               enum_package=None, entries=None, fixed_type=None):
-    self.original_enum_name = original_enum_name
-    self.class_name_override = class_name_override
-    self.enum_package = enum_package
+class StringDefinition(object):

[raymes, 2016/02/16 00:58:21]

I guess this is actually a list of strings. So perhaps we should call it
StringListDefinition?

[lshang, 2016/02/16 23:00:19]

Done.

+  def __init__(self, class_name=None, string_package=None, entries=None):
+    self.class_name = class_name
+    self.string_package = string_package
     self.entries = collections.OrderedDict(entries or [])
-    self.prefix_to_strip = None
-    self.fixed_type = fixed_type
 
   def AppendEntry(self, key, value):
     if key in self.entries:
       raise Exception('Multiple definitions of key %s found.' % key)
     self.entries[key] = value
 
-  @property
-  def class_name(self):
-    return self.class_name_override or self.original_enum_name
-
   def Finalize(self):
     self._Validate()
-    self._AssignEntryIndices()
-    self._StripPrefix()
 
   def _Validate(self):
     assert self.class_name
-    assert self.enum_package
+    assert self.string_package
     assert self.entries
-    if self.fixed_type and self.fixed_type not in ENUM_FIXED_TYPE_WHITELIST:
-      raise Exception('Fixed type %s for enum %s not whitelisted.' %
-          (self.fixed_type, self.class_name))
-
-  def _AssignEntryIndices(self):
-    # Enums, if given no value, are given the value of the previous enum + 1.
-    if not all(self.entries.values()):
-      prev_enum_value = -1
-      for key, value in self.entries.iteritems():
-        if not value:
-          self.entries[key] = prev_enum_value + 1
-        elif value in self.entries:
-          self.entries[key] = self.entries[value]
-        else:
-          try:
-            self.entries[key] = int(value)
-          except ValueError:
-            raise Exception('Could not interpret integer from enum value "%s" '
-                            'for key %s.' % (value, key))
-        prev_enum_value = self.entries[key]
 
 
-  def _StripPrefix(self):
-    prefix_to_strip = self.prefix_to_strip
-    if not prefix_to_strip:
-      prefix_to_strip = self.original_enum_name
-      prefix_to_strip = re.sub('(?!^)([A-Z]+)', r'_\1', prefix_to_strip).upper()
-      prefix_to_strip += '_'
-      if not all([w.startswith(prefix_to_strip) for w in self.entries.keys()]):
-        prefix_to_strip = ''
+class DirectiveSet(object):
+  class_name_key = 'CLASS_NAME'
+  string_package_key = 'STRING_PACKAGE'
 
-    entries = collections.OrderedDict()
-    for (k, v) in self.entries.iteritems():
-      stripped_key = k.replace(prefix_to_strip, '', 1)
-      if isinstance(v, basestring):
-        stripped_value = v.replace(prefix_to_strip, '', 1)
-      else:
-        stripped_value = v
-      entries[stripped_key] = stripped_value
-
-    self.entries = entries
-
-class DirectiveSet(object):
-  class_name_override_key = 'CLASS_NAME_OVERRIDE'
-  enum_package_key = 'ENUM_PACKAGE'
-  prefix_to_strip_key = 'PREFIX_TO_STRIP'
-
-  known_keys = [class_name_override_key, enum_package_key, prefix_to_strip_key]
+  known_keys = [class_name_key, string_package_key]
 
   def __init__(self):
     self._directives = {}
 
   def Update(self, key, value):
     if key not in DirectiveSet.known_keys:
       raise Exception("Unknown directive: " + key)
     self._directives[key] = value
 
   @property
   def empty(self):
     return len(self._directives) == 0
 
   def UpdateDefinition(self, definition):
-    definition.class_name_override = self._directives.get(
-        DirectiveSet.class_name_override_key, '')
-    definition.enum_package = self._directives.get(
-        DirectiveSet.enum_package_key)
-    definition.prefix_to_strip = self._directives.get(
-        DirectiveSet.prefix_to_strip_key)
+    definition.class_name = self._directives.get(
+        DirectiveSet.class_name_key)
+    definition.string_package = self._directives.get(
+        DirectiveSet.string_package_key)
 
 
 class HeaderParser(object):

[raymes, 2016/02/16 00:58:20]

This is no longer parsing a header file, so perhaps we should name it
differently

[lshang, 2016/02/16 23:00:20]

Done.

   single_line_comment_re = re.compile(r'\s*//')
   multi_line_comment_start_re = re.compile(r'\s*/\*')
-  enum_line_re = re.compile(r'^\s*(\w+)(\s*\=\s*([^,\n]+))?,?')
-  enum_end_re = re.compile(r'^\s*}\s*;\.*$')
+  multi_line_comment_end_re = re.compile(r'\s*\*/')
+  single_string_line_re = re.compile(
+      r'^\s*const\s*char\s*(\w+)\[\]\s*\=\s*([^,\n]+)?"([^"]*)";')

[raymes, 2016/02/16 00:58:21]

We may not need ([^,\n]+)?

[lshang, 2016/02/16 23:00:19]

Done.

+  multi_string_line_start_re = re.compile(
+      r'^\s*const\s*char\s*(\w+)\[\]\s*\=?\s*$')
+  multi_string_line_end_re = re.compile(
+      r'^\s*\=?\s*"([^"]*)";')
   generator_directive_re = re.compile(
       r'^\s*//\s+GENERATED_JAVA_(\w+)\s*:\s*([\.\w]+)$')
   multi_line_generator_directive_start_re = re.compile(
       r'^\s*//\s+GENERATED_JAVA_(\w+)\s*:\s*\(([\.\w]*)$')
   multi_line_directive_continuation_re = re.compile(
       r'^\s*//\s+([\.\w]+)$')
   multi_line_directive_end_re = re.compile(
       r'^\s*//\s+([\.\w]*)\)$')
 
-  optional_class_or_struct_re = r'(class|struct)?'
-  enum_name_re = r'(\w+)'
-  optional_fixed_type_re = r'(\:\s*(\w+\s*\w+?))?'
-  enum_start_re = re.compile(r'^\s*(?:\[cpp.*\])?\s*enum\s+' +
-      optional_class_or_struct_re + '\s*' + enum_name_re + '\s*' +
-      optional_fixed_type_re + '\s*{\s*$')
 
   def __init__(self, lines, path=None):
     self._lines = lines
     self._path = path
-    self._enum_definitions = []
-    self._in_enum = False
+    self._string_definitions = []
     self._current_definition = None
     self._generator_directives = DirectiveSet()
     self._multi_line_generator_directive = None
+    self._current_key = None

[raymes, 2016/02/16 00:58:21]

maybe call this:
_current_multiline_string_key

[lshang, 2016/02/16 23:00:20]

Done.

+    self._in_comment = False
 
   def _ApplyGeneratorDirectives(self):
     self._generator_directives.UpdateDefinition(self._current_definition)
     self._generator_directives = DirectiveSet()
 
   def ParseDefinitions(self):
+    self._current_definition = StringDefinition()
     for line in self._lines:
       self._ParseLine(line)
-    return self._enum_definitions
+    self._ApplyGeneratorDirectives()
+    self._current_definition.Finalize()
+    return self._current_definition
 
   def _ParseLine(self, line):
     if self._multi_line_generator_directive:
       self._ParseMultiLineDirectiveLine(line)
-    elif not self._in_enum:
-      self._ParseRegularLine(line)
+    elif self._in_comment:
+      self._ParseCommentLine(line)
     else:
-      self._ParseEnumLine(line)
+      self._ParseStringLine(line)
 
-  def _ParseEnumLine(self, line):
+  def _ParseStringLine(self, line):
+    generator_directive = HeaderParser.generator_directive_re.match(line)
+    multi_line_generator_directive_start = (
+        HeaderParser.multi_line_generator_directive_start_re.match(line))
+    if generator_directive:
+      directive_name = generator_directive.groups()[0]
+      directive_value = generator_directive.groups()[1]
+      self._generator_directives.Update(directive_name, directive_value)
+      return
+    elif multi_line_generator_directive_start:
+      directive_name = multi_line_generator_directive_start.groups()[0]
+      directive_value = multi_line_generator_directive_start.groups()[1]
+      self._multi_line_generator_directive = (directive_name, [directive_value])
+      return
+
     if HeaderParser.single_line_comment_re.match(line):
       return
     if HeaderParser.multi_line_comment_start_re.match(line):
-      raise Exception('Multi-line comments in enums are not supported in ' +
-                      self._path)
-    enum_end = HeaderParser.enum_end_re.match(line)
-    enum_entry = HeaderParser.enum_line_re.match(line)
-    if enum_end:
-      self._ApplyGeneratorDirectives()
-      self._current_definition.Finalize()
-      self._enum_definitions.append(self._current_definition)
-      self._in_enum = False
-    elif enum_entry:
-      enum_key = enum_entry.groups()[0]
-      enum_value = enum_entry.groups()[2]
-      self._current_definition.AppendEntry(enum_key, enum_value)
+      self._in_comment = True

[raymes, 2016/02/16 00:58:21]

Could we just ignore multi line comments, as in the enum version?

[lshang, 2016/02/16 23:00:19]

Done.

+      return
+
+    string_multi_line_start = (
+        HeaderParser.multi_string_line_start_re.match(line))
+    single_string_line = HeaderParser.single_string_line_re.match(line)
+    if string_multi_line_start or single_string_line:
+      if self._generator_directives.empty:
+        return
+    if string_multi_line_start:
+      self._current_key = string_multi_line_start.groups()[0]
+      return
+    elif single_string_line:
+      string_key = single_string_line.groups()[0]
+      string_value = single_string_line.groups()[2]
+      self._current_definition.AppendEntry(string_key, string_value)
+      return
+
+    string_multi_line_end = HeaderParser.multi_string_line_end_re.match(line)
+    if string_multi_line_end:
+      string_key = self._current_key

[raymes, 2016/02/16 00:58:21]

Maybe a slightly different approach would be safer:

if self._current_key:
  string_multi_line_end = HeaderParser.multi_string_line_end_re.match(line)
  if (!string_multi_line_end)
    raise Exception('...')
  ...

You would need to move that higher up in the function though

[lshang, 2016/02/16 23:00:20]

Done.

+      string_value = string_multi_line_end.groups()[0]
+      self._current_definition.AppendEntry(string_key, string_value)
+      return
+
 
   def _ParseMultiLineDirectiveLine(self, line):
     multi_line_directive_continuation = (
         HeaderParser.multi_line_directive_continuation_re.match(line))
     multi_line_directive_end = (
         HeaderParser.multi_line_directive_end_re.match(line))
 
     if multi_line_directive_continuation:
       value_cont = multi_line_directive_continuation.groups()[0]
       self._multi_line_generator_directive[1].append(value_cont)
     elif multi_line_directive_end:
       directive_name = self._multi_line_generator_directive[0]
       directive_value = "".join(self._multi_line_generator_directive[1])
       directive_value += multi_line_directive_end.groups()[0]
       self._multi_line_generator_directive = None
       self._generator_directives.Update(directive_name, directive_value)
     else:
       raise Exception('Malformed multi-line directive declaration in ' +
                       self._path)
 
-  def _ParseRegularLine(self, line):
-    enum_start = HeaderParser.enum_start_re.match(line)
-    generator_directive = HeaderParser.generator_directive_re.match(line)
-    multi_line_generator_directive_start = (
-        HeaderParser.multi_line_generator_directive_start_re.match(line))
+  def _ParseCommentLine(self, line):
+    multi_line_comment_end = HeaderParser.multi_line_comment_end_re.match(line)
 
-    if generator_directive:
-      directive_name = generator_directive.groups()[0]
-      directive_value = generator_directive.groups()[1]
-      self._generator_directives.Update(directive_name, directive_value)
-    elif multi_line_generator_directive_start:
-      directive_name = multi_line_generator_directive_start.groups()[0]
-      directive_value = multi_line_generator_directive_start.groups()[1]
-      self._multi_line_generator_directive = (directive_name, [directive_value])
-    elif enum_start:
-      if self._generator_directives.empty:
-        return
-      self._current_definition = EnumDefinition(
-          original_enum_name=enum_start.groups()[1],
-          fixed_type=enum_start.groups()[3])
-      self._in_enum = True
+    if multi_line_comment_end:
+      self._in_comment = False
+    return
+
 
 def GetScriptName():
-  return os.path.basename(os.path.abspath(sys.argv[0]))
+  script_components = os.path.abspath(sys.argv[0]).split(os.path.sep)
+  build_index = script_components.index('build')
+  return os.sep.join(script_components[build_index:])

[raymes, 2016/02/16 00:58:21]

Hmm I'm not sure I understand why this changed?

[lshang, 2016/02/16 23:00:19]

Done.
java_cpp_enum.py changed this on 2016-01-12, but I referred to the old version
of it. Should keep it up-to-date.

 
 def DoGenerate(source_paths):
   for source_path in source_paths:
-    enum_definitions = DoParseHeaderFile(source_path)
-    if not enum_definitions:
-      raise Exception('No enums found in %s\n'
-                      'Did you forget prefixing enums with '
-                      '"// GENERATED_JAVA_ENUM_PACKAGE: foo"?' %
+    string_definition = DoParseHeaderFile(source_path)
+    if not string_definition:
+      raise Exception('No strings found in %s\n'
+                      'Did you forget prefixing strings with '
+                      '"// GENERATED_JAVA_STRING_PACKAGE: foo"?' %
                       source_path)
-    for enum_definition in enum_definitions:
-      package_path = enum_definition.enum_package.replace('.', os.path.sep)
-      file_name = enum_definition.class_name + '.java'
-      output_path = os.path.join(package_path, file_name)
-      output = GenerateOutput(source_path, enum_definition)
-      yield output_path, output
-
+    #for string_definition in string_definitions:
+    package_path = string_definition.string_package.replace('.', os.path.sep)
+    file_name = string_definition.class_name + '.java'
+    output_path = os.path.join(package_path, file_name)
+    output = GenerateOutput(source_path, string_definition)
+    yield output_path, output
 
 def DoParseHeaderFile(path):
   with open(path) as f:
     return HeaderParser(f.readlines(), path).ParseDefinitions()
 
 
-def GenerateOutput(source_path, enum_definition):
+def GenerateOutput(source_path, string_definition):
   template = Template("""
 // Copyright ${YEAR} The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // This file is autogenerated by
 //     ${SCRIPT_NAME}
 // From
 //     ${SOURCE_PATH}
 
 package ${PACKAGE};
 
 public class ${CLASS_NAME} {
-${ENUM_ENTRIES}
+${STRING_ENTRIES}
 }
 """)
 
-  enum_template = Template('  public static final int ${NAME} = ${VALUE};')
-  enum_entries_string = []
-  for enum_name, enum_value in enum_definition.entries.iteritems():
+  string_template = (
+      Template('  public static final String ${NAME} = "${VALUE}";'))
+  string_entries_string = []
+  for string_name, string_value in string_definition.entries.iteritems():
     values = {
-        'NAME': enum_name,
-        'VALUE': enum_value,
+        'NAME': string_name,
+        'VALUE': string_value,
     }
-    enum_entries_string.append(enum_template.substitute(values))
-  enum_entries_string = '\n'.join(enum_entries_string)
+    string_entries_string.append(string_template.substitute(values))
+  string_entries_string = '\n'.join(string_entries_string)
 
   values = {
-      'CLASS_NAME': enum_definition.class_name,
-      'ENUM_ENTRIES': enum_entries_string,
-      'PACKAGE': enum_definition.enum_package,
+      'CLASS_NAME': string_definition.class_name,
+      'STRING_ENTRIES': string_entries_string,
+      'PACKAGE': string_definition.string_package,
       'SCRIPT_NAME': GetScriptName(),
       'SOURCE_PATH': source_path,
       'YEAR': str(date.today().year)
   }
   return template.substitute(values)
 
 
 def AssertFilesList(output_paths, assert_files_list):
   actual = set(output_paths)
   expected = set(assert_files_list)
@@ skipping 60 matching lines @@
     if options.assert_files_list:
       AssertFilesList(output_paths, options.assert_files_list)
 
     if options.verbose:
       print 'Output paths:'
       print '\n'.join(output_paths)
 
     # Used by GYP.
     return ' '.join(output_paths)
 
-
 if __name__ == '__main__':
   DoMain(sys.argv[1:])