Initial commit of the Dart Chrome Extension APIs generators

tools/json_schema_compiler/compiler.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Generator for C++ structs from api json files.
 
 The purpose of this tool is to remove the need for hand-written code that
 converts to and from base::Value types when receiving javascript api calls.
 Originally written for generating code for extension apis. Reference schemas
 are in chrome/common/extensions/api.
 
 Usage example:
   compiler.py --root /home/Work/src --namespace extensions windows.json
     tabs.json
   compiler.py --destdir gen --root /home/Work/src
     --namespace extensions windows.json tabs.json
 """
 
 import cc_generator
 import cpp_type_generator
 import h_generator
+import dart_generator
 import idl_schema
 import json_schema
 import model
 import schema_bundle_generator
 
 import optparse
 import os.path
 import sys
 
+# list of supported languages to build; first is default
+SUPPORTED_LANGS = ['c++', 'dart']
+
 def load_schema(schema):
   schema_filename, schema_extension = os.path.splitext(schema)
 
   if schema_extension == '.json':
     api_defs = json_schema.Load(schema)
   elif schema_extension == '.idl':
     api_defs = idl_schema.Load(schema)
   else:
-    sys.exit("Did not recognize file extension %s for schema %s" %
+    sys.exit('Did not recognize file extension %s for schema %s' %
              (schema_extension, schema))
   if len(api_defs) != 1:
-    sys.exit("File %s has multiple schemas. Files are only allowed to contain a"
-             " single schema." % schema)
+    sys.exit('File %s has multiple schemas. Files are only allowed to contain a'
+             ' single schema.' % schema)
 
   return api_defs
 
-def handle_single_schema(filename, dest_dir, root, root_namespace):
+def handle_single_schema(filename, dest_dir, root, root_namespace, lang,
+    hooks_file):
   schema = os.path.normpath(filename)
   schema_filename, schema_extension = os.path.splitext(schema)
   path, short_filename = os.path.split(schema_filename)
-  api_defs = json_schema.DeleteNocompileNodes(load_schema(schema))
+  api_defs = json_schema.DeleteNocompileNodes(load_schema(schema), lang)
 
   api_model = model.Model()
 
   for target_namespace in api_defs:
     referenced_schemas = target_namespace.get('dependencies', [])
     # Load type dependencies into the model.
     # TODO(miket): do we need this in IDL?
     for referenced_schema in referenced_schemas:
       split_schema = referenced_schema.split(':', 1)
       if len(split_schema) > 1:
@@ skipping 11 matching lines @@
             os.path.relpath(referenced_schema_path, opts.root))
 
     # Gets the relative path from opts.root to the schema to correctly determine
     # the include path.
     relpath = os.path.relpath(schema, opts.root)
     namespace = api_model.AddNamespace(target_namespace, relpath)
     if not namespace:
       continue
 
     if short_filename != namespace.unix_name:
-      sys.exit("Filename %s is illegal. Name files using unix_hacker style." %
+      sys.exit('Filename %s is illegal. Name files using unix_hacker style.' %
                filename)
 
     # The output filename must match the input filename for gyp to deal with it
     # properly.
     out_file = namespace.unix_name
-    type_generator = cpp_type_generator.CppTypeGenerator(
-        root_namespace, namespace, namespace.unix_name)
-    for referenced_namespace in api_model.namespaces.values():
-      if referenced_namespace == namespace:
-        continue
-      type_generator.AddNamespace(
-          referenced_namespace,
-          referenced_namespace.unix_name)
 
-    h_code = (h_generator.HGenerator(namespace, type_generator)
-        .Generate().Render())
-    cc_code = (cc_generator.CCGenerator(namespace, type_generator)
-        .Generate().Render())
+    if lang.lower() == 'c++':

[not at google - send to devlin, 2013/01/25 18:14:33]

Unfortunately, now compiler.py has three personalities - there is a branch up in
the stack on bundle vs not-bundle mode, and a branch lower in the stack about
c++ vs dart mode.

However, really there are just 3 generators. A Generator takes a namespace (plus
dependencies) and returns Code.

Well actually, really there are 4 generators, it just happens that c++ implies
both h_generator and cc_generator.

Anyhow, I think this all can be factored much more cleanly, and now is as good a
time as any to do that.

Let's formalise what a Generator is in code: it has a Generate method which
takes a namespace and returns a Code object. I'd also go and do a bit of
renaming so that this makes more sense, like
 - SchemaBundleGenerator -> CppBundleGenerator, rather than have
GenerateAPIHeaders etc methods have an api_h_generator property
 - Add a CppBundleGenerator with similar property h_generator and cc_generator

This file can then look something like

if opts.bundle:
  cpp_bundle_generator = CppBundleGenerator(...)
  generators = [
    ('generated_api.h': cpp_bundle_generator.api_h_generator),
    ('generated_schemas.cc': cpp_bundle_generator.schemas_cc_generator),
    ('generated_schemas.h': cpp_bundle_generator.schemas_h_generator)
  ]
elif opts.lang == 'c++':
  cpp_generator = CppGenerator(...)
  generators = [
    ('%s.h' % namespace.name), cpp_generator.h_generator),
    ('%s.cc' % namespace.name), cpp_generator.cc_generator)
  ]
elif opts.lang == 'dart':
  generators = [
    ('%s.dart' % namespace.name), DartGenerator(...)
  ]

for filename, generator in generators:
  with open(os.path.join(dest_dir, filename)) as f:
    f.write(generator.Generate(namespace).Render())

With that in mind I also think it makes more sense to have just a single --mode
flag on the compiler, with cpp_bundle, cpp, and dart options.

P.S. now that the Generate methods take a namespace rather than the constructor,
I'd suggest you make those generators look something like

class HGenerator(object):
  def __init__(self, type_generator):
    self._type_generator = type_generator

  def Generate(self, namespace);
    return _Generator(self._type_generator, namespace).Generate()

class _Generator(object):
  # exactly the contents that HGenerator used to have

[sashab, 2013/01/30 05:26:03]

Unfortunately, it doesn't seem that simple, for a few reasons:
  * The current cpp generation seems to pull in all dependencies for a file, as
well as the file itself, whereas the bundle generator doesn't check for
dependencies
    - This also means that the cpp generator can output multiple schemas
(multiple cpp files) for one IDL file
    - There's a comment about whether this is necessary for IDL. For simplicity,
I've taken it out for now.
  * Depending on the language, nodes may or may not be deleted (like nocompile).
For now, this is hardcoded as an if-statement checking for c++ options (bundle
or lang=c++).

[not at google - send to devlin, 2013/01/31 02:08:48]

I think these are surmountable. If a generator needs to output multiple files,
then just make it multiple generators (I think the code snippet I generated
handles this). It didn't handle the nocompile thing - but in that code snippet,
we could build the namespace for each generator and make the node removal one of
the variables, like

generators = [
  (('%s.dart' % namespace.name), 'nodart', DartGenerator(...))
]

+      type_generator = cpp_type_generator.CppTypeGenerator(
+          root_namespace, namespace, namespace.unix_name)
+      for referenced_namespace in api_model.namespaces.values():
+        if referenced_namespace == namespace:
+          continue
+        type_generator.AddNamespace(
+            referenced_namespace,
+            referenced_namespace.unix_name)
 
-    if dest_dir:
-      with open(
-          os.path.join(dest_dir, namespace.source_file_dir, out_file + '.cc'),
-          'w') as cc_file:
-        cc_file.write(cc_code)
-      with open(
-          os.path.join(dest_dir, namespace.source_file_dir, out_file + '.h'),
-          'w') as h_file:
-        h_file.write(h_code)
+      h_code = (h_generator.HGenerator(namespace, type_generator)
+          .Generate().Render())
+      cc_code = (cc_generator.CCGenerator(namespace, type_generator)
+          .Generate().Render())
+
+      if dest_dir:
+        with open(
+            os.path.join(dest_dir, namespace.source_file_dir, out_file + '.cc'),
+            'w') as cc_file:
+          cc_file.write(cc_code)
+        with open(
+            os.path.join(dest_dir, namespace.source_file_dir, out_file + '.h'),
+            'w') as h_file:
+          h_file.write(h_code)
+      else:
+        print '%s.h' % out_file
+        print
+        print h_code
+        print
+        print '%s.cc' % out_file
+        print
+        print cc_code
+
+    elif lang.lower() == 'dart':
+      dart_code = (dart_generator.DartGenerator(namespace, hooks_file)
+          .Generate().Render())
+
+      if dest_dir:
+        with open(os.path.join(dest_dir, out_file + '.dart'), 'w') as dart_file:
+          dart_file.write(dart_code)
+      else:
+        print '%s.dart' % out_file
+        print
+        print dart_code
+
     else:
-      print '%s.h' % out_file
-      print
-      print h_code
-      print
-      print '%s.cc' % out_file
-      print
-      print cc_code
+      raise Exception('Unrecognised language %s. Supported languages are %s.' %
+        (lang, SUPPORTED_LANGS))
 
-def handle_bundle_schema(filenames, dest_dir, root, root_namespace):
+def handle_bundle_schema(filenames, dest_dir, root, root_namespace, lang):
   # Merge the source files into a single list of schemas.
   api_defs = []
   for filename in filenames:
     schema = os.path.normpath(filename)
     schema_filename, schema_extension = os.path.splitext(schema)
     api_defs.extend(load_schema(schema))
 
   api_model = model.Model()
   relpath = os.path.relpath(os.path.normpath(filenames[0]), root)
 
   for target_namespace, schema_filename in zip(api_defs, filenames):
     namespace = api_model.AddNamespace(target_namespace, relpath)
     path, filename = os.path.split(schema_filename)
     short_filename, extension = os.path.splitext(filename)
 
     # Filenames are checked against the unix_names of the namespaces they
     # generate because the gyp uses the names of the JSON files to generate
     # the names of the .cc and .h files. We want these to be using unix_names.
     if namespace.unix_name != short_filename:
-      sys.exit("Filename %s is illegal. Name files using unix_hacker style." %
+      sys.exit('Filename %s is illegal. Name files using unix_hacker style.' %
                schema_filename)
 
-  type_generator = cpp_type_generator.CppTypeGenerator(root_namespace)
-  for referenced_namespace in api_model.namespaces.values():
-    type_generator.AddNamespace(
-        referenced_namespace,
-        referenced_namespace.unix_name)
+  if lang == 'c++':
+    type_generator = cpp_type_generator.CppTypeGenerator(root_namespace)
+    for referenced_namespace in api_model.namespaces.values():
+      type_generator.AddNamespace(
+          referenced_namespace,
+          referenced_namespace.unix_name)
 
-  generator = schema_bundle_generator.SchemaBundleGenerator(
-      api_model, api_defs, type_generator)
-  api_h_code = generator.GenerateAPIHeader().Render()
-  schemas_h_code = generator.GenerateSchemasHeader().Render()
-  schemas_cc_code = generator.GenerateSchemasCC().Render()
+    generator = schema_bundle_generator.SchemaBundleGenerator(
+        api_model, api_defs, type_generator)
+    api_h_code = generator.GenerateAPIHeader().Render()
+    schemas_h_code = generator.GenerateSchemasHeader().Render()
+    schemas_cc_code = generator.GenerateSchemasCC().Render()
 
-  if dest_dir:
-    basedir = os.path.join(dest_dir, 'chrome/common/extensions/api')
-    with open(os.path.join(basedir, 'generated_api.h'), 'w') as h_file:
-      h_file.write(api_h_code)
-    with open(os.path.join(basedir, 'generated_schemas.h'), 'w') as h_file:
-      h_file.write(schemas_h_code)
-    with open(os.path.join(basedir, 'generated_schemas.cc'), 'w') as cc_file:
-      cc_file.write(schemas_cc_code)
+    if dest_dir:
+      basedir = os.path.join(dest_dir, 'chrome/common/extensions/api')
+      with open(os.path.join(basedir, 'generated_api.h'), 'w') as h_file:
+        h_file.write(api_h_code)
+      with open(os.path.join(basedir, 'generated_schemas.h'), 'w') as h_file:
+        h_file.write(schemas_h_code)
+      with open(os.path.join(basedir, 'generated_schemas.cc'), 'w') as cc_file:
+        cc_file.write(schemas_cc_code)
+    else:

[not at google - send to devlin, 2013/01/25 18:14:33]

I think the time has come to delete this branch

[sashab, 2013/02/04 05:09:27]

Done.

+      print 'generated_api.h'
+      print
+      print api_h_code
+      print
+      print 'generated_schemas.h'
+      print
+      print schemas_h_code
+      print
+      print 'generated_schemas.cc'
+      print
+      print schemas_cc_code
+
+  elif lang == 'dart':
+    raise Exception('Dart is not supported in bundle mode.')
+
   else:
-    print 'generated_api.h'
-    print
-    print api_h_code
-    print
-    print 'generated_schemas.h'
-    print
-    print schemas_h_code
-    print
-    print 'generated_schemas.cc'
-    print
-    print schemas_cc_code
+    raise Exception('Unrecognised language %s. Supported languages are %s.' %
+      (lang, SUPPORTED_LANGS))
+
 
 if __name__ == '__main__':
   parser = optparse.OptionParser(
       description='Generates a C++ model of an API from JSON schema',
       usage='usage: %prog [option]... schema')
   parser.add_option('-r', '--root', default='.',
       help='logical include root directory. Path to schema files from specified'
       'dir will be the include path.')
   parser.add_option('-d', '--destdir',
       help='root directory to output generated files.')
   parser.add_option('-n', '--namespace', default='generated_api_schemas',
       help='C++ namespace for generated files. e.g extensions::api.')
-  parser.add_option('-b', '--bundle', action="store_true", help=
+  parser.add_option('-b', '--bundle', action='store_true', help=
 '''if supplied, causes compiler to generate bundle files for the given set of
 source files.''')
+  parser.add_option('-l', '--lang', default=SUPPORTED_LANGS[0],
+      help=
+'''The language to generate the output in. Currently supported options are
+%s.''' % SUPPORTED_LANGS)
+  parser.add_option('-H', '--hooks-file', dest='hooks_file',
+      help='Adds custom hooks from the given file (Dart only).', metavar='FILE')
 
   (opts, args) = parser.parse_args()
 
   if not args:
     sys.exit(0) # This is OK as a no-op
   dest_dir = opts.destdir
   root_namespace = opts.namespace
 
   if opts.bundle:
-    handle_bundle_schema(args, dest_dir, opts.root, root_namespace)
+    handle_bundle_schema(args, dest_dir, opts.root, root_namespace, opts.lang)
   else:
-    handle_single_schema(args[0], dest_dir, opts.root, root_namespace)
+    handle_single_schema(args[0], dest_dir, opts.root, root_namespace,
+        opts.lang, opts.hooks_file)