Speed up fremontcut/dartdomgenerator

lib/html/scripts/databasebuilder.py

 #!/usr/bin/python
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import copy
 import database
 import idlparser
 import logging
+import multiprocessing
 import os
 import os.path
 import re
 
 from idlnode import *
 
 _logger = logging.getLogger('databasebuilder')
 
 # Used in source annotations to specify the parent interface declaring
 # a displaced declaration. The 'via' attribute specifies the parent interface
@@ skipping 36 matching lines @@
     self.source_attributes = source_attributes
     self.idl_syntax = idl_syntax
     self.idl_defines = idl_defines
     self.type_rename_map = type_rename_map
     self.rename_operation_arguments_on_merge = \
         rename_operation_arguments_on_merge
     self.add_new_interfaces = add_new_interfaces
     self.obsolete_old_declarations = obsolete_old_declarations
 
 
+def _load_idl_file(file_name, import_options, result_queue):
+  """Loads an IDL file into memory"""
+  idl_parser = idlparser.IDLParser(import_options.idl_syntax)
+
+  try:
+    f = open(file_name, 'r')
+    content = f.read()
+    f.close()
+
+    idl_ast = idl_parser.parse(
+      content,
+      defines=import_options.idl_defines)
+    result = IDLFile(idl_ast, file_name)
+    result_queue.put(result, False)
+    return 0
+  except SyntaxError, e:
+    result_queue.put(RuntimeError('Failed to load file %s: %s' % (file_name, e)),

[Anton Muhin, 2012/09/26 17:53:09]

nit: is line too long?  I am personally fine with it, so feel free to ignore

[vsm, 2012/09/28 16:17:25]

Done.

+                     False)
+    return 1
+  except:
+    result_queue.put('Unknown error loading %s' % file_name, False)
+    return 1
+
 class DatabaseBuilder(object):
   def __init__(self, database):
     """DatabaseBuilder is used for importing and merging interfaces into
     the Database"""
     self._database = database
     self._imported_interfaces = []
     self._impl_stmts = []
 
-  def _load_idl_file(self, file_name, import_options):
-    """Loads an IDL file intor memory"""
-    idl_parser = idlparser.IDLParser(import_options.idl_syntax)
-
-    try:
-      f = open(file_name, 'r')
-      content = f.read()
-      f.close()
-
-      idl_ast = idl_parser.parse(content,
-        defines=import_options.idl_defines)
-      return IDLFile(idl_ast, file_name)
-    except SyntaxError, e:
-      raise RuntimeError('Failed to load file %s: %s' % (file_name, e))
-
   def _resolve_type_defs(self, idl_file):
     type_def_map = {}
     # build map
     for type_def in idl_file.all(IDLTypeDef):
       if type_def.type.id != type_def.id: # sanity check
         type_def_map[type_def.id] = type_def.type.id
     # use the map
     for type_node in idl_file.all(IDLType):
       while type_node.id in type_def_map:
         type_node.id = type_def_map[type_node.id]
 
   def _strip_ext_attributes(self, idl_file):
     """Strips unuseful extended attributes."""
     for ext_attrs in idl_file.all(IDLExtAttrs):
       # TODO: Decide which attributes are uninteresting.
       pass
 
   def _rename_types(self, idl_file, import_options):
     """Rename interface and type names with names provided in the
     options. Also clears scopes from scoped names"""
 
     def rename(name):
       name_parts = name.split('::')
       name = name_parts[-1]
       if name in import_options.type_rename_map:
         name = import_options.type_rename_map[name]
       return name
 
     def rename_node(idl_node):
-      idl_node.id = rename(idl_node.id)
+      new_name = rename(idl_node.id)
+      if new_name != idl_node.id:
+        idl_node.id = new_name
+        if isinstance(idl_node, IDLInterface):

[Anton Muhin, 2012/09/26 17:53:09]

up to you, but might be more natural to have a method on IDLNode which is
overridden on IDLInterface.

[vsm, 2012/09/28 16:17:25]

Done.

+          idl_node.doc_js_name = new_name

[Anton Muhin, 2012/09/26 17:53:09]

I am not familiar with this part of the system, but new_name looks like name
dart:html lib, judging from name.  If that's true, resetting
doc_js_name/javascript_binding_name with it looks a bit weird.

[vsm, 2012/09/28 16:17:25]

It's a little confusing, but the renaming during database building is about
normalization (DOMCoreException to DOMException).  It should make things closer
to the right name.

On 2012/09/26 17:53:09, Anton Muhin wrote:

+          idl_node.javascript_binding_name = new_name
+          for member in idl_node.operations:
+            member.doc_js_interface_name = new_name
+          for member in idl_node.attributes:
+            member.doc_js_interface_name = new_name
+          for member in idl_node.constants:
+            member.doc_js_interface_name = new_name
 
     def rename_ext_attrs(ext_attrs_node):
       for type_valued_attribute_name in ['Supplemental']:
         if type_valued_attribute_name in ext_attrs_node:
           value = ext_attrs_node[type_valued_attribute_name]
           if isinstance(value, str):
             ext_attrs_node[type_valued_attribute_name] = rename(value)
 
     map(rename_node, idl_file.all(IDLInterface))
     map(rename_node, idl_file.all(IDLType))
@@ skipping 186 matching lines @@
         not new_interface.is_supplemental):
       old_interface.annotations[source] = new_interface.annotations[source]
       changed = True
 
     def merge_list(what):
       old_list = old_interface.__dict__[what]
       new_list = new_interface.__dict__[what]
 
       if what != 'parents' and old_interface.id != new_interface.id:
         for node in new_list:
+          node.doc_js_interface_name = old_interface.id
           node.ext_attrs['ImplementedBy'] = new_interface.id
 
       changed = self._merge_nodes(old_list, new_list, import_options)
 
       # Delete list items with zero remaining annotations.
       if changed and import_options.obsolete_old_declarations:
 
         def has_annotations(idl_node):
           return len(idl_node.annotations)
 
@@ skipping 79 matching lines @@
         else:
           raise Exception("Supplemental target '%s' not found", target)
 
     # Step 4: Resolve 'implements' statements
     for impl_stmt, import_options in self._impl_stmts:
       self._merge_impl_stmt(impl_stmt, import_options)
 
     self._impl_stmts = []
     self._imported_interfaces = []
 
-  def import_idl_file(self, file_path,
-            import_options=DatabaseBuilderOptions()):
-    """Parses, loads into memory and cleans up and IDL file"""
-    idl_file = self._load_idl_file(file_path, import_options)
+  def import_idl_files(self, file_paths, import_options):
+    # Parse the IDL files in parallel.
+    result_queue = multiprocessing.Queue(len(file_paths))
+    jobs = [ multiprocessing.Process(target=_load_idl_file,
+                                     args=(file_path, import_options,
+                                           result_queue))
+             for file_path in file_paths ]
+    for job in jobs:
+      job.start()
+    try:
+      for job in jobs:
+        # Timeout and throw after 5 sec.
+        result = result_queue.get(True, 5)
+        if isinstance(result, IDLFile):
+          self._process_idl_file(result, import_options)
+        else:
+          raise result
+    except:
+      # Clean up child processes on error.
+      for job in jobs:
+        job.terminate()

[Anton Muhin, 2012/09/26 17:53:09]

if you just terminate the parent process, will it terminate all the children
processes?

And can job.start() fail?  If yes, you guard is apparently not enough.

[vsm, 2012/09/28 16:17:25]

Not clear from the docs if start can throw.  I've moved it into the try anyway.

The child processes do not call any blocking APIs now, so it should not block. 
On linux, I tried killing the parent and the child processes do not appear to
hang around.

On 2012/09/26 17:53:09, Anton Muhin wrote:

+      raise
 
+  def _process_idl_file(self, idl_file,
+                        import_options):
     self._strip_ext_attributes(idl_file)
     self._resolve_type_defs(idl_file)
     self._rename_types(idl_file, import_options)
 
     def enabled(idl_node):
       return self._is_node_enabled(idl_node, import_options.idl_defines)
 
     for module in idl_file.modules:
       for interface in module.interfaces:
         if not self._is_node_enabled(interface, import_options.idl_defines):
-          _logger.info('skipping interface %s/%s (source=%s file=%s)'
-            % (module.id, interface.id, import_options.source,
-               file_path))
+          _logger.info('skipping interface %s/%s (source=%s)'
+            % (module.id, interface.id, import_options.source))
           continue
 
-        _logger.info('importing interface %s/%s (source=%s file=%s)'
-          % (module.id, interface.id, import_options.source,
-             file_path))
+        _logger.info('importing interface %s/%s (source=%s)'
+          % (module.id, interface.id, import_options.source))
         interface.attributes = filter(enabled, interface.attributes)
         interface.operations = filter(enabled, interface.operations)
         self._imported_interfaces.append((interface, module.id, import_options))
 
       for implStmt in module.implementsStatements:
         self._impl_stmts.append((implStmt, import_options))
 
   def _is_node_enabled(self, node, idl_defines):
     if not 'Conditional' in node.ext_attrs:
       return True
@@ skipping 93 matching lines @@
       # TODO(antonm): Ideally we'd like to have pristine copy of WebKit IDLs and fetch
       # this information directly from it.  Unfortunately right now database is massaged
       # a lot so it's difficult to maintain necessary information on DOMWindow itself.
       interface = self._database.GetInterface(options.type_rename_map.get(type, type))
       if 'V8EnabledPerContext' in attr.ext_attrs:
         interface.ext_attrs['synthesizedV8EnabledPerContext'] = \
             attr.ext_attrs['V8EnabledPerContext']
       if 'V8EnabledAtRuntime' in attr.ext_attrs:
         interface.ext_attrs['synthesizedV8EnabledAtRuntime'] = \
             attr.ext_attrs['V8EnabledAtRuntime'] or attr.id