Speed up fremontcut/dartdomgenerator

lib/html/scripts/databasebuilder.py

 #!/usr/bin/python
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import copy
 import database
 import idlparser
 import logging
+import multiprocessing
 import os
 import os.path
 import re
 
 from idlnode import *
 
 _logger = logging.getLogger('databasebuilder')
 
 # Used in source annotations to specify the parent interface declaring
 # a displaced declaration. The 'via' attribute specifies the parent interface
@@ skipping 36 matching lines @@
     self.source_attributes = source_attributes
     self.idl_syntax = idl_syntax
     self.idl_defines = idl_defines
     self.type_rename_map = type_rename_map
     self.rename_operation_arguments_on_merge = \
         rename_operation_arguments_on_merge
     self.add_new_interfaces = add_new_interfaces
     self.obsolete_old_declarations = obsolete_old_declarations
 
 
+def _load_idl_file(file_name, import_options, result_queue):
+  """Loads an IDL file into memory"""
+  idl_parser = idlparser.IDLParser(import_options.idl_syntax)
+
+  try:
+    f = open(file_name, 'r')
+    content = f.read()
+    f.close()
+
+    idl_ast = idl_parser.parse(
+      content,
+      defines=import_options.idl_defines)
+    result = IDLFile(idl_ast, file_name)
+    result_queue.put(result, False)
+    return 0
+  except SyntaxError, e:
+    result_queue.put(RuntimeError('Failed to load file %s: %s'
+                                  % (file_name, e)),
+                     False)
+    return 1
+  except:
+    result_queue.put('Unknown error loading %s' % file_name, False)
+    return 1
+
 class DatabaseBuilder(object):
   def __init__(self, database):
     """DatabaseBuilder is used for importing and merging interfaces into
     the Database"""
     self._database = database
     self._imported_interfaces = []
     self._impl_stmts = []
 
-  def _load_idl_file(self, file_name, import_options):
-    """Loads an IDL file intor memory"""
-    idl_parser = idlparser.IDLParser(import_options.idl_syntax)
-
-    try:
-      f = open(file_name, 'r')
-      content = f.read()
-      f.close()
-
-      idl_ast = idl_parser.parse(content,
-        defines=import_options.idl_defines)
-      return IDLFile(idl_ast, file_name)
-    except SyntaxError, e:
-      raise RuntimeError('Failed to load file %s: %s' % (file_name, e))
-
   def _resolve_type_defs(self, idl_file):
     type_def_map = {}
     # build map
     for type_def in idl_file.all(IDLTypeDef):
       if type_def.type.id != type_def.id: # sanity check
         type_def_map[type_def.id] = type_def.type.id
     # use the map
     for type_node in idl_file.all(IDLType):
       while type_node.id in type_def_map:
         type_node.id = type_def_map[type_node.id]
 
   def _strip_ext_attributes(self, idl_file):
     """Strips unuseful extended attributes."""
     for ext_attrs in idl_file.all(IDLExtAttrs):
       # TODO: Decide which attributes are uninteresting.
       pass
 
   def _rename_types(self, idl_file, import_options):
     """Rename interface and type names with names provided in the
     options. Also clears scopes from scoped names"""
 
     def rename(name):
       name_parts = name.split('::')
       name = name_parts[-1]
       if name in import_options.type_rename_map:
         name = import_options.type_rename_map[name]
       return name
 
     def rename_node(idl_node):
-      idl_node.id = rename(idl_node.id)
+      idl_node.reset_id(rename(idl_node.id))
 
     def rename_ext_attrs(ext_attrs_node):
       for type_valued_attribute_name in ['Supplemental']:
         if type_valued_attribute_name in ext_attrs_node:
           value = ext_attrs_node[type_valued_attribute_name]
           if isinstance(value, str):
             ext_attrs_node[type_valued_attribute_name] = rename(value)
 
     map(rename_node, idl_file.all(IDLInterface))
     map(rename_node, idl_file.all(IDLType))
@@ skipping 186 matching lines @@
         not new_interface.is_supplemental):
       old_interface.annotations[source] = new_interface.annotations[source]
       changed = True
 
     def merge_list(what):
       old_list = old_interface.__dict__[what]
       new_list = new_interface.__dict__[what]
 
       if what != 'parents' and old_interface.id != new_interface.id:
         for node in new_list:
+          node.doc_js_interface_name = old_interface.id
           node.ext_attrs['ImplementedBy'] = new_interface.id
 
       changed = self._merge_nodes(old_list, new_list, import_options)
 
       # Delete list items with zero remaining annotations.
       if changed and import_options.obsolete_old_declarations:
 
         def has_annotations(idl_node):
           return len(idl_node.annotations)
 
@@ skipping 79 matching lines @@
         else:
           raise Exception("Supplemental target '%s' not found", target)
 
     # Step 4: Resolve 'implements' statements
     for impl_stmt, import_options in self._impl_stmts:
       self._merge_impl_stmt(impl_stmt, import_options)
 
     self._impl_stmts = []
     self._imported_interfaces = []
 
-  def import_idl_file(self, file_path,
-            import_options=DatabaseBuilderOptions()):
-    """Parses, loads into memory and cleans up and IDL file"""
-    idl_file = self._load_idl_file(file_path, import_options)
+  def import_idl_files(self, file_paths, import_options):
+    # Parse the IDL files in parallel.
+    result_queue = multiprocessing.Queue(len(file_paths))
+    jobs = [ multiprocessing.Process(target=_load_idl_file,
+                                     args=(file_path, import_options,
+                                           result_queue))
+             for file_path in file_paths ]
+    try:
+      for job in jobs:
+        job.start()
+      for job in jobs:
+        # Timeout and throw after 5 sec.
+        result = result_queue.get(True, 5)
+        if isinstance(result, IDLFile):
+          self._process_idl_file(result, import_options)
+        else:
+          raise result
+    except:
+      # Clean up child processes on error.
+      for job in jobs:
+        job.terminate()
+      raise
 
+  def _process_idl_file(self, idl_file,
+                        import_options):
     self._strip_ext_attributes(idl_file)
     self._resolve_type_defs(idl_file)
     self._rename_types(idl_file, import_options)
 
     def enabled(idl_node):
       return self._is_node_enabled(idl_node, import_options.idl_defines)
 
     for module in idl_file.modules:
       for interface in module.interfaces:
         if not self._is_node_enabled(interface, import_options.idl_defines):
-          _logger.info('skipping interface %s/%s (source=%s file=%s)'
-            % (module.id, interface.id, import_options.source,
-               file_path))
+          _logger.info('skipping interface %s/%s (source=%s)'
+            % (module.id, interface.id, import_options.source))
           continue
 
-        _logger.info('importing interface %s/%s (source=%s file=%s)'
-          % (module.id, interface.id, import_options.source,
-             file_path))
+        _logger.info('importing interface %s/%s (source=%s)'
+          % (module.id, interface.id, import_options.source))
         interface.attributes = filter(enabled, interface.attributes)
         interface.operations = filter(enabled, interface.operations)
         self._imported_interfaces.append((interface, module.id, import_options))
 
       for implStmt in module.implementsStatements:
         self._impl_stmts.append((implStmt, import_options))
 
   def _is_node_enabled(self, node, idl_defines):
     if not 'Conditional' in node.ext_attrs:
       return True
@@ skipping 93 matching lines @@
       # TODO(antonm): Ideally we'd like to have pristine copy of WebKit IDLs and fetch
       # this information directly from it.  Unfortunately right now database is massaged
       # a lot so it's difficult to maintain necessary information on DOMWindow itself.
       interface = self._database.GetInterface(options.type_rename_map.get(type, type))
       if 'V8EnabledPerContext' in attr.ext_attrs:
         interface.ext_attrs['synthesizedV8EnabledPerContext'] = \
             attr.ext_attrs['V8EnabledPerContext']
       if 'V8EnabledAtRuntime' in attr.ext_attrs:
         interface.ext_attrs['synthesizedV8EnabledAtRuntime'] = \
             attr.ext_attrs['V8EnabledAtRuntime'] or attr.id