Faster run-bindings-tests

Source/bindings/scripts/compute_dependencies.py

 #!/usr/bin/python
 #
 # Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
@@ skipping 57 matching lines @@
     """Raised if the IDL file implementing an interface cannot be found."""
     pass
 
 
 def parse_options():
     usage = 'Usage: %prog [options] [generated1.idl]...'
     parser = optparse.OptionParser(usage=usage)
     parser.add_option('--event-names-file', help='output file')
     parser.add_option('--idl-files-list', help='file listing IDL files')
     parser.add_option('--interface-dependencies-file', help='output file')
-    parser.add_option('--interfaces-info-file', help='output pickle file')

[Nils Barth (inactive), 2014/02/24 08:38:56]

We need this separate file for the build, so could you not remove it?
In the build, the compute_dependencies step happens before the parallel compile,
so we need a separate file.

You can see the build in:
Source/bindings/generated_bindings.gyp
...and I'll document it at:
http://www.chromium.org/developers/design-documents/idl-build

[terry, 2014/02/27 23:40:26]

Done.

[Nils Barth (inactive), 2014/03/03 06:45:48]

Just to clarify:
it's fine to not write InterfacesInfo.pickle in r-b-t,
as you suggest,
but we do need it in the build.

     parser.add_option('--window-constructors-file', help='output file')
     parser.add_option('--workerglobalscope-constructors-file', help='output file')
     parser.add_option('--sharedworkerglobalscope-constructors-file', help='output file')
     parser.add_option('--dedicatedworkerglobalscope-constructors-file', help='output file')
     parser.add_option('--serviceworkerglobalscope-constructors-file', help='output file')
     parser.add_option('--write-file-only-if-changed', type='int', help='if true, do not write an output file if it would be identical to the existing one, which avoids unnecessary rebuilds in ninja')
     options, args = parser.parse_args()
     if options.event_names_file is None:
         parser.error('Must specify an output file using --event-names-file.')
     if options.interface_dependencies_file is None:
@@ skipping 229 matching lines @@
 
 def add_paths_to_partials_dict(partial_interface_name, full_path, this_include_path=None):
     paths_dict = partial_interface_files.setdefault(partial_interface_name,
                                                     {'full_paths': [],
                                                      'include_paths': []})
     paths_dict['full_paths'].append(full_path)
     if this_include_path:
         paths_dict['include_paths'].append(this_include_path)
 
 
-def generate_dependencies(idl_filename):
+def generate_dependencies(idl_filename, idl_full_path, idl_file_contents):
     """Compute dependencies for IDL file, returning True if main (non-partial) interface"""
-    full_path = os.path.realpath(idl_filename)
-    idl_file_contents = get_file_contents(full_path)
 
     extended_attributes = get_interface_extended_attributes_from_idl(idl_file_contents)
     implemented_as = extended_attributes.get('ImplementedAs')
     # FIXME: remove [NoHeader] once switch to Python
     this_include_path = (include_path(idl_filename, implemented_as)
                          if 'NoHeader' not in extended_attributes else None)
 
     # Handle partial interfaces
     partial_interface_name = get_partial_interface_name_from_idl(idl_file_contents)
     if partial_interface_name:
-        add_paths_to_partials_dict(partial_interface_name, full_path, this_include_path)
+        add_paths_to_partials_dict(partial_interface_name, idl_full_path, this_include_path)
         return False
 
     # If not a partial interface, the basename is the interface name
     interface_name, _ = os.path.splitext(os.path.basename(idl_filename))
 
     interfaces_info[interface_name] = {
-        'full_path': full_path,
+        'full_path': idl_full_path,
         'implements_interfaces': get_implemented_interfaces_from_idl(idl_file_contents, interface_name),
         'is_callback_interface': is_callback_interface_from_idl(idl_file_contents),
         # Interfaces that are referenced (used as types) and that we introspect
         # during code generation (beyond interface-level data ([ImplementedAs],
         # is_callback_interface, ancestors, and inherited extended attributes):
         # deep dependencies.
         # These cause rebuilds of referrers, due to the dependency, so these
         # should be minimized; currently only targets of [PutForwards].
         'referenced_interfaces': get_put_forward_interfaces_from_idl(idl_file_contents),
     }
@@ skipping 25 matching lines @@
         named_constructor = extended_attributes['NamedConstructor']
         # Extract function name, namely everything before opening '('
         constructor_name = re.sub(r'\(.*', '', named_constructor)
         # Note the reduplicated 'ConstructorConstructor'
         attribute_string = 'attribute %sConstructorConstructor %s' % (interface_name, constructor_name)
         attributes_list.append(extended_string + attribute_string)
 
     return attributes_list
 
 
-def record_global_constructors_and_extended_attributes(idl_filename, global_constructors):
+def record_global_constructors_and_extended_attributes(idl_filename, idl_file_contents, global_constructors):
     interface_name, _ = os.path.splitext(os.path.basename(idl_filename))
-    full_path = os.path.realpath(idl_filename)
-    idl_file_contents = get_file_contents(full_path)
     extended_attributes = get_interface_extended_attributes_from_idl(idl_file_contents)
 
     # Record extended attributes
     extended_attributes_by_interface[interface_name] = extended_attributes
 
     # Record global constructors
     if (not is_callback_interface_from_idl(idl_file_contents) and
         'NoInterfaceObject' not in extended_attributes):
         global_contexts = extended_attributes.get('GlobalContext', 'Window').split('&')
         new_constructor_list = generate_constructor_attribute_list(interface_name, extended_attributes)
@@ skipping 59 matching lines @@
         global_constructors:
             dict of global objects -> list of constructors on that object
     """
     global_constructors = dict([
         (global_object, [])
         for global_object in global_constructors_filenames])
 
     # Generate dependencies, and (for main IDL files), record
     # global_constructors and extended_attributes_by_interface.
     for idl_filename in idl_files:
+        interface_name, _ = os.path.splitext(os.path.basename(idl_filename))
+        full_path = os.path.realpath(idl_filename)
+        idl_file_contents = get_file_contents(full_path)
+
         # Test skips partial interfaces
-        if generate_dependencies(idl_filename):
-            record_global_constructors_and_extended_attributes(idl_filename, global_constructors)
+        if generate_dependencies(idl_filename, full_path, idl_file_contents):
+            record_global_constructors_and_extended_attributes(idl_filename,
+                                                               idl_file_contents,
+                                                               global_constructors)
 
     for interface_name in interfaces_info:
         generate_ancestors_and_inherited_extended_attributes(interface_name)
 
     # Add constructors on global objects to partial interfaces
     # These are all partial interfaces, but the files are dynamically generated,
     # so they need to be handled separately from static partial interfaces.
     for global_object, constructor_filename in global_constructors_filenames.iteritems():
         if global_object in interfaces_info:
             # No include path needed, as already included in the header file
@@ skipping 22 matching lines @@
         interface_info['dependencies_full_paths'] = (
                 partial_interfaces_full_paths +
                 implemented_interfaces_full_paths)
         interface_info['dependencies_include_paths'] = (
                 partial_interfaces_include_paths +
                 implemented_interfaces_include_paths)
 
     return global_constructors
 
 
+def clear_globals():
+    interfaces_info.clear()
+    partial_interface_files.clear()
+    parent_interfaces.clear()
+    extended_attributes_by_interface.clear()  # interface name -> extended attributes
+
+
+def compute(idl_files_list, interface_dependencies_file,
+            window_constructors_file, workerglobalscope_constructors_file,
+            sharedworkerglobalscope_constructors_file,
+            dedicatedworkerglobalscope_constructors_file,
+            serviceworkerglobalscope_constructors_file,
+            event_names_file, only_if_changed):
+
+    clear_globals()
+
+    global_constructors_filenames = {
+            'Window': window_constructors_file,
+            'WorkerGlobalScope': workerglobalscope_constructors_file,
+            'SharedWorkerGlobalScope': sharedworkerglobalscope_constructors_file,
+            'DedicatedWorkerGlobalScope': dedicatedworkerglobalscope_constructors_file,
+            'ServiceWorkerGlobalScope': serviceworkerglobalscope_constructors_file,
+    }
+
+    global_constructors = parse_idl_files(idl_files_list, global_constructors_filenames)
+
+    write_dependencies_file(interface_dependencies_file, only_if_changed)
+    for interface_name, filename in global_constructors_filenames.iteritems():
+        if interface_name in interfaces_info:
+            write_global_constructors_partial_interface(interface_name, filename, global_constructors[interface_name], only_if_changed)
+    write_event_names_file(event_names_file, only_if_changed)
+    return interfaces_info
+
+
 ################################################################################
 
 def main():
     options, args = parse_options()
 
     # Static IDL files are passed in a file (generated at GYP time), due to OS
     # command line length limits
     with open(options.idl_files_list) as idl_files_list:
         idl_files = [line.rstrip('\n') for line in idl_files_list]
     # Generated IDL files are passed at the command line, since these are in the
     # build directory, which is determined at build time, not GYP time, so these
     # cannot be included in the file listing static files
     idl_files.extend(args)
 
-    only_if_changed = options.write_file_only_if_changed
-    global_constructors_filenames = {
-        'Window': options.window_constructors_file,
-        'WorkerGlobalScope': options.workerglobalscope_constructors_file,
-        'SharedWorkerGlobalScope': options.sharedworkerglobalscope_constructors_file,
-        'DedicatedWorkerGlobalScope': options.dedicatedworkerglobalscope_constructors_file,
-        'ServiceWorkerGlobalScope': options.serviceworkerglobalscope_constructors_file,
-        }
-
-    global_constructors = parse_idl_files(idl_files, global_constructors_filenames)
-
-    write_dependencies_file(options.interface_dependencies_file, only_if_changed)
-    write_pickle_file(options.interfaces_info_file, interfaces_info, only_if_changed)
-    for interface_name, filename in global_constructors_filenames.iteritems():
-        if interface_name in interfaces_info:
-            write_global_constructors_partial_interface(interface_name, filename, global_constructors[interface_name], only_if_changed)
-    write_event_names_file(options.event_names_file, only_if_changed)
-
+    compute(idl_files,
+            options.interface_dependencies_file,
+            options.window_constructors_file,
+            options.workerglobalscope_constructors_file,
+            options.sharedworkerglobalscope_constructors_file,
+            options.dedicatedworkerglobalscope_constructors_file,
+            options.serviceworkerglobalscope_constructors_file,
+            options.event_names_file,
+            options.write_file_only_if_changed)
 
 if __name__ == '__main__':
     main()