IDL parser rewrite in Python

Source/bindings/derived_sources.gyp

 #
 # Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 33 matching lines @@
     ],
     'python_idl_files': [
       '<@(python_core_idl_files)',
       '<@(python_modules_idl_files)',
       '<@(python_svg_idl_files)',
     ],
     'perl_and_python_idl_files': [
         '<@(deprecated_perl_idl_files)',
         '<@(python_idl_files)',
     ],
+    'webcore_test_support_idl_files': [
+        '<@(deprecated_perl_webcore_test_support_idl_files)',
+        '<@(python_webcore_test_support_idl_files)',
+    ],
 
     'bindings_output_dir': '<(SHARED_INTERMEDIATE_DIR)/webkit/bindings',
     'generated_global_constructors_idl_files': [
          '<(SHARED_INTERMEDIATE_DIR)/WindowConstructors.idl',
          '<(SHARED_INTERMEDIATE_DIR)/WorkerGlobalScopeConstructors.idl',
          '<(SHARED_INTERMEDIATE_DIR)/SharedWorkerGlobalScopeConstructors.idl',
          '<(SHARED_INTERMEDIATE_DIR)/DedicatedWorkerGlobalScopeConstructors.idl',
     ],
 
     'conditions': [
@@ skipping 81 matching lines @@
          '<(SHARED_INTERMEDIATE_DIR)/SharedWorkerGlobalScopeConstructors.idl',
          '--dedicatedworkerglobalscope-constructors-file',
          '<(SHARED_INTERMEDIATE_DIR)/DedicatedWorkerGlobalScopeConstructors.idl',
          '--event-names-file',
          '<(SHARED_INTERMEDIATE_DIR)/EventNames.in',
          '<@(write_file_only_if_changed)',
        ],
        'message': 'Resolving partial interfaces dependencies in all IDL files',
       }]
     },
-    {
-      'target_name': 'deprecated_perl_bindings_sources',
-      'type': 'none',
-      # The 'binding' rule generates .h files, so mark as hard_dependency, per:
-      # https://code.google.com/p/gyp/wiki/InputFormatReference#Linking_Dependencies
-      'hard_dependency': 1,
-      'dependencies': [
-        'interface_dependencies',
-        '../core/core_derived_sources.gyp:generate_test_support_idls',
-      ],
-      'sources': [
-        '<@(deprecated_perl_idl_files)',
-        '<@(deprecated_perl_webcore_test_support_idl_files)',
-      ],
-      'rules': [{
-        'rule_name': 'deprecated_perl_binding',
-        'extension': 'idl',
-        'msvs_external_rule': 1,
-        'inputs': [
-          'scripts/deprecated_generate_bindings.pl',
-          'scripts/deprecated_code_generator_v8.pm',
-          'scripts/deprecated_idl_parser.pm',
-          'scripts/deprecated_idl_serializer.pm',
-          '../core/scripts/preprocessor.pm',
-          'scripts/IDLAttributes.txt',
-          # FIXME: If the dependency structure changes, we rebuild all files,
-          # since we're not computing dependencies file-by-file in the build.
-          '<(SHARED_INTERMEDIATE_DIR)/InterfaceDependencies.txt',
-          # FIXME: Similarly, if any partial interface changes, rebuild
-          # everything, since every IDL potentially depends on them, because
-          # we're not computing dependencies file-by-file.
-          #
-          # If a new partial interface is added, need to regyp to update these
-          # dependencies, as these are computed statically at gyp runtime.
-          '<!@pymod_do_main(list_idl_files_with_partial_interface <@(perl_and_python_idl_files))',
-          # Generated IDLs are all partial interfaces, hence everything
-          # potentially depends on them.
-          '<@(generated_global_constructors_idl_files)',
-        ],
-        'outputs': [
-          '<(bindings_output_dir)/V8<(RULE_INPUT_ROOT).cpp',
-          '<(bindings_output_dir)/V8<(RULE_INPUT_ROOT).h',
-        ],
-        'variables': {
-          # IDL include paths. The generator will search recursively for IDL
-          # files under these locations.
-          'generator_include_dirs': [
-            '--include', '../core',
-            '--include', '../modules',
-            '--include', '<(SHARED_INTERMEDIATE_DIR)/webkit',
-          ],
-          # Hook for embedders to specify extra directories to find IDL files.
-          'extra_blink_generator_include_dirs%': [],
-        },
-        'msvs_cygwin_shell': 0,
-        # sanitize-win-build-log.sed uses a regex which matches this command
-        # line (Perl script + .idl file being processed).
-        # Update that regex if command line changes (other than changing flags)
-        'action': [
-          '<(perl_exe)',
-          '-w',
-          '-Iscripts',
-          '-I../core/scripts',
-          '-I../../../JSON/out/lib/perl5',
-          'scripts/deprecated_generate_bindings.pl',
-          '--outputDir',
-          '<(bindings_output_dir)',
-          '--idlAttributesFile',
-          'scripts/IDLAttributes.txt',
-          '<@(generator_include_dirs)',
-          '<@(extra_blink_generator_include_dirs)',
-          '--interfaceDependenciesFile',
-          '<(SHARED_INTERMEDIATE_DIR)/InterfaceDependencies.txt',
-          '--additionalIdlFiles',
-          '<(deprecated_perl_webcore_test_support_idl_files)',
-          '<@(preprocessor)',
-          '<@(write_file_only_if_changed)',
-          '<(RULE_INPUT_PATH)',
-        ],
-        'message': 'Generating binding from <(RULE_INPUT_PATH)',
-      }],
-    },
+    #{
+    #  'target_name': 'deprecated_perl_bindings_sources',
+    #  'type': 'none',
+    #  # The 'binding' rule generates .h files, so mark as hard_dependency, per:
+    #  # https://code.google.com/p/gyp/wiki/InputFormatReference#Linking_Dependencies
+    #  'hard_dependency': 1,
+    #  'dependencies': [
+    #    'interface_dependencies',
+    #    '../core/core_derived_sources.gyp:generate_test_support_idls',
+    #  ],
+    #  'sources': [
+    #    '<@(deprecated_perl_idl_files)',
+    #    '<@(deprecated_perl_webcore_test_support_idl_files)',
+    #  ],
+    #  'rules': [{
+    #    'rule_name': 'deprecated_perl_binding',
+    #    'extension': 'idl',
+    #    'msvs_external_rule': 1,
+    #    'inputs': [
+    #      'scripts/deprecated_generate_bindings.pl',
+    #      'scripts/deprecated_code_generator_v8.pm',
+    #      'scripts/deprecated_idl_parser.pm',
+    #      'scripts/deprecated_idl_serializer.pm',
+    #      '../core/scripts/preprocessor.pm',
+    #      'scripts/IDLAttributes.txt',
+    #      # FIXME: If the dependency structure changes, we rebuild all files,
+    #      # since we're not computing dependencies file-by-file in the build.
+    #      '<(SHARED_INTERMEDIATE_DIR)/InterfaceDependencies.txt',
+    #      # FIXME: Similarly, if any partial interface changes, rebuild
+    #      # everything, since every IDL potentially depends on them, because
+    #      # we're not computing dependencies file-by-file.
+    #      #
+    #      # If a new partial interface is added, need to regyp to update these
+    #      # dependencies, as these are computed statically at gyp runtime.
+    #      '<!@pymod_do_main(list_idl_files_with_partial_interface <@(perl_and_python_idl_files))',
+    #      # Generated IDLs are all partial interfaces, hence everything
+    #      # potentially depends on them.
+    #      '<@(generated_global_constructors_idl_files)',
+    #    ],
+    #    'outputs': [
+    #      '<(bindings_output_dir)/V8<(RULE_INPUT_ROOT).cpp',
+    #      '<(bindings_output_dir)/V8<(RULE_INPUT_ROOT).h',
+    #    ],
+    #    'variables': {
+    #      # IDL include paths. The generator will search recursively for IDL
+    #      # files under these locations.
+    #      'generator_include_dirs': [
+    #        '--include', '../core',
+    #        '--include', '../modules',
+    #        '--include', '<(SHARED_INTERMEDIATE_DIR)/webkit',
+    #      ],
+    #      # Hook for embedders to specify extra directories to find IDL files.
+    #      'extra_blink_generator_include_dirs%': [],
+    #    },
+    #    'msvs_cygwin_shell': 0,
+    #    # sanitize-win-build-log.sed uses a regex which matches this command
+    #    # line (Perl script + .idl file being processed).
+    #    # Update that regex if command line changes (other than changing flags)
+    #    'action': [
+    #      '<(perl_exe)',
+    #      '-w',
+    #      '-Iscripts',
+    #      '-I../core/scripts',
+    #      '-I../../../JSON/out/lib/perl5',
+    #      'scripts/deprecated_generate_bindings.pl',
+    #      '--outputDir',
+    #      '<(bindings_output_dir)',
+    #      '--idlAttributesFile',
+    #      'scripts/IDLAttributes.txt',
+    #      '<@(generator_include_dirs)',
+    #      '<@(extra_blink_generator_include_dirs)',
+    #      '--interfaceDependenciesFile',
+    #      '<(SHARED_INTERMEDIATE_DIR)/InterfaceDependencies.txt',
+    #      '--additionalIdlFiles',
+    #      '<(deprecated_perl_webcore_test_support_idl_files)',
+    #      '<@(preprocessor)',
+    #      '<@(write_file_only_if_changed)',
+    #      '<(RULE_INPUT_PATH)',
+    #    ],
+    #    'message': 'Generating binding from <(RULE_INPUT_PATH)',
+    #  }],
+    #},

[haraken, 2013/07/16 14:17:51]

Remove all changes for testing before landing.

[Nils Barth (inactive), 2013/07/17 12:05:09]

Will do! I’ve added FIXMEs, and it’ll be caught by the try bots, but could you
please hold off the ok until these are removed?

     {
       'target_name': 'python_bindings_sources',
       'type': 'none',
       # The 'binding' rule generates .h files, so mark as hard_dependency, per:
       # https://code.google.com/p/gyp/wiki/InputFormatReference#Linking_Dependencies
       'hard_dependency': 1,
       'dependencies': [
         'interface_dependencies',
         '../core/core_derived_sources.gyp:generate_test_support_idls',
       ],
       'sources': [
         '<@(python_idl_files)',
         '<@(python_webcore_test_support_idl_files)',
+        # FIXME: testing so dump JSON
+        '<@(deprecated_perl_idl_files)',
+        '<@(deprecated_perl_webcore_test_support_idl_files)',
       ],
       'rules': [{
         'rule_name': 'python_binding',
         'extension': 'idl',
         'msvs_external_rule': 1,
         'inputs': [
           'scripts/idl_compiler.py',
+          '../../../ply/lex.py',
+          '../../../ply/yacc.py',
+          '../../../../tools/idl_parser/idl_lexer.py',
+          '../../../../tools/idl_parser/idl_node.py',
+          '../../../../tools/idl_parser/idl_parser.py',
+          'scripts/blink_idl_lexer.py',
+          'scripts/blink_idl_parser.py',
+          'scripts/code_generator_v8.py',
+          'scripts/idl_definitions.py',
+          'scripts/idl_definitions_builder.py',
           'scripts/idl_reader.py',
+          'scripts/idl_validator.py',
           'scripts/interface_dependency_resolver.py',
-          'scripts/code_generator_v8.py',
           'scripts/IDLAttributes.txt',
           # FIXME: If the dependency structure changes, we rebuild all files,
           # since we're not computing dependencies file-by-file in the build.
           '<(SHARED_INTERMEDIATE_DIR)/InterfaceDependencies.txt',
           # FIXME: Similarly, if any partial interface changes, rebuild
           # everything, since every IDL potentially depends on them, because
           # we're not computing dependencies file-by-file.
           '<!@pymod_do_main(list_idl_files_with_partial_interface <@(perl_and_python_idl_files))',
           # Generated IDLs are all partial interfaces, hence everything
           # potentially depends on them.
@@ skipping 23 matching lines @@
           'scripts/idl_compiler.py',
           '--output-dir',
           '<(bindings_output_dir)',
           '--idl-attributes-file',
           'scripts/IDLAttributes.txt',
           '<@(generator_include_dirs)',
           '<@(extra_blink_generator_include_dirs)',
           '--interface-dependencies-file',
           '<(SHARED_INTERMEDIATE_DIR)/InterfaceDependencies.txt',
           '--additional-idl-files',
-          '<(python_webcore_test_support_idl_files)',
+          '<(webcore_test_support_idl_files)',
           '<@(write_file_only_if_changed)',
+          # FIXME: testing, so dump JSON

[haraken, 2013/07/16 14:17:51]

I'm OK with leaving this option for debugging purpose until you finish migrating
everything to the Python flow.

However, more detailed comment please.

e.g.: # FIXME: This option is only for debugging and will be removed once we
complete migrating all IDL files from the Perl flow to the Python flow.

[Nils Barth (inactive), 2013/07/17 12:05:09]

Ok, this makes it easier.
(Put comment in idl_compiler.py)

+          '--dump-json-and-pickle',
           '<(RULE_INPUT_PATH)',
         ],
         'message': 'Generating binding from <(RULE_INPUT_PATH)',
       }],
     },
     {
       'target_name': 'bindings_derived_sources',
       'type': 'none',
       'dependencies': [
         'interface_dependencies',
-        'deprecated_perl_bindings_sources',
+        # 'deprecated_perl_bindings_sources',
         'python_bindings_sources',
       ],
       'actions': [{
         'action_name': 'derived_sources_all_in_one',
         'inputs': [
           '../core/scripts/action_derivedsourcesallinone.py',
           '<(SHARED_INTERMEDIATE_DIR)/InterfaceDependencies.txt',
         ],
         'outputs': [
           '<@(derived_sources_aggregate_files)',
         ],
         'action': [
           'python',
           '../core/scripts/action_derivedsourcesallinone.py',
           '<(SHARED_INTERMEDIATE_DIR)/InterfaceDependencies.txt',
           '--',
           '<@(derived_sources_aggregate_files)',
         ],
         'message': 'Generating bindings derived sources',
       }],
     },
   ],
 }