DevTools: speed up closure dependency checking

third_party/WebKit/Source/devtools/scripts/compile_frontend.py

 #!/usr/bin/env python
 # Copyright (c) 2012 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 20 matching lines @@
 import os.path as path
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 
 from build import modular_build
 from build import generate_protocol_externs
 
+import dependency_preprocessor
 import utils
 
 try:
     import simplejson as json
 except ImportError:
     import json
 
-
-if len(sys.argv) == 2 and sys.argv[1] == '--help':
-    print("Usage: %s [module_names]" % path.basename(sys.argv[0]))
-    print("  module_names    list of modules for which the Closure compilation should run.")
-    print("                  If absent, the entire frontend will be compiled.")
-    sys.exit(0)
-
 is_cygwin = sys.platform == 'cygwin'
 
 
 def popen(arguments):
     return subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
 def to_platform_path(filepath):
     if not is_cygwin:
         return filepath
     return re.sub(r'^/cygdrive/(\w)', '\\1:', filepath)
 
 
 def to_platform_path_exact(filepath):
     if not is_cygwin:
         return filepath
     output, _ = popen(['cygpath', '-w', filepath]).communicate()
     # pylint: disable=E1103
     return output.strip().replace('\\', '\\\\')
 
 scripts_path = path.dirname(path.abspath(__file__))
 devtools_path = path.dirname(scripts_path)
 inspector_path = path.join(path.dirname(devtools_path), 'core', 'inspector')
 # TODO(dgozman): move these checks to v8.
 v8_inspector_path = path.normpath(path.join(path.dirname(devtools_path), os.pardir, os.pardir, os.pardir, 'v8', 'src', 'inspector'))
 devtools_frontend_path = path.join(devtools_path, 'front_end')
 global_externs_file = to_platform_path(path.join(devtools_frontend_path, 'externs.js'))
 protocol_externs_file = path.join(devtools_frontend_path, 'protocol_externs.js')
-
-jsmodule_name_prefix = 'jsmodule_'
-runtime_module_name = '_runtime'
+runtime_file = to_platform_path(path.join(devtools_frontend_path, 'Runtime.js'))
 
 type_checked_jsdoc_tags_list = ['param', 'return', 'type', 'enum']
 type_checked_jsdoc_tags_or = '|'.join(type_checked_jsdoc_tags_list)
 
 # Basic regex for invalid JsDoc types: an object type name ([A-Z][_A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
 invalid_type_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*\{.*(?<![!?:._A-Za-z0-9])([A-Z][_A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}')
 invalid_type_designator_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*.*(?<![{: ])([?!])=?\}')
 invalid_non_object_type_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*\{.*(![a-z]+)[^/]*\}')
 error_warning_regex = re.compile(r'WARNING|ERROR')
 loaded_css_regex = re.compile(r'(?:registerRequiredCSS|WebInspector\.View\.createStyleElement)\s*\(\s*"(.+)"\s*\)')
@@ skipping 124 matching lines @@
         sys.exit(1)
     print 'Java executable: %s%s' % (java_path, '' if has_server_jvm else ' (no server JVM)')
     return exec_command
 
 java_exec = find_java()
 
 closure_compiler_jar = to_platform_path(path.join(scripts_path, 'closure', 'compiler.jar'))
 closure_runner_jar = to_platform_path(path.join(scripts_path, 'closure', 'closure_runner', 'closure_runner.jar'))
 jsdoc_validator_jar = to_platform_path(path.join(scripts_path, 'jsdoc_validator', 'jsdoc_validator.jar'))
 
-modules_dir = tempfile.mkdtemp()
 common_closure_args = [
     '--summary_detail_level', '3',
     '--jscomp_error', 'visibility',
     '--jscomp_warning', 'missingOverride',
     '--compilation_level', 'SIMPLE_OPTIMIZATIONS',
     '--warning_level', 'VERBOSE',
     '--language_in=ES6_STRICT',
     '--language_out=ES5_STRICT',
     '--extra_annotation_name', 'suppressReceiverCheck',
     '--extra_annotation_name', 'suppressGlobalPropertiesCheck',
     '--checks-only',
-    '--module_output_path_prefix', to_platform_path_exact(modules_dir + path.sep)
 ]
 
 worker_modules_by_name = {}
 dependents_by_module_name = {}
 
 for module_name in descriptors.application:
     module = descriptors.modules[module_name]
     if descriptors.application[module_name].get('type', None) == 'worker':

[dgozman, 2017/01/10 21:47:35]

FYI: this is never true anymore, so we should follow-up.

[chenwilliam, 2017/01/10 23:08:56]

made a todo.

         worker_modules_by_name[module_name] = module
     for dep in module.get('dependencies', []):
         list = dependents_by_module_name.get(dep)
         if not list:
             list = []
             dependents_by_module_name[dep] = list
         list.append(module_name)
 
 
 def check_conditional_dependencies():
@@ skipping 34 matching lines @@
             if referencing_module:
                 log_error('Duplicate use of %s in "%s" (previously seen in "%s")' % (source, name, referencing_module))
             seen_files[source] = name
 
     for module_name in worker_modules_by_name:
         check_module(worker_modules_by_name[module_name], {}, {})
 
 print 'Checking duplicate files across modules...'
 check_duplicate_files()
 
+print 'Compiling frontend...'
 
-def module_arg(module_name):
-    return ' --module ' + jsmodule_name_prefix + module_name
+temp_devtools_path = tempfile.mkdtemp()
 
 
-def modules_to_check():
-    if len(sys.argv) == 1:
-        return descriptors.sorted_modules()
-    print 'Compiling only these modules: %s' % sys.argv[1:]
-    return [module for module in descriptors.sorted_modules() if module in set(sys.argv[1:])]
+def prepare_closure_frontend_compile():
+    temp_frontend_path = path.join(temp_devtools_path, 'front_end')
+    checker = dependency_preprocessor.DependencyPreprocessor(descriptors, temp_frontend_path, devtools_frontend_path)
+    checker.enforce_dependencies()
 
+    command = common_closure_args + [
+        '--externs', to_platform_path(global_externs_file),
+        '--js', runtime_file,
+    ]
 
-def dump_module(name, recursively, processed_modules):
-    if name in processed_modules:
-        return ''
-    processed_modules[name] = True
-    module = modules_by_name[name]
+    all_files = descriptors.all_compiled_files()
+    args = []
+    for file in all_files:
+        args.extend(['--js', file])
+        if "InspectorBackend.js" in file:
+            args.extend(['--js', protocol_externs_file])
+    command += args
+    command = [arg.replace(devtools_frontend_path, temp_frontend_path) for arg in command]

[dgozman, 2017/01/10 21:47:34]

Please double this is necessary.

[chenwilliam, 2017/01/10 23:08:56]

just double checked it's a full path.

+    compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
+    try:
+        compiler_args_file.write('devtools_frontend %s' % (' '.join(command)))
+    finally:
+        compiler_args_file.close()
+    return compiler_args_file.name
 
-    command = ''
-    dependencies = module.get('dependencies', [])
-    if recursively:
-        for dependency in dependencies:
-            command += dump_module(dependency, recursively, processed_modules)
-    command += module_arg(name) + ':'
-    filtered_scripts = descriptors.module_compiled_files(name)
-    filtered_scripts = [path.join(devtools_frontend_path, name, script) for script in filtered_scripts]
-    if name == 'protocol':
-        filtered_scripts.append(protocol_externs_file)
-    command += str(len(filtered_scripts))
-    first_dependency = True
-    for dependency in dependencies + [runtime_module_name]:
-        if first_dependency:
-            command += ':'
-        else:
-            command += ','
-        first_dependency = False
-        command += jsmodule_name_prefix + dependency
-    for script in filtered_scripts:
-        command += ' --js ' + to_platform_path(script)
-    return command
+compiler_args_file_path = prepare_closure_frontend_compile()
+frontend_compile_proc = popen(java_exec + ['-jar', closure_runner_jar, '--compiler-args-file', to_platform_path_exact(compiler_args_file_path)])
 
-print 'Compiling frontend...'
+print 'Compiling devtools_compatibility.js...'
 
-compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
-try:
-    runtime_js_path = to_platform_path(path.join(devtools_frontend_path, 'Runtime.js'))
-    checked_modules = modules_to_check()
-    for name in checked_modules:
-        closure_args = ' '.join(common_closure_args)
-        closure_args += ' --externs ' + to_platform_path(global_externs_file)
-        runtime_module = module_arg(runtime_module_name) + ':1 --js ' + runtime_js_path
-        closure_args += runtime_module + dump_module(name, True, {})
-        compiler_args_file.write('%s %s%s' % (name, closure_args, os.linesep))
-finally:
-    compiler_args_file.close()
-
-modular_compiler_proc = popen(java_exec + ['-jar', closure_runner_jar, '--compiler-args-file', to_platform_path_exact(compiler_args_file.name)])
-
-spawned_compiler_command = java_exec + [
+closure_compiler_command = java_exec + [
     '-jar',
     closure_compiler_jar
 ] + common_closure_args
 
-print 'Compiling devtools_compatibility.js...'
-
-command = spawned_compiler_command + [
+devtools_js_compile_command = closure_compiler_command + [
     '--externs', to_platform_path(global_externs_file),
     '--externs', to_platform_path(path.join(devtools_frontend_path, 'host', 'InspectorFrontendHostAPI.js')),
     '--jscomp_off=externsValidation',
-    '--module', jsmodule_name_prefix + 'devtools__compatibility_js' + ':1',
     '--js', to_platform_path(path.join(devtools_frontend_path, 'devtools_compatibility.js'))
 ]
-devtools_js_compile_proc = popen(command)
+devtools_js_compile_proc = popen(devtools_js_compile_command)
 
 print 'Verifying JSDoc comments...'
 errors_found |= verify_jsdoc()
 (jsdoc_validator_proc, jsdoc_validator_file_list) = verify_jsdoc_extra()
 
 print
 
 (jsdoc_validator_out, _) = jsdoc_validator_proc.communicate()
 if jsdoc_validator_out:
     print ('JSDoc validator output:%s%s' % (os.linesep, jsdoc_validator_out))
     errors_found = True
 
 os.remove(jsdoc_validator_file_list.name)
 
-(module_compile_out, _) = modular_compiler_proc.communicate()
-print 'Modular compilation output:'
-
-start_module_regex = re.compile(r'^@@ START_MODULE:(.+) @@$')
-end_module_regex = re.compile(r'^@@ END_MODULE @@$')
-
-in_module = False
-skipped_modules = {}
-error_count = 0
-
-def skip_dependents(module_name):
-    for skipped_module in dependents_by_module_name.get(module_name, []):
-        skipped_modules[skipped_module] = True
-
-has_module_output = False
-
-# pylint: disable=E1103
-for line in module_compile_out.splitlines():
-    if not in_module:
-        match = re.search(start_module_regex, line)
-        if not match:
-            continue
-        in_module = True
-        has_module_output = True
-        module_error_count = 0
-        module_output = []
-        module_name = match.group(1)
-        skip_module = skipped_modules.get(module_name)
-        if skip_module:
-            skip_dependents(module_name)
-    else:
-        match = re.search(end_module_regex, line)
-        if not match:
-            if not skip_module:
-                module_output.append(line)
-                if has_errors(line):
-                    error_count += 1
-                    module_error_count += 1
-                    skip_dependents(module_name)
-            continue
-
-        in_module = False
-        if skip_module:
-            print 'Skipping module %s...' % module_name
-        elif not module_error_count:
-            print 'Module %s compiled successfully: %s' % (module_name, module_output[0])
-        else:
-            print 'Module %s compile failed: %s errors%s' % (module_name, module_error_count, os.linesep)
-            print os.linesep.join(module_output)
-
-if not has_module_output:
-    print module_compile_out
-
-if error_count:
-    print 'Total Closure errors: %d%s' % (error_count, os.linesep)
-    errors_found = True
-
 (devtools_js_compile_out, _) = devtools_js_compile_proc.communicate()
 print 'devtools_compatibility.js compilation output:%s' % os.linesep, devtools_js_compile_out
 errors_found |= has_errors(devtools_js_compile_out)
 
-os.remove(compiler_args_file.name)
+(frontend_compile_out, _) = frontend_compile_proc.communicate()
+print 'devtools frontend compilation output:'
+for line in frontend_compile_out.splitlines():
+    if "@@ START_MODULE" in line or "@@ END_MODULE" in line:
+        continue
+    print line
+errors_found |= has_errors(frontend_compile_out)
+
 os.remove(protocol_externs_file)
-shutil.rmtree(modules_dir, True)
+os.remove(compiler_args_file_path)
+shutil.rmtree(temp_devtools_path, True)
 
 if errors_found:
     print 'ERRORS DETECTED'
     sys.exit(1)
+print 'DONE - compiled without errors'