DevTools: Get rid of frontend_modules.json

Source/devtools/scripts/compile_frontend.py

 #!/usr/bin/env python
 # Copyright (c) 2012 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 12 matching lines @@
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import os
 import os.path as path
 import generate_protocol_externs
+import modular_build
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 try:
     import simplejson as json
 except ImportError:
     import json
 
 scripts_path = path.dirname(path.abspath(__file__))
 devtools_path = path.dirname(scripts_path)
 inspector_path = path.join(path.dirname(devtools_path), "core", "inspector")
 devtools_frontend_path = path.join(devtools_path, "front_end")
 global_externs_file = path.join(devtools_frontend_path, "externs.js")
 protocol_externs_file = path.join(devtools_frontend_path, "protocol_externs.js")
 webgl_rendering_context_idl_path = path.join(path.dirname(devtools_path), "core", "html", "canvas", "WebGLRenderingContextBase.idl")
 injected_script_source_name = path.join(inspector_path, "InjectedScriptSource.js")
 canvas_injected_script_source_name = path.join(inspector_path, "InjectedScriptCanvasModuleSource.js")
 closure_compiler_jar = path.join(scripts_path, "closure", "compiler.jar")
 closure_runner_jar = path.join(scripts_path, "compiler-runner", "closure-runner.jar")
 jsdoc_validator_jar = path.join(scripts_path, "jsdoc-validator", "jsdoc-validator.jar")
 java_exec = "java -Xms1024m -server -XX:+TieredCompilation"
 
 generate_protocol_externs.generate_protocol_externs(protocol_externs_file, path.join(devtools_path, "protocol.json"))
 
 jsmodule_name_prefix = "jsmodule_"
-frontend_modules_name = "frontend_modules.json"
 runtime_module_name = "_runtime"
 module_initializer_name = "_module.js"
 
 
 def log_error(message):
     print "ERROR: " + message
 
 
 def error_excepthook(exctype, value, traceback):
     print "ERROR:"
     sys.__excepthook__(exctype, value, traceback)
 sys.excepthook = error_excepthook
 
-try:
-    with open(path.join(scripts_path, frontend_modules_name), "rt") as js_modules_file:
-        modules = json.loads(js_modules_file.read())
-except:
-    log_error("Failed to read %s" % frontend_modules_name)
-    raise
+
+loader = modular_build.DescriptorLoader(devtools_frontend_path)
+descriptors = loader.load_application('devtools.json')
+modules_by_name = descriptors.modules
 
 type_checked_jsdoc_tags_list = ["param", "return", "type", "enum"]
 
 type_checked_jsdoc_tags_or = "|".join(type_checked_jsdoc_tags_list)
 
 # Basic regex for invalid JsDoc types: an object type name ([A-Z][A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
 invalid_type_regex = re.compile(r"@(?:" + type_checked_jsdoc_tags_or + r")\s*\{.*(?<![!?:.A-Za-z0-9])([A-Z][A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}")
 
 invalid_type_designator_regex = re.compile(r"@(?:" + type_checked_jsdoc_tags_or + r")\s*.*(?<![{: ])([?!])=?\}")
 
 error_warning_regex = re.compile(r"(?:WARNING|ERROR)")
 
 errors_found = False
 
 
 def run_in_shell(command_line):
     return subprocess.Popen(command_line, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
 
 
 def hasErrors(output):
     return re.search(error_warning_regex, output) != None
 
 
-def dump_all_checked_files():
-    files = {}
-    for module in modules:
-        for source in module["sources"]:
-            files[path.join(devtools_frontend_path, source)] = True
-    return files.keys()
-
-
 def verify_jsdoc_extra(additional_files):
-    return run_in_shell("%s -jar %s %s" % (java_exec, jsdoc_validator_jar, " ".join(dump_all_checked_files() + additional_files)))
+    return run_in_shell("%s -jar %s %s" % (java_exec, jsdoc_validator_jar, " ".join(descriptors.all_compiled_files() + additional_files)))
 
 
 def verify_jsdoc(additional_files):
     def file_list():
-        result = []
-        for module in modules:
-            for file_name in module["sources"]:
-                result.append(path.join(devtools_frontend_path, file_name))
-        for file in additional_files:
-            result.append(file)
-        return result
+        return descriptors.all_compiled_files() + additional_files
 
     errors_found = False
     for full_file_name in file_list():
         lineIndex = 0
         with open(full_file_name, "r") as sourceFile:
             for line in sourceFile:
                 line = line.rstrip()
                 lineIndex += 1
                 if not line:
                     continue
@@ skipping 28 matching lines @@
         sys.exit(1)
     print "Java executable: " + re.sub(r"\n$", "", javaPath)
 
 check_java_path()
 
 modules_dir = tempfile.mkdtemp()
 common_closure_args = " --summary_detail_level 3 --jscomp_error visibility --compilation_level SIMPLE_OPTIMIZATIONS --warning_level VERBOSE --language_in ECMASCRIPT5 --accept_const_keyword --module_output_path_prefix %s/" % modules_dir
 
 spawned_compiler_command = "%s -jar %s %s \\\n" % (java_exec, closure_compiler_jar, common_closure_args)
 
-modules_by_name = {}
 standalone_modules_by_name = {}
 dependents_by_module_name = {}
 
-for module in modules:
-    name = module["name"]
-    modules_by_name[name] = module
+for name in modules_by_name:
+    module = modules_by_name[name]
     if "standalone" in module:
         standalone_modules_by_name[name] = module
-    for dep in module["dependencies"]:
+    for dep in module.get("dependencies", []):
         list = dependents_by_module_name.get(dep)
         if not list:
             list = []
             dependents_by_module_name[dep] = list
         list.append(name)
 
 
 def verify_standalone_modules():
-    for module in modules:
-        for dependency in module["dependencies"]:
+    for name in modules_by_name:
+        for dependency in modules_by_name[name].get("dependencies", []):
             if dependency in standalone_modules_by_name:
-                log_error("Standalone module '%s' may not be present among the dependencies of '%s'" % (dependency, module["name"]))
+                log_error("Standalone module '%s' may not be present among the dependencies of '%s'" % (dependency, name))
                 errors_found = True
 
 verify_standalone_modules()
 
 
 def check_duplicate_files():
 
     def check_module(module, seen_files, seen_modules):
         name = module["name"]
         seen_modules[name] = True
-        for dep_name in module["dependencies"]:
+        for dep_name in module.get("dependencies", []):
             if not dep_name in seen_modules:
                 check_module(modules_by_name[dep_name], seen_files, seen_modules)
-        for source in module["sources"]:
+        for source in module.get("scripts", []):
             referencing_module = seen_files.get(source)
             if referencing_module:
                 log_error("Duplicate use of %s in '%s' (previously seen in '%s')" % (source, name, referencing_module))
             seen_files[source] = name
 
     for module_name in standalone_modules_by_name:
         check_module(standalone_modules_by_name[module_name], {}, {})
 
 print "Checking duplicate files across modules..."
 check_duplicate_files()
 
 
 def module_arg(module_name):
     return " --module " + jsmodule_name_prefix + module_name
 
 
 def dump_module(name, recursively, processed_modules):
     if name in processed_modules:
         return ""
     processed_modules[name] = True
     module = modules_by_name[name]
+    skipped_scripts = set(module.get("skip_compilation", []))
+
+    def filter_scripts():
+        result = []
+        for script in module.get("scripts", []):
+            if script in skipped_scripts:
+                continue
+            result.append(script)
+        return result
+
     command = ""
     if recursively:
-        for dependency in module["dependencies"]:
+        for dependency in module.get("dependencies", []):
             command += dump_module(dependency, recursively, processed_modules)
-    command += module_arg(module["name"]) + ":"
-    command += str(len(module["sources"]))
+    command += module_arg(name) + ":"
+    filtered_scripts = filter_scripts()
+    command += str(len(filtered_scripts))
     firstDependency = True
-    for dependency in module["dependencies"] + [runtime_module_name]:
+    for dependency in module.get("dependencies", []) + [runtime_module_name]:
         if firstDependency:
             command += ":"
         else:
             command += ","
         firstDependency = False
         command += jsmodule_name_prefix + dependency
-    for script in module["sources"]:
-        command += " --js " + path.join(devtools_frontend_path, script)
+    for script in filtered_scripts:
+        command += " --js " + path.join(devtools_frontend_path, name, script)
     return command
 
 print "Compiling frontend..."
 
 compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
-closure_runner_command = "%s -jar %s --compiler-args-file %s" % (java_exec, closure_runner_jar, compiler_args_file.name)
+try:
+    closure_runner_command = "%s -jar %s --compiler-args-file %s" % (java_exec, closure_runner_jar, compiler_args_file.name)
+    
+    for name in descriptors.sorted_modules():
+        closure_args = common_closure_args
+        closure_args += " --externs " + global_externs_file
+        closure_args += " --externs " + protocol_externs_file
+        runtime_module = module_arg(runtime_module_name) + ":1 --js " + path.join(devtools_frontend_path, "Runtime.js")
+        closure_args += runtime_module + dump_module(name, True, {})
+        compiler_args_file.write("%s %s\n" % (name, closure_args))
+finally:
+    compiler_args_file.close()
 
-for module in modules:
-    closure_args = common_closure_args
-    closure_args += " --externs " + global_externs_file
-    closure_args += " --externs " + protocol_externs_file
-    runtime_module = module_arg(runtime_module_name) + ":1 --js " + path.join(devtools_frontend_path, "Runtime.js")
-    closure_args += runtime_module + dump_module(module["name"], True, {})
-    compiler_args_file.write("%s %s\n" % (module["name"], closure_args))
-
-compiler_args_file.close()
 modular_compiler_proc = run_in_shell(closure_runner_command)
 
 
 def unclosure_injected_script(sourceFileName, outFileName):
     with open(sourceFileName, "r") as sourceFile:
         source = sourceFile.read()
 
     def replace_function(matchobj):
         return re.sub(r"@param", "param", matchobj.group(1) or "") + "\n//" + matchobj.group(2)
 
@@ skipping 111 matching lines @@
 errors_found |= hasErrors(validateInjectedScriptOut)
 
 if errors_found:
     print "ERRORS DETECTED"
 
 os.remove(injectedScriptSourceTmpFile)
 os.remove(injectedScriptCanvasModuleSourceTmpFile)
 os.remove(compiler_args_file.name)
 os.remove(protocol_externs_file)
 shutil.rmtree(modules_dir, True)