DevTools: speed up closure dependency checking

third_party/WebKit/Source/devtools/scripts/dependency_checker.py

+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import codecs
+import multiprocessing
+from os import path
+import re
+import shutil
+
+skip_modules = ['extensions']

[dgozman, 2016/12/20 07:02:18]

why?

[chenwilliam, 2016/12/20 20:23:35]

Good catch. I thought it was an issue because it was renaming local variables
with the same name as namespaces but it still compiles so I've removed it.

+
+scripts_path = path.dirname(path.abspath(__file__))
+devtools_path = path.dirname(scripts_path)
+devtools_frontend_path = path.join(devtools_path, 'front_end')
+
+
+class DependencyChecker(object):
+
+    def __init__(self, descriptors, temp_frontend_path):
+        self.descriptors = descriptors
+        self.temp_frontend_path = temp_frontend_path
+        self.module_descriptors = descriptors.modules
+        self.modules = set(self.descriptors.sorted_modules())
+        self._copy_frontend()
+
+    def enforce_dependencies(self):
+        arg_list = []
+        for module in self.modules:
+            if module in skip_modules:
+                continue
+            dependencies = set(self.descriptors.sorted_dependencies_closure(module))
+            excluded_modules = self.modules - {module} - dependencies - self._implicit_dependencies(module)
+            excluded_namespaces = [self._map_module_to_namespace(m) for m in excluded_modules]
+            file_paths = [path.join(self.temp_frontend_path, module, file_name)
+                          for file_name in self.descriptors.module_compiled_files(module)]
+            arg = {
+                'excluded_namespaces': excluded_namespaces,
+                'file_paths': file_paths,
+            }
+            arg_list.append(arg)
+        parallelize(poison_module, arg_list)
+
+    def _implicit_dependencies(self, module):
+        """Finds implicit dependencies for workers (which include single files from other modules)"""
+        implicit_dependencies = set()
+        for module_file in self.descriptors.module_compiled_files(module):
+            if "../" in module_file:
+                components = module_file.split('/')
+                implicit_dependencies.add(components[1])
+        return implicit_dependencies
+
+    def _copy_frontend(self):
+        if path.exists(self.temp_frontend_path):
+            shutil.rmtree(self.temp_frontend_path)
+        shutil.copytree(devtools_frontend_path, self.temp_frontend_path)
+
+    def _map_module_to_namespace(self, module):
+        if module == 'css_tracker':
+            return 'CSSTracker'
+        if module == 'ui_lazy':

[dgozman, 2016/12/20 07:02:18]

components_lazy?

[chenwilliam, 2016/12/20 20:23:35]

components_lazy is handled properly by the default logic (to_camel_case).

+            return 'UILazy'
+        if module == 'sdk' or module == 'ui':
+            return module.upper()
+        return self.to_camel_case(module)
+
+    def to_camel_case(self, snake_string):
+        components = snake_string.split('_')
+        return ''.join(x.title() for x in components)
+
+
+def poison_module(target):
+    excluded_namespaces = target['excluded_namespaces']
+    file_paths = target['file_paths']
+    for file_path in file_paths:
+        with codecs.open(file_path, 'r', 'utf-8') as file:
+            file_contents = file.read()
+        for namespace in excluded_namespaces:
+            file_contents = poison_contents_for_namespace(namespace, file_contents)
+        with codecs.open(file_path, 'w', 'utf-8') as file:
+            file.write(file_contents)
+
+
+def poison_contents_for_namespace(namespace, file_contents):
+    regex = r'([^.]\b)' + namespace + r'(\b[^:])'
+    replace = r'\1$$UndeclaredDependency_%s$$\2' % namespace
+    return re.sub(regex, replace, file_contents)
+
+
+def parallelize(fn, arg_list):
+    number_of_processes = min(multiprocessing.cpu_count(), 8)
+    pool = multiprocessing.Pool(number_of_processes)
+    pool.map(fn, arg_list)
+    pool.close()
+    pool.join()
+