DevTools: speed up closure dependency checking

third_party/WebKit/Source/devtools/scripts/dependency_preprocessor.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import codecs
+import multiprocessing
+from os import path
+import re
+import shutil
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+special_case_namespaces_path = path.join(path.dirname(path.abspath(__file__)), 'special_case_namespaces.json')
+
+
+class DependencyPreprocessor(object):
+
+    def __init__(self, descriptors, temp_frontend_path, devtools_frontend_path):
+        self.descriptors = descriptors
+        self.temp_frontend_path = temp_frontend_path
+        self.module_descriptors = descriptors.modules
+        self.modules = set(self.descriptors.sorted_modules())
+        shutil.copytree(devtools_frontend_path, self.temp_frontend_path)
+        with open(special_case_namespaces_path) as json_file:
+            self._special_case_namespaces = json.load(json_file)
+
+    def enforce_dependencies(self):
+        arg_list = []
+        for module in self.modules:
+            dependencies = set(self.descriptors.sorted_dependencies_closure(module))
+            excluded_modules = self.modules - {module} - dependencies - self._transitive_implicit_dependencies(module)
+            excluded_namespaces = [self._map_module_to_namespace(m) for m in excluded_modules]
+            file_paths = [path.join(self.temp_frontend_path, module, file_name)
+                          for file_name in self.descriptors.module_compiled_files(module)]
+            arg = {
+                'excluded_namespaces': excluded_namespaces,
+                'file_paths': file_paths,
+            }
+            arg_list.append(arg)
+        parallelize(poison_module, arg_list)
+
+    def _transitive_implicit_dependencies(self, module):
+        """Finds implicit dependencies for workers (which include single files from other modules)"""
+        explicit_dependencies = self.descriptors.sorted_dependencies_closure(module)
+        implicit_dependencies = set()
+        for explicit_dependency in explicit_dependencies:
+            implicit_dependencies |= self._implicit_dependencies_for_module(explicit_dependency)
+        return implicit_dependencies
+
+    def _implicit_dependencies_for_module(self, module):
+        implicit_dependencies = set()
+        for module_file in self.descriptors.module_compiled_files(module):
+            if "../" in module_file:
+                components = module_file.split('/')
+                implicit_dependencies.add(components[1])
+        return implicit_dependencies
+
+    def _map_module_to_namespace(self, module):
+        return self._special_case_namespaces.get(module, self._to_camel_case(module))
+
+    def _to_camel_case(self, snake_string):
+        components = snake_string.split('_')
+        return ''.join(x.title() for x in components)
+
+
+def poison_module(target):
+    excluded_namespaces = target['excluded_namespaces']
+    file_paths = target['file_paths']
+    for file_path in file_paths:
+        with codecs.open(file_path, 'r', 'utf-8') as file:
+            file_contents = file.read()
+        for namespace in excluded_namespaces:
+            file_contents = poison_contents_for_namespace(file_contents, namespace)
+        with codecs.open(file_path, 'w', 'utf-8') as file:
+            file.write(file_contents)
+
+
+def poison_contents_for_namespace(file_contents, namespace):
+    regex = r'([^.]\b)' + namespace + r'(\b[^:])'

[dgozman, 2017/01/10 21:47:35]

Let's have a comment that this should actually be
[^.]\s*\b     and      \b\s*[^:]
but we rely on clang-format never producing this.


We can do a single run, maybe it's faster?
[^.]\b(SDK|UI|Console)\b[^:]
\1$UndeclaredDependency_\2$$\3

[chenwilliam, 2017/01/10 23:08:56]

Done. Changed to single run (~1 sec faster)

+    replace = r'\1$$UndeclaredDependency_%s$$\2' % namespace
+    return re.sub(regex, replace, file_contents)
+
+
+def parallelize(fn, arg_list):
+    number_of_processes = min(multiprocessing.cpu_count(), 8)
+    pool = multiprocessing.Pool(number_of_processes)
+    pool.map(fn, arg_list)
+    pool.close()
+    pool.join()