DevTools: Refactor build script to copy module files in debug_devtools mode

Source/devtools/scripts/concatenate_application_code.py

Index: Source/devtools/scripts/concatenate_application_code.py
diff --git a/Source/devtools/scripts/concatenate_application_code.py b/Source/devtools/scripts/concatenate_application_code.py
index 978dd061fd3f7d896e93082abc71f4cdcbcb0fe2..2418b92292c255da0bbce3fd1b53bdc27fc28fca 100755
--- a/Source/devtools/scripts/concatenate_application_code.py
+++ b/Source/devtools/scripts/concatenate_application_code.py
@@ -5,9 +5,14 @@
 # found in the LICENSE file.
 
 """
-Concatenates autostart modules, application modules' module.json descriptors,
-and the application loader into a single script.
-Also concatenates all workers' dependencies into individual worker loader scripts.
+Release:
+  - Concatenates autostart modules, application modules' module.json descriptors,
+    and the application loader into a single script.
+  - Concatenates all workers' dependencies into individual worker loader scripts.
+  - Builds app.html referencing the application script.
+Debug:
+  - Copies the module directories into their destinations.
+  - Copies app.html as-is.
 """
 
 from cStringIO import StringIO
@@ -17,6 +22,7 @@ import copy
 import modular_build
 import os
 import re
+import shutil
 import sys
 
 try:
@@ -34,103 +40,191 @@ sys.path.append(rjsmin_path)
 import rjsmin
 
 
-def minify_if_needed(javascript, minify):
-    return rjsmin.jsmin(javascript) if minify else javascript
+def minify_js(javascript):
+    return rjsmin.jsmin(javascript)
 
 
 def concatenated_module_filename(module_name, output_dir):
     return path.join(output_dir, module_name + '_module.js')
 
 
-def concatenate_autostart_modules(descriptors, application_dir, output_dir, output):
-    non_autostart = set()
-    sorted_module_names = descriptors.sorted_modules()
-    for name in sorted_module_names:
-        desc = descriptors.modules[name]
-        name = desc['name']
-        type = descriptors.application[name].get('type')
-        if type == 'autostart':
-            deps = set(desc.get('dependencies', []))
-            non_autostart_deps = deps & non_autostart
-            if len(non_autostart_deps):
-                bail_error('Non-autostart dependencies specified for the autostarted module "%s": %s' % (name, non_autostart_deps))
-            output.write('\n/* Module %s */\n' % name)
-            modular_build.concatenate_scripts(desc.get('scripts'), path.join(application_dir, name), output_dir, output)
-        elif type != 'worker':
-            non_autostart.add(name)
-
-
-def concatenate_application_script(application_name, descriptors, application_dir, output_dir, minify):
-    application_loader_name = application_name + '.js'
-    output = StringIO()
-    runtime_contents = read_file(path.join(application_dir, 'Runtime.js'))
-    runtime_contents = re.sub('var allDescriptors = \[\];', 'var allDescriptors = %s;' % release_module_descriptors(descriptors.modules).replace("\\", "\\\\"), runtime_contents, 1)
-    output.write('/* Runtime.js */\n')
-    output.write(runtime_contents)
-    output.write('\n/* Autostart modules */\n')
-    concatenate_autostart_modules(descriptors, application_dir, output_dir, output)
-    output.write('/* Application descriptor %s */\n' % (application_name + '.json'))
-    output.write('applicationDescriptor = ')
-    output.write(descriptors.application_json)
-    output.write(';\n/* Application loader */\n')
-    output.write(read_file(path.join(application_dir, application_loader_name)))
-
-    write_file(path.join(output_dir, application_loader_name), minify_if_needed(output.getvalue(), minify))
-    output.close()
-
-
-def concatenate_dynamic_module(module_name, descriptors, application_dir, output_dir, minify):
-    scripts = descriptors.modules[module_name].get('scripts')
-    if not scripts:
-        return
-    module_dir = path.join(application_dir, module_name)
-    output = StringIO()
-    modular_build.concatenate_scripts(scripts, module_dir, output_dir, output)
-    output_file_path = concatenated_module_filename(module_name, output_dir)
-    write_file(output_file_path, minify_if_needed(output.getvalue(), minify))
-    output.close()
-
-
-def concatenate_worker(module_name, descriptors, application_dir, output_dir, minify):
-    descriptor = descriptors.modules[module_name]
-    scripts = descriptor.get('scripts')
-    if not scripts:
-        return
-    worker_dir = path.join(application_dir, module_name)
-    output_file_path = concatenated_module_filename(module_name, output_dir)
-
-    output = StringIO()
-    output.write('/* Worker %s */\n' % module_name)
-    dependencies = descriptors.sorted_dependencies_closure(module_name)
-    dep_descriptors = []
-    for dep_name in dependencies:
-        dep_descriptor = descriptors.modules[dep_name]
-        dep_descriptors.append(dep_descriptor)
-        scripts = dep_descriptor.get('scripts')
-        if scripts:
-            output.write('\n/* Module %s */\n' % dep_name)
-            modular_build.concatenate_scripts(scripts, path.join(application_dir, dep_name), output_dir, output)
-
-    write_file(output_file_path, minify_if_needed(output.getvalue(), minify))
-    output.close()
-
-
-def release_module_descriptors(module_descriptors):
-    result = []
-    for name in module_descriptors:
-        module = copy.copy(module_descriptors[name])
-        # Clear scripts, as they are not used at runtime
-        # (only the fact of their presence is important).
-        if module.get('scripts'):
-            module['scripts'] = []
-        result.append(module)
-    return json.dumps(result)
-
-
-def build_application(application_name, loader, application_dir, output_dir, minify):
+def hardlink_or_copy_dir(src, dest):
+    if path.exists(dest):
+        shutil.rmtree(dest)
+    os.mkdir(dest)
+    for root, dirs, files in os.walk(src):
+        for name in files:
+            src_name = path.join(src, name)

[dgozman, 2014/10/13 13:01:52]

src_name = path.join(root, name)

[apavlov, 2014/10/13 15:02:21]

Done.

+            dest_name = path.join(dest, name)

[dgozman, 2014/10/13 13:01:52]

dest_name = path.join(dest, path.relpath(root, src))

[apavlov, 2014/10/13 15:02:21]

dest_name = path.join(dest, path.join(path.relpath(src, root), name))
actually, since we have to specify the target name, not the directory name

+            if hasattr(os, 'link'):
+                os.link(src_name, dest_name)
+            else:
+                shutil.copy(src_name, dest_name)
+        for name in dirs:
+            hardlink_or_copy_dir(path.join(src, name), path.join(dest, name))

[dgozman, 2014/10/13 13:01:52]

os.walk is already recursive.

[apavlov, 2014/10/13 15:02:21]

Acknowledged.

+
+
+class AppBuilder:

[dgozman, 2014/10/13 13:01:52]

You should probably have |build_app| function defined in this base class?

[apavlov, 2014/10/13 15:02:21]

We'd like to get a clear error thrown in case build_app() is not defined in the
concrete subclass, and I don't feel like mangling around with the ugly ABC
support...

+    def __init__(self, application_name, descriptors, application_dir, output_dir):
+        self.application_name = application_name
+        self.descriptors = descriptors
+        self.application_dir = application_dir
+        self.output_dir = output_dir
+
+
+# Outputs:
+#   app_name.html

[dgozman, 2014/10/13 13:01:52]

Maybe <app_name>.html and <app_name>.js ?

[apavlov, 2014/10/13 15:02:21]

Ugh, it's a bit outdated patch... I've got this locally

+#   app_name.js
+#   <module_name>_module.js
+class ReleaseBuilder(AppBuilder):
+    def __init__(self, application_name, descriptors, application_dir, output_dir):
+        AppBuilder.__init__(self, application_name, descriptors, application_dir, output_dir)
+
+    def build_app(self):
+        self._build_html()
+        self._build_app_script()
+        for module in filter(lambda desc: not desc.get('type'), self.descriptors.application.values()):
+            self._concatenate_dynamic_module(module['name'])
+        for module in filter(lambda desc: desc.get('type') == 'worker', self.descriptors.application.values()):
+            self._concatenate_worker(module['name'])
+
+    def _build_html(self):
+        html_name = self.application_name + '.html'
+        output = StringIO()
+        with open(path.join(self.application_dir, html_name), 'r') as app_input_html:

[dgozman, 2014/10/13 13:01:52]

This whole processing tells us that we should generate html from template, and
not have multiple *.html files.

[apavlov, 2014/10/13 15:02:21]

This is just code moved from generate_devtools_html.py (sorry, forgot to delete
the file)

[dgozman, 2014/10/14 09:21:08]

I see, but my point about generating is still valid.

[apavlov, 2014/10/14 10:02:58]

I'm not questioning this, I'm just saying that we should confine the scope of
this patch to simple code motion and rewriting the multiple 'copies' rules in
Python.

+            for line in app_input_html:
+                if '<script ' in line or '<link ' in line:
+                    continue
+                if '</head>' in line:
+                    output.write(self._generate_include_tag("%s.css" % self.application_name))
+                    output.write(self._generate_include_tag("%s.js" % self.application_name))
+                output.write(line)
+
+        write_file(path.join(self.output_dir, html_name), output.getvalue())
+        output.close()
+
+    def _build_app_script(self):
+        script_name = self.application_name + '.js'
+        output = StringIO()
+        self._concatenate_application_script(output)
+        write_file(path.join(self.output_dir, script_name), output.getvalue())
+        output.close()
+
+    def _generate_include_tag(self, resource_path):
+        if (resource_path.endswith('.js')):
+            return '    <script type="text/javascript" src="%s"></script>\n' % resource_path
+        elif (resource_path.endswith('.css')):
+            return '    <link rel="stylesheet" type="text/css" href="%s">\n' % resource_path
+        else:
+            assert resource_path
+
+    def _release_module_descriptors(self):
+        module_descriptors = self.descriptors.modules
+        result = []
+        for name in module_descriptors:
+            module = copy.copy(module_descriptors[name])
+            # Clear scripts, as they are not used at runtime
+            # (only the fact of their presence is important).
+            if module.get('scripts'):
+                module['scripts'] = []
+            result.append(module)
+        return json.dumps(result)
+
+    def _concatenate_autostart_modules(self, output):
+        non_autostart = set()
+        sorted_module_names = self.descriptors.sorted_modules()
+        for name in sorted_module_names:
+            desc = self.descriptors.modules[name]
+            name = desc['name']
+            type = self.descriptors.application[name].get('type')
+            if type == 'autostart':
+                deps = set(desc.get('dependencies', []))
+                non_autostart_deps = deps & non_autostart
+                if len(non_autostart_deps):
+                    bail_error('Non-autostart dependencies specified for the autostarted module "%s": %s' % (name, non_autostart_deps))
+                output.write('\n/* Module %s */\n' % name)
+                modular_build.concatenate_scripts(desc.get('scripts'), path.join(self.application_dir, name), self.output_dir, output)
+            elif type != 'worker':
+                non_autostart.add(name)
+
+    def _concatenate_application_script(self, output):
+        application_loader_name = self.application_name + '.js'
+        runtime_contents = read_file(path.join(self.application_dir, 'Runtime.js'))
+        runtime_contents = re.sub('var allDescriptors = \[\];', 'var allDescriptors = %s;' % self._release_module_descriptors().replace("\\", "\\\\"), runtime_contents, 1)
+        output.write('/* Runtime.js */\n')
+        output.write(runtime_contents)
+        output.write('\n/* Autostart modules */\n')
+        self._concatenate_autostart_modules(output)
+        output.write('/* Application descriptor %s */\n' % (self.application_name + '.json'))
+        output.write('applicationDescriptor = ')
+        output.write(self.descriptors.application_json)
+        output.write('\n/* Application loader */\n')
+        output.write(read_file(path.join(self.application_dir, application_loader_name)))
+
+    def _concatenate_dynamic_module(self, module_name):
+        module = self.descriptors.modules[module_name]
+        scripts = module.get('scripts')
+        if not scripts:
+            return
+        module_dir = path.join(self.application_dir, module_name)
+        output = StringIO()
+        modular_build.concatenate_scripts(scripts, module_dir, self.output_dir, output)
+        output_file_path = concatenated_module_filename(module_name, self.output_dir)
+        write_file(output_file_path, minify_js(output.getvalue()))
+        output.close()
+
+    def _concatenate_worker(self, module_name):
+        descriptor = self.descriptors.modules[module_name]
+        scripts = descriptor.get('scripts')
+        if not scripts:
+            return
+
+        output = StringIO()
+        output.write('/* Worker %s */\n' % module_name)
+        dep_descriptors = []
+        for dep_name in self.descriptors.sorted_dependencies_closure(module_name):
+            dep_descriptor = self.descriptors.modules[dep_name]
+            dep_descriptors.append(dep_descriptor)
+            scripts = dep_descriptor.get('scripts')
+            if scripts:
+                output.write('\n/* Module %s */\n' % dep_name)
+                modular_build.concatenate_scripts(scripts, path.join(self.application_dir, dep_name), self.output_dir, output)
+
+        output_file_path = concatenated_module_filename(module_name, self.output_dir)
+        write_file(output_file_path, minify_js(output.getvalue()))
+        output.close()
+
+
+# Outputs:
+#   app_name.html as-is

[dgozman, 2014/10/13 13:01:52]

<app_name>.html ?

[apavlov, 2014/10/13 15:02:21]

Done.

+#   <module_name>/<all_files>
+class DebugBuilder(AppBuilder):
+    def __init__(self, application_name, descriptors, application_dir, output_dir):
+        AppBuilder.__init__(self, application_name, descriptors, application_dir, output_dir)
+
+    def build_app(self):
+        self._build_html()
+        for module_name in self.descriptors.modules:
+            module = self.descriptors.modules[module_name]
+            input_module_dir = path.join(self.application_dir, module_name)
+            output_module_dir = path.join(self.output_dir, module_name)
+            hardlink_or_copy_dir(input_module_dir, output_module_dir)
+
+    def _build_html(self):
+        html_name = self.application_name + '.html'
+        output = StringIO()
+        with open(path.join(self.application_dir, html_name), 'r') as app_input_html:
+            for line in app_input_html:
+                output.write(line)
+
+        write_file(path.join(self.output_dir, html_name), output.getvalue())

[dgozman, 2014/10/13 13:01:52]

Why not shutil.copy?

[apavlov, 2014/10/13 15:02:21]

I will add something else into here in a subsequent patch, so this is to set up
the required methods in advance.

[dgozman, 2014/10/14 09:21:08]

Maybe change shutil.copy to this processing in the follow-up patch itself?

[apavlov, 2014/10/14 10:02:58]

Done, even though exactly this code used to run when building debug_devtools
HTML, and I was aiming for a plain refactoring in this part.

+        output.close()
+
+
+def build_application(application_name, loader, application_dir, output_dir, release_mode):
     descriptors = loader.load_application(application_name + '.json')
-    concatenate_application_script(application_name, descriptors, application_dir, output_dir, minify)
-    for module in filter(lambda desc: not desc.get('type'), descriptors.application.values()):
-        concatenate_dynamic_module(module['name'], descriptors, application_dir, output_dir, minify)
-    for module in filter(lambda desc: desc.get('type') == 'worker', descriptors.application.values()):
-        concatenate_worker(module['name'], descriptors, application_dir, output_dir, minify)
+    if release_mode:
+        builder = ReleaseBuilder(application_name, descriptors, application_dir, output_dir)
+    else:
+        builder = DebugBuilder(application_name, descriptors, application_dir, output_dir)
+    builder.build_app()