Added dependency script injection and fixed other script injection.

src/caterpillar.py

Index: src/caterpillar.py
diff --git a/src/caterpillar.py b/src/caterpillar.py
index 57676db47791291bf7dcb6f0596e541005b2e2e0..cae9a6b54ea6ba3168a7b85e0c9a9e4580bc178c 100755
--- a/src/caterpillar.py
+++ b/src/caterpillar.py
@@ -68,6 +68,12 @@ MAX_CACHE_VERSION = 1000000
 # Where this file is located (so we can find resources).
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
+# Maps dependency managers to the folder they install dependencies into.
+DEPENDENCY_MANAGER_INSTALL_FOLDER = {
+  'bower': 'bower_components',
+  'npm': 'node_modules',
+}
+
 SW_FORMAT_STRING = """/**
  * @file Service worker generated by Caterpillar.
  */
@@ -333,14 +339,14 @@ def insert_todos_into_directory(output_dir):
         path = os.path.join(dirpath, filename)
         insert_todos_into_file(path)
 
-def generate_service_worker(output_dir, ca_manifest, polyfill_paths,
+def generate_service_worker(output_dir, ca_manifest, required_js_paths,
                             boilerplate_dir):
   """Generates code for a service worker.
 
   Args:
     output_dir: Directory of the web app that this service worker will run in.
     ca_manifest: Chrome App manifest dictionary.
-    polyfill_paths: List of paths to required polyfill scripts, relative to the
+    required_js_paths: List of paths to required scripts, relative to the
       boilerplate directory.
     boilerplate_dir: Caterpillar script directory within output web app.
 
@@ -370,11 +376,11 @@ def generate_service_worker(output_dir, ca_manifest, polyfill_paths,
 
   # The polyfills we get as input are relative to the boilerplate directory, but
   # the service worker is in the root directory, so we need to change the paths.
-  polyfills_paths = [os.path.join(boilerplate_dir, path)
-                     for path in polyfill_paths]
+  required_js_paths = [os.path.join(boilerplate_dir, path)
+                       for path in required_js_paths]
 
   background_scripts = ca_manifest['app']['background'].get('scripts', [])
-  for script in polyfill_paths + background_scripts:
+  for script in required_js_paths + background_scripts:
     logging.debug('Importing `%s` to the service worker.', script)
     sw_js += "importScripts('{}');\n".format(script)
 
@@ -392,14 +398,14 @@ def copy_script(script, directory):
   logging.debug('Writing `%s` to `%s`.', path, new_path)
   shutil.copyfile(path, new_path)
 
-def add_service_worker(output_dir, ca_manifest, polyfill_paths,
+def add_service_worker(output_dir, ca_manifest, required_js_paths,
                        boilerplate_dir):
   """Adds service worker scripts to a web app.
 
   Args:
     output_dir: Path to web app to add service worker scripts to.
     ca_manifest: Chrome App manifest dictionary.
-    polyfill_paths: List of paths to required polyfill scripts, relative to the
+    required_js_paths: List of paths to required scripts, relative to the
       boilerplate directory.
     boilerplate_dir: Caterpillar script directory within web app.
   """
@@ -409,7 +415,7 @@ def add_service_worker(output_dir, ca_manifest, polyfill_paths,
   copy_script(REGISTER_SCRIPT_NAME, boilerplate_path)
   copy_script(SW_STATIC_SCRIPT_NAME, boilerplate_path)
 
-  sw_js = generate_service_worker(output_dir, ca_manifest, polyfill_paths,
+  sw_js = generate_service_worker(output_dir, ca_manifest, required_js_paths,
                                   boilerplate_dir)
 
   # We can now write the service worker. Note that it must be in the root.
@@ -563,17 +569,40 @@ def convert_app(input_dir, output_dir, config, force=False):
       polyfillable.append(api)
     else:
       not_polyfillable.append(api)
+  if 'runtime' not in polyfillable:

[Matt Giuca, 2016/01/21 00:23:40]

why?

[Matthew Alger, 2016/01/21 00:29:30]

I don't want to add it twice, and polyfillable is a list.

[Matt Giuca, 2016/01/27 00:35:25]

I mean why do you special-case runtime here? (Why does it need to be added even
if it isn't used?)

[Matthew Alger, 2016/01/27 00:36:38]

All Chrome Apps errors are stored in chrome.runtime.lastError.

Perhaps a better solution would be to stub chrome.runtime in chrome.caterpillar,
so the special case isn't needed?

+    polyfillable.insert(0, 'runtime')
 
   logging.info('Polyfilled Chrome APIs: %s', ', '.join(polyfillable))
   logging.warning('Could not polyfill Chrome APIs: %s',
                   ', '.join(not_polyfillable))
 
+  # Read in the polyfill manifests and store their dependencies. We can't
+  # install them yet, though, since that has to be done after editing code or
+  # the dependencies will also be edited.
+  polyfill_manifests = polyfill_manifest.load_many(polyfillable)
+  dependencies = [dependency
+                  for manifest in polyfill_manifests.values()
+                  for dependency in manifest['dependencies']]
+
   # List of paths of static code to be copied from Caterpillar into the output
   # web app, relative to Caterpillar's JS source directory.
-  required_js_paths = [
+  required_always_paths = [

[Matt Giuca, 2016/01/21 00:23:40]

Why the name change?

[Matthew Alger, 2016/01/21 00:29:30]

Added three different lists with a similar name - calling this required_js_paths
would be misleading, since it's not all the JS. It's just the files that are
always required.

[Matt Giuca, 2016/01/27 00:35:25]

Acknowledged.

     'caterpillar.js',
     REGISTER_SCRIPT_NAME,
-  ] + polyfill_paths(polyfillable)
+  ]
+
+  # The dependencies and polyfills are also requirements, but we need to handle
+  # them differently, so they're split up into two lists.
+  required_dependency_paths = []
+  for dependency in dependencies:
+    # Note that dependencies are installed into the root, but we need paths
+    # relative to Caterpillar's boilerplate directory.
+    dependency_path = os.path.join('..',
+        DEPENDENCY_MANAGER_INSTALL_FOLDER[dependency['manager']],
+        dependency['name'], dependency['path'])
+    required_dependency_paths.append(dependency_path)
+
+  required_polyfill_paths = polyfill_paths(polyfillable)
 
   # Read in and check the manifest file.
   try:
@@ -602,26 +631,23 @@ def convert_app(input_dir, output_dir, config, force=False):
   # This is adding TODOs, injecting tags, etc. - anything that involves editing
   # user code directly. This must be done before the static code is copied
   # across, or the polyfills will have TODOs added to them.
-  edit_code(output_dir, required_js_paths, ca_manifest, config)
+  required_script_paths = (required_always_paths + required_dependency_paths
+                           + required_polyfill_paths)  # Order significant here.

[Matt Giuca, 2016/01/21 00:23:40]

nit: Don't squeeze this comment here; put it above.

[Matthew Alger, 2016/01/21 00:29:30]

Done.

+  edit_code(output_dir, required_script_paths, ca_manifest, config)
 
   # We want the static SW file to be copied in too, so we add it here.
   # We have to add it after edit_code or it would be included in the HTML, but
   # this is service worker-only code, and shouldn't be included there.
-  required_js_paths.append(SW_STATIC_SCRIPT_NAME)
+  required_always_paths.append(SW_STATIC_SCRIPT_NAME)
 
   # Copy static code from Caterpillar into the output web app.
   # This must be done before the service worker is generated, or these files
   # will not be cached.
-  copy_static_code(required_js_paths, output_dir, boilerplate_dir)
+  required_static_paths = required_always_paths + required_polyfill_paths
+  copy_static_code(required_static_paths, output_dir, boilerplate_dir)
 
-  # Read in the polyfill manifests and install polyfill dependencies.
-  # This must be done after editing code (or the dependencies will also be
-  # edited) and before the service worker is generated (or the dependencies
-  # won't be cached).
-  polyfill_manifests = polyfill_manifest.load_many(polyfillable)
-  dependencies = [dependency
-                  for manifest in polyfill_manifests.values()
-                  for dependency in manifest['dependencies']]
+  # Install the polyfill dependencies. This must be done before the service
+  # worker is generated, or the dependencies won't be cached.
   try:
     install_dependencies(dependencies, output_dir)
   except ValueError as e:
@@ -629,7 +655,8 @@ def convert_app(input_dir, output_dir, config, force=False):
     return
 
   # Generate and write a service worker.
-  add_service_worker(output_dir, ca_manifest, polyfill_paths(polyfillable),
+  required_sw_paths = required_dependency_paths + required_polyfill_paths
+  add_service_worker(output_dir, ca_manifest, required_sw_paths,
                      boilerplate_dir)
 
   logging.info('Conversion complete.')