Added dependency script injection and fixed other script injection.

src/caterpillar.py

 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 
 # Copyright 2015 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
@@ skipping 50 matching lines @@
 
 # Name of the service worker static script.
 SW_STATIC_SCRIPT_NAME = 'sw_static.js'
 
 # Largest number that the cache version can be.
 MAX_CACHE_VERSION = 1000000
 
 # Where this file is located (so we can find resources).
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
+# Maps dependency managers to the folder they install dependencies into.
+DEPENDENCY_MANAGER_INSTALL_FOLDER = {
+  'bower': 'bower_components',
+  'npm': 'node_modules',
+}
+
 SW_FORMAT_STRING = """/**
  * @file Service worker generated by Caterpillar.
  */
 
 /**
  * @const Current cache version.
  *
  * Increment this to force cache to clear.
  */
 var CACHE_VERSION = {cache_version};
@@ skipping 245 matching lines @@
     output_dir: Directory of the web app to insert TODOs into.
   """
   logging.debug('Inserting TODOs.')
   dirwalk = os.walk(output_dir)
   for (dirpath, _, filenames) in dirwalk:
     for filename in filenames:
       if filename.endswith('.js'):
         path = os.path.join(dirpath, filename)
         insert_todos_into_file(path)
 
-def generate_service_worker(output_dir, ca_manifest, polyfill_paths,
+def generate_service_worker(output_dir, ca_manifest, required_js_paths,
                             boilerplate_dir):
   """Generates code for a service worker.
 
   Args:
     output_dir: Directory of the web app that this service worker will run in.
     ca_manifest: Chrome App manifest dictionary.
-    polyfill_paths: List of paths to required polyfill scripts, relative to the
+    required_js_paths: List of paths to required scripts, relative to the
       boilerplate directory.
     boilerplate_dir: Caterpillar script directory within output web app.
 
   Returns:
     JavaScript string.
   """
   # Get the paths of files we will cache.
   all_filepaths = []
   logging.debug('Looking for files to cache.')
   dirwalk = os.walk(output_dir)
   for (dirpath, _, filenames) in dirwalk:
     # Add the relative file paths of each file to the filepaths list.
     all_filepaths.extend(
       os.path.relpath(os.path.join(dirpath, filename), output_dir)
       for filename in filenames)
   logging.debug('Cached files:\n\t%s', '\n\t'.join(all_filepaths))
   # Format the file paths as JavaScript strings.
   all_filepaths = ["'{}'".format(fp) for fp in all_filepaths]
 
   logging.debug('Generating service worker.')
 
   sw_js = SW_FORMAT_STRING.format(
     cache_version=random.randrange(MAX_CACHE_VERSION),
     joined_filepaths=',\n  '.join(all_filepaths),
     boilerplate_dir=boilerplate_dir
   )
 
   # The polyfills we get as input are relative to the boilerplate directory, but
   # the service worker is in the root directory, so we need to change the paths.
-  polyfills_paths = [os.path.join(boilerplate_dir, path)
-                     for path in polyfill_paths]
+  required_js_paths = [os.path.join(boilerplate_dir, path)
+                       for path in required_js_paths]
 
   background_scripts = ca_manifest['app']['background'].get('scripts', [])
-  for script in polyfill_paths + background_scripts:
+  for script in required_js_paths + background_scripts:
     logging.debug('Importing `%s` to the service worker.', script)
     sw_js += "importScripts('{}');\n".format(script)
 
   return sw_js
 
 def copy_script(script, directory):
   """Copies a script from Caterpillar into the given directory.
 
   Args:
     script: Caterpillar JavaScript filename.
     directory: Path to directory.
   """
   path = os.path.join(SCRIPT_DIR, 'js', script)
   new_path = os.path.join(directory, script)
   logging.debug('Writing `%s` to `%s`.', path, new_path)
   shutil.copyfile(path, new_path)
 
-def add_service_worker(output_dir, ca_manifest, polyfill_paths,
+def add_service_worker(output_dir, ca_manifest, required_js_paths,
                        boilerplate_dir):
   """Adds service worker scripts to a web app.
 
   Args:
     output_dir: Path to web app to add service worker scripts to.
     ca_manifest: Chrome App manifest dictionary.
-    polyfill_paths: List of paths to required polyfill scripts, relative to the
+    required_js_paths: List of paths to required scripts, relative to the
       boilerplate directory.
     boilerplate_dir: Caterpillar script directory within web app.
   """
   # We have to copy the other scripts before we generate the service worker
   # caching script, or else they won't be cached.
   boilerplate_path = os.path.join(output_dir, boilerplate_dir)
   copy_script(REGISTER_SCRIPT_NAME, boilerplate_path)
   copy_script(SW_STATIC_SCRIPT_NAME, boilerplate_path)
 
-  sw_js = generate_service_worker(output_dir, ca_manifest, polyfill_paths,
+  sw_js = generate_service_worker(output_dir, ca_manifest, required_js_paths,
                                   boilerplate_dir)
 
   # We can now write the service worker. Note that it must be in the root.
   sw_path = os.path.join(output_dir, SW_SCRIPT_NAME)
   logging.debug('Writing service worker to `%s`.', sw_path)
   with open(sw_path, 'w') as sw_file:
     sw_file.write(sw_js)
 
 class InstallationError(Exception):
   """Exception raised when a dependency fails to install."""
@@ skipping 133 matching lines @@
     logging.info('Found Chrome APIs: %s', ', '.join(apis))
 
   # Determine which Chrome Apps APIs can be polyfilled, and which cannot.
   polyfillable = []
   not_polyfillable = []
   for api in apis:
     if api in POLYFILLS:
       polyfillable.append(api)
     else:
       not_polyfillable.append(api)
+  if 'runtime' not in polyfillable:

[Matt Giuca, 2016/01/21 00:23:40]

why?

[Matthew Alger, 2016/01/21 00:29:30]

I don't want to add it twice, and polyfillable is a list.

[Matt Giuca, 2016/01/27 00:35:25]

I mean why do you special-case runtime here? (Why does it need to be added even
if it isn't used?)

[Matthew Alger, 2016/01/27 00:36:38]

All Chrome Apps errors are stored in chrome.runtime.lastError.

Perhaps a better solution would be to stub chrome.runtime in chrome.caterpillar,
so the special case isn't needed?

+    polyfillable.insert(0, 'runtime')
 
   logging.info('Polyfilled Chrome APIs: %s', ', '.join(polyfillable))
   logging.warning('Could not polyfill Chrome APIs: %s',
                   ', '.join(not_polyfillable))
 
+  # Read in the polyfill manifests and store their dependencies. We can't
+  # install them yet, though, since that has to be done after editing code or
+  # the dependencies will also be edited.
+  polyfill_manifests = polyfill_manifest.load_many(polyfillable)
+  dependencies = [dependency
+                  for manifest in polyfill_manifests.values()
+                  for dependency in manifest['dependencies']]
+
   # List of paths of static code to be copied from Caterpillar into the output
   # web app, relative to Caterpillar's JS source directory.
-  required_js_paths = [
+  required_always_paths = [

[Matt Giuca, 2016/01/21 00:23:40]

Why the name change?

[Matthew Alger, 2016/01/21 00:29:30]

Added three different lists with a similar name - calling this required_js_paths
would be misleading, since it's not all the JS. It's just the files that are
always required.

[Matt Giuca, 2016/01/27 00:35:25]

Acknowledged.

     'caterpillar.js',
     REGISTER_SCRIPT_NAME,
-  ] + polyfill_paths(polyfillable)
+  ]
+
+  # The dependencies and polyfills are also requirements, but we need to handle
+  # them differently, so they're split up into two lists.
+  required_dependency_paths = []
+  for dependency in dependencies:
+    # Note that dependencies are installed into the root, but we need paths
+    # relative to Caterpillar's boilerplate directory.
+    dependency_path = os.path.join('..',
+        DEPENDENCY_MANAGER_INSTALL_FOLDER[dependency['manager']],
+        dependency['name'], dependency['path'])
+    required_dependency_paths.append(dependency_path)
+
+  required_polyfill_paths = polyfill_paths(polyfillable)
 
   # Read in and check the manifest file.
   try:
     ca_manifest = chrome_app.manifest.get(input_dir)
     chrome_app.manifest.verify(ca_manifest)
   except ValueError as e:
     logging.error(e.message)
     return
 
   # TODO(alger): Identify background scripts and determine start_url.
   start_url = config['start_url']
   logging.info('Got start URL from config file: `%s`', start_url)
 
   # Generate a progressive web app manifest.
   pwa_manifest = generate_web_manifest(ca_manifest, start_url)
   pwa_manifest_path = os.path.join(output_dir, PWA_MANIFEST_FILENAME)
   with open(pwa_manifest_path, 'w') as pwa_manifest_file:
     json.dump(pwa_manifest, pwa_manifest_file, indent=4, sort_keys=True)
   logging.debug('Wrote `%s` to `%s`.', PWA_MANIFEST_FILENAME, pwa_manifest_path)
 
   # Remove unnecessary files from the output web app. This must be done before
   # the service worker is generated, or these files will be cached.
   cleanup_output_dir(output_dir)
 
   # Edit the HTML and JS code of the output web app.
   # This is adding TODOs, injecting tags, etc. - anything that involves editing
   # user code directly. This must be done before the static code is copied
   # across, or the polyfills will have TODOs added to them.
-  edit_code(output_dir, required_js_paths, ca_manifest, config)
+  required_script_paths = (required_always_paths + required_dependency_paths
+                           + required_polyfill_paths)  # Order significant here.

[Matt Giuca, 2016/01/21 00:23:40]

nit: Don't squeeze this comment here; put it above.

[Matthew Alger, 2016/01/21 00:29:30]

Done.

+  edit_code(output_dir, required_script_paths, ca_manifest, config)
 
   # We want the static SW file to be copied in too, so we add it here.
   # We have to add it after edit_code or it would be included in the HTML, but
   # this is service worker-only code, and shouldn't be included there.
-  required_js_paths.append(SW_STATIC_SCRIPT_NAME)
+  required_always_paths.append(SW_STATIC_SCRIPT_NAME)
 
   # Copy static code from Caterpillar into the output web app.
   # This must be done before the service worker is generated, or these files
   # will not be cached.
-  copy_static_code(required_js_paths, output_dir, boilerplate_dir)
+  required_static_paths = required_always_paths + required_polyfill_paths
+  copy_static_code(required_static_paths, output_dir, boilerplate_dir)
 
-  # Read in the polyfill manifests and install polyfill dependencies.
-  # This must be done after editing code (or the dependencies will also be
-  # edited) and before the service worker is generated (or the dependencies
-  # won't be cached).
-  polyfill_manifests = polyfill_manifest.load_many(polyfillable)
-  dependencies = [dependency
-                  for manifest in polyfill_manifests.values()
-                  for dependency in manifest['dependencies']]
+  # Install the polyfill dependencies. This must be done before the service
+  # worker is generated, or the dependencies won't be cached.
   try:
     install_dependencies(dependencies, output_dir)
   except ValueError as e:
     logging.error(e.message)
     return
 
   # Generate and write a service worker.
-  add_service_worker(output_dir, ca_manifest, polyfill_paths(polyfillable),
+  required_sw_paths = required_dependency_paths + required_polyfill_paths
+  add_service_worker(output_dir, ca_manifest, required_sw_paths,
                      boilerplate_dir)
 
   logging.info('Conversion complete.')
 
 class Formatter(logging.Formatter):
   """Caterpillar logging formatter.
 
   Adds color to the logged information.
   """
   def format(self, record):
@@ skipping 50 matching lines @@
   # Main program.
   if args.mode == 'config':
     configuration.generate_and_save(args.output, args.interactive)
 
   elif args.mode == 'convert':
     config = configuration.load(args.config)
     convert_app(args.input, args.output, config, args.force)
 
 if __name__ == '__main__':
   sys.exit(main())