Reland: Make V8 extras a separate type of native

tools/js2c.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 322 matching lines @@
 """
 
 
 def BuildFilterChain(macro_filename, message_template_file):
   """Build the chain of filter functions to be applied to the sources.
 
   Args:
     macro_filename: Name of the macro file, if any.
 
   Returns:
-    A function (string -> string) that reads a source file and processes it.
+    A function (string -> string) that processes a source file.
   """
-  filter_chain = [ReadFile]
+  filter_chain = []
 
   if macro_filename:
     (consts, macros) = ReadMacros(ReadFile(macro_filename))
     filter_chain.append(lambda l: ExpandConstants(l, consts))
     filter_chain.append(lambda l: ExpandMacros(l, macros))
 
   if message_template_file:
     message_templates = ReadMessageTemplates(ReadFile(message_template_file))
     filter_chain.append(lambda l: ExpandConstants(l, message_templates))
 
   filter_chain.extend([
     RemoveCommentsAndTrailingWhitespace,
     ExpandInlineMacros,
     ExpandInlineConstants,
     Validate,
     jsmin.JavaScriptMinifier().JSMinify
   ])
 
   def chain(f1, f2):
     return lambda x: f2(f1(x))
 
   return reduce(chain, filter_chain)
 
 def BuildExtraFilterChain():
-  return lambda x: RemoveCommentsAndTrailingWhitespace(Validate(ReadFile(x)))
+  return lambda x: RemoveCommentsAndTrailingWhitespace(Validate(x))
 
 class Sources:
   def __init__(self):
     self.names = []
     self.modules = []
     self.is_debugger_id = []
 
 
 def IsDebuggerFile(filename):
   return filename.endswith("-debugger.js")
 
 def IsMacroFile(filename):
   return filename.endswith("macros.py")
 
 def IsMessageTemplateFile(filename):
   return filename.endswith("messages.h")
 
 
-def PrepareSources(source_files, extra_files, emit_js):
+def PrepareSources(source_files, native_type, emit_js):
   """Read, prepare and assemble the list of source files.
 
   Args:
     source_files: List of JavaScript-ish source files. A file named macros.py
         will be treated as a list of macros.
-    extra_files: List of JavaScript-ish extra source files, passed in
-        externally from V8. Will not be minified or macro-ified.
+    native_type: String corresponding to a NativeType enum value, allowing us
+        to treat different types of sources differently.
     emit_js: True if we should skip the byte conversion and just leave the
         sources as JS strings.
 
   Returns:
     An instance of Sources.
   """
   macro_file = None
   macro_files = filter(IsMacroFile, source_files)
   assert len(macro_files) in [0, 1]
   if macro_files:
     source_files.remove(macro_files[0])
     macro_file = macro_files[0]
 
   message_template_file = None
   message_template_files = filter(IsMessageTemplateFile, source_files)
   assert len(message_template_files) in [0, 1]
   if message_template_files:
     source_files.remove(message_template_files[0])
     message_template_file = message_template_files[0]
 
-  filters = BuildFilterChain(macro_file, message_template_file)
-  extra_filters = BuildExtraFilterChain()
+  filters = None
+  if native_type == "EXTRA":
+    filters = BuildExtraFilterChain()
+  else:
+    filters = BuildFilterChain(macro_file, message_template_file)
 
   # Sort 'debugger' sources first.
   source_files = sorted(source_files,
                         lambda l,r: IsDebuggerFile(r) - IsDebuggerFile(l))
 
+  source_files_and_contents = [(f, ReadFile(f)) for f in source_files]
+
+  # Have a single not-quite-empty source file if there are none present;
+  # otherwise you get errors trying to compile an empty C++ array.
+  # It cannot be empty (or whitespace, which gets trimmed to empty), as
+  # the deserialization code assumes each file is nonempty.
+  if not source_files_and_contents:
+    source_files_and_contents = [("dummy.js", "void 0;")]
+
   result = Sources()
 
-  for source in source_files:
+  for (source, contents) in source_files_and_contents:
     try:
-      lines = filters(source)
+      lines = filters(contents)
     except Error as e:
       raise Error("In file %s:\n%s" % (source, str(e)))
 
     result.modules.append(lines)
 
     is_debugger = IsDebuggerFile(source)
     result.is_debugger_id.append(is_debugger)
 
     name = os.path.basename(source)[:-3]
     result.names.append(name if not is_debugger else name[:-9])
 
-  for extra in extra_files:
-    try:
-      lines = extra_filters(extra)
-    except Error as e:
-      raise Error("In file %s:\n%s" % (extra, str(e)))
-
-    result.modules.append(lines)
-    name = os.path.basename(extra)[:-3]
-    result.names.append(name)
-
   return result
 
 
 def BuildMetadata(sources, source_bytes, native_type):
   """Build the meta data required to generate a libaries file.
 
   Args:
     sources: A Sources instance with the prepared sources.
     source_bytes: A list of source bytes.
         (The concatenation of all sources; might be compressed.)
@@ skipping 83 matching lines @@
     PutStr(output, sources.modules[i]);
 
   PutInt(output, len(sources.names) - debug_sources)
   for i in xrange(debug_sources, len(sources.names)):
     PutStr(output, sources.names[i]);
     PutStr(output, sources.modules[i]);
 
   output.close()
 
 
-def JS2C(sources, extra_sources, target, native_type, raw_file, startup_blob, emitJS):
-  prepared_sources = PrepareSources(sources, extra_sources, emitJS)
+def JS2C(sources, target, native_type, raw_file, startup_blob, emit_js):
+  prepared_sources = PrepareSources(sources, native_type, emit_js)
   sources_output = "".join(prepared_sources.modules)
   metadata = BuildMetadata(prepared_sources, sources_output, native_type)
 
   # Optionally emit raw file.
   if raw_file:
     output = open(raw_file, "w")
     output.write(sources_output)
     output.close()
 
   if startup_blob:
     WriteStartupBlob(prepared_sources, startup_blob)
 
   # Emit resulting source file.
   output = open(target, "w")
-  if emitJS:
+  if emit_js:
     output.write(sources_output)
   else:
     output.write(HEADER_TEMPLATE % metadata)
   output.close()
 
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument("out.cc",
                       help="output filename")
   parser.add_argument("type",
-                      help="type parameter for NativesCollection template")
+                      help="type parameter for NativesCollection template " +
+                           "(see NativeType enum)")
   parser.add_argument("sources.js",
                       help="JS internal sources or macros.py.",
-                      nargs="+")
+                      nargs="*")
   parser.add_argument("--raw",
                       help="file to write the processed sources array to.")
   parser.add_argument("--startup_blob",
                       help="file to write the startup blob to.")
-  parser.add_argument("--extra",
-                      help="extra JS sources.",
-                      nargs="*")
   parser.add_argument("--js",
                       help="writes a JS file output instead of a C file",
                       action="store_true")
 
   args = vars(parser.parse_args())
-  JS2C(args["sources.js"], args["extra"] or [], args["out.cc"], args["type"], args["raw"], args["startup_blob"],
-       args["js"])
+  JS2C(args["sources.js"], args["out.cc"], args["type"], args["raw"], args["startup_blob"], args["js"])
 
 
 if __name__ == "__main__":
   main()