Compress sources of JS libraries in addition to the snapshot.

tools/js2c.py

 #!/usr/bin/env python
 #
 # Copyright 2006-2008 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 15 matching lines @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 # This is a utility for converting JavaScript source code into C-style
 # char arrays. It is used for embedded JavaScript code in the V8
 # library.
 
 import os, re, sys, string
 import jsmin
+import bz2
+
+
+def ToCAsciiArray(lines):
+  result = []
+  for chr in lines:
+    value = ord(chr)
+    assert value < 128
+    result.append(str(value))
+  return ", ".join(result)
 
 
 def ToCArray(lines):
   result = []
   for chr in lines:
-    value = ord(chr)
-    assert value < 128
-    result.append(str(value))
-  result.append("0")
+    result.append(str(ord(chr)))
   return ", ".join(result)
 
 
 def RemoveCommentsAndTrailingWhitespace(lines):
   lines = re.sub(r'//.*\n', '\n', lines) # end-of-line comments
   lines = re.sub(re.compile(r'/\*.*?\*/', re.DOTALL), '', lines) # comments.
   lines = re.sub(r'\s+\n+', '\n', lines) # trailing whitespace
   return lines
 
 
@@ skipping 25 matching lines @@
 
 
 def ParseValue(string):
   string = string.strip()
   if string.startswith('[') and string.endswith(']'):
     return string.lstrip('[').rstrip(']').split()
   else:
     return string
 
 
-EVAL_PATTERN = re.compile(r'\beval\s*\(');
-WITH_PATTERN = re.compile(r'\bwith\s*\(');
+EVAL_PATTERN = re.compile(r'\beval\s*\(')
+WITH_PATTERN = re.compile(r'\bwith\s*\(')
 
 
 def Validate(lines, file):
   lines = RemoveCommentsAndTrailingWhitespace(lines)
   # Because of simplified context setup, eval and with is not
   # allowed in the natives files.
   eval_match = EVAL_PATTERN.search(lines)
   if eval_match:
     raise ("Eval disallowed in natives: %s" % file)
   with_match = WITH_PATTERN.search(lines)
@@ skipping 103 matching lines @@
 
 HEADER_TEMPLATE = """\
 // Copyright 2011 Google Inc. All Rights Reserved.
 
 // This file was generated from .js source files by SCons.  If you
 // want to make changes to this file you should either change the
 // javascript source files or the SConstruct script.
 
 #include "v8.h"
 #include "natives.h"
+#include "utils.h"
 
 namespace v8 {
 namespace internal {
 
-%(source_lines)s\
+  static const byte sources[] = { %(sources_data)s };
+
+%(raw_sources_declaration)s\
 
   template <>
   int NativesCollection<%(type)s>::GetBuiltinsCount() {
     return %(builtin_count)i;
   }
 
   template <>
   int NativesCollection<%(type)s>::GetDebuggerCount() {
     return %(debugger_count)i;
   }
 
   template <>
   int NativesCollection<%(type)s>::GetIndex(const char* name) {
 %(get_index_cases)s\
     return -1;
   }
 
   template <>
-  Vector<const char> NativesCollection<%(type)s>::GetScriptSource(int index) {
-%(get_script_source_cases)s\
+  int NativesCollection<%(type)s>::GetRawScriptsSize() {
+    return %(raw_total_length)i;
+  }
+
+  template <>
+  Vector<const char> NativesCollection<%(type)s>::GetRawScriptSource(int index) {
+%(get_raw_script_source_cases)s\
     return Vector<const char>("", 0);
   }
 
   template <>
   Vector<const char> NativesCollection<%(type)s>::GetScriptName(int index) {
 %(get_script_name_cases)s\
     return Vector<const char>("", 0);
   }
 
+  template <>
+  Vector<const byte> NativesCollection<%(type)s>::GetScriptsSource() {
+    return Vector<const byte>(sources, %(total_length)i);
+  }
+
+  template <>
+  void NativesCollection<%(type)s>::SetRawScriptsSource(Vector<const char> raw_source) {
+    ASSERT(%(raw_total_length)i == raw_source.length());
+    raw_sources = raw_source.start();
+  }
+
 }  // internal
 }  // v8
 """
 
 
-SOURCE_DECLARATION = """\
-  static const char %(id)s[] = { %(data)s };
+RAW_SOURCES_COMPRESSION_DECLARATION = """\
+  static const char* raw_sources = NULL;
 """
 
 
-GET_DEBUGGER_INDEX_CASE = """\
+RAW_SOURCES_DECLARATION = """\
+  static const char* raw_sources = reinterpret_cast<const char*>(sources);
+"""
+
+
+GET_INDEX_CASE = """\
     if (strcmp(name, "%(id)s") == 0) return %(i)i;
 """
 
 
-GET_DEBUGGER_SCRIPT_SOURCE_CASE = """\
-    if (index == %(i)i) return Vector<const char>(%(id)s, %(length)i);
+GET_RAW_SCRIPT_SOURCE_CASE = """\
+    if (index == %(i)i) return Vector<const char>(raw_sources + %(offset)i, %(raw_length)i);
 """
 
 
-GET_DEBUGGER_SCRIPT_NAME_CASE = """\
+GET_SCRIPT_NAME_CASE = """\
     if (index == %(i)i) return Vector<const char>("%(name)s", %(length)i);
 """
 
 def JS2C(source, target, env):
   ids = []
   debugger_ids = []
   modules = []
   # Locate the macros file name.
   consts = []
   macros = []
   for s in source:
     if 'macros.py' == (os.path.split(str(s))[1]):
       (consts, macros) = ReadMacros(ReadLines(str(s)))
     else:
       modules.append(s)
 
-  # Build source code lines
-  source_lines = [ ]
-
   minifier = jsmin.JavaScriptMinifier()
 
+  module_offset = 0
+  all_sources = []
   for module in modules:
     filename = str(module)
     debugger = filename.endswith('-debugger.js')
     lines = ReadFile(filename)
     lines = ExpandConstants(lines, consts)
     lines = ExpandMacros(lines, macros)
     Validate(lines, filename)
     lines = minifier.JSMinify(lines)
-    data = ToCArray(lines)
     id = (os.path.split(filename)[1])[:-3]
     if debugger: id = id[:-9]
+    raw_length = len(lines)
     if debugger:
-      debugger_ids.append((id, len(lines)))
+      debugger_ids.append((id, raw_length, module_offset))
     else:
-      ids.append((id, len(lines)))
-    source_lines.append(SOURCE_DECLARATION % { 'id': id, 'data': data })
+      ids.append((id, raw_length, module_offset))
+    all_sources.append(lines)
+    module_offset += raw_length
+  total_length = raw_total_length = module_offset
+
+  if env['COMPRESSION'] == 'off':
+    raw_sources_declaration = RAW_SOURCES_DECLARATION
+    sources_data = ToCAsciiArray("".join(all_sources))
+  else:
+    raw_sources_declaration = RAW_SOURCES_COMPRESSION_DECLARATION
+    if env['COMPRESSION'] == 'bz2':
+      all_sources = bz2.compress("".join(all_sources))
+    total_length = len(all_sources)
+    sources_data = ToCArray(all_sources)
 
   # Build debugger support functions
   get_index_cases = [ ]
-  get_script_source_cases = [ ]
+  get_raw_script_source_cases = [ ]
   get_script_name_cases = [ ]
 
   i = 0
-  for (id, length) in debugger_ids:
+  for (id, raw_length, module_offset) in debugger_ids + ids:
     native_name = "native %s.js" % id
-    get_index_cases.append(GET_DEBUGGER_INDEX_CASE % { 'id': id, 'i': i })
-    get_script_source_cases.append(GET_DEBUGGER_SCRIPT_SOURCE_CASE % {
-      'id': id,
-      'length': length,
-      'i': i
-    })
-    get_script_name_cases.append(GET_DEBUGGER_SCRIPT_NAME_CASE % {
-      'name': native_name,
-      'length': len(native_name),
-      'i': i
-    });
-    i = i + 1
-
-  for (id, length) in ids:
-    native_name = "native %s.js" % id
-    get_index_cases.append(GET_DEBUGGER_INDEX_CASE % { 'id': id, 'i': i })
-    get_script_source_cases.append(GET_DEBUGGER_SCRIPT_SOURCE_CASE % {
-      'id': id,
-      'length': length,
-      'i': i
-    })
-    get_script_name_cases.append(GET_DEBUGGER_SCRIPT_NAME_CASE % {
-      'name': native_name,
-      'length': len(native_name),
-      'i': i
-    });
+    get_index_cases.append(GET_INDEX_CASE % { 'id': id, 'i': i })
+    get_raw_script_source_cases.append(GET_RAW_SCRIPT_SOURCE_CASE % {
+        'offset': module_offset,
+        'raw_length': raw_length,
+        'i': i
+        })
+    get_script_name_cases.append(GET_SCRIPT_NAME_CASE % {
+        'name': native_name,
+        'length': len(native_name),
+        'i': i
+        })
     i = i + 1
 
   # Emit result
   output = open(str(target[0]), "w")
   output.write(HEADER_TEMPLATE % {
     'builtin_count': len(ids) + len(debugger_ids),
     'debugger_count': len(debugger_ids),
-    'source_lines': "\n".join(source_lines),
+    'sources_data': sources_data,
+    'raw_sources_declaration': raw_sources_declaration,
+    'raw_total_length': raw_total_length,
+    'total_length': total_length,
     'get_index_cases': "".join(get_index_cases),
-    'get_script_source_cases': "".join(get_script_source_cases),
+    'get_raw_script_source_cases': "".join(get_raw_script_source_cases),
     'get_script_name_cases': "".join(get_script_name_cases),
     'type': env['TYPE']
   })
   output.close()
 
 def main():
   natives = sys.argv[1]
   type = sys.argv[2]
-  source_files = sys.argv[3:]
-  JS2C(source_files, [natives], { 'TYPE': type })
+  compression = sys.argv[3]
+  source_files = sys.argv[4:]
+  JS2C(source_files, [natives], { 'TYPE': type, 'COMPRESSION': compression })
 
 if __name__ == "__main__":
   main()