Compress sources of JS libraries in addition to the snapshot.

tools/js2c.py

 #!/usr/bin/env python
 #
 # Copyright 2006-2008 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 15 matching lines @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 # This is a utility for converting JavaScript source code into C-style
 # char arrays. It is used for embedded JavaScript code in the V8
 # library.
 
 import os, re, sys, string
 import jsmin
+import bz2
+
+
+def ToCAsciiArray(lines):
+  result = []
+  for chr in lines:
+    value = ord(chr)
+    assert value < 128
+    result.append(str(value))
+  return ", ".join(result)
 
 
 def ToCArray(lines):
   result = []
   for chr in lines:
-    value = ord(chr)
-    assert value < 128
-    result.append(str(value))
-  result.append("0")
+    result.append(str(ord(chr)))
   return ", ".join(result)
 
 
 def RemoveCommentsAndTrailingWhitespace(lines):
   lines = re.sub(r'//.*\n', '\n', lines) # end-of-line comments
   lines = re.sub(re.compile(r'/\*.*?\*/', re.DOTALL), '', lines) # comments.
   lines = re.sub(r'\s+\n+', '\n', lines) # trailing whitespace
   return lines
 
 
@@ skipping 25 matching lines @@
 
 
 def ParseValue(string):
   string = string.strip()
   if string.startswith('[') and string.endswith(']'):
     return string.lstrip('[').rstrip(']').split()
   else:
     return string
 
 
-EVAL_PATTERN = re.compile(r'\beval\s*\(');
-WITH_PATTERN = re.compile(r'\bwith\s*\(');
+EVAL_PATTERN = re.compile(r'\beval\s*\(')
+WITH_PATTERN = re.compile(r'\bwith\s*\(')
 
 
 def Validate(lines, file):
   lines = RemoveCommentsAndTrailingWhitespace(lines)
   # Because of simplified context setup, eval and with is not
   # allowed in the natives files.
   eval_match = EVAL_PATTERN.search(lines)
   if eval_match:
     raise ("Eval disallowed in natives: %s" % file)
   with_match = WITH_PATTERN.search(lines)
@@ skipping 103 matching lines @@
 
 HEADER_TEMPLATE = """\
 // Copyright 2011 Google Inc. All Rights Reserved.
 
 // This file was generated from .js source files by SCons.  If you
 // want to make changes to this file you should either change the
 // javascript source files or the SConstruct script.
 
 #include "v8.h"
 #include "natives.h"
+#include "utils.h"
 
 namespace v8 {
 namespace internal {
 
 %(source_lines)s\
 
+%(raw_source_lines)s\
+
   template <>
   int NativesCollection<%(type)s>::GetBuiltinsCount() {
     return %(builtin_count)i;
   }
 
   template <>
   int NativesCollection<%(type)s>::GetDebuggerCount() {
     return %(debugger_count)i;
   }
 
   template <>
   int NativesCollection<%(type)s>::GetIndex(const char* name) {
 %(get_index_cases)s\
     return -1;
   }
 
   template <>
-  Vector<const char> NativesCollection<%(type)s>::GetScriptSource(int index) {
-%(get_script_source_cases)s\
+  int NativesCollection<%(type)s>::GetRawScriptSize(int index) {
+%(get_raw_script_size_cases)s\
+    return -1;
+  }
+
+  template <>
+  Vector<const char> NativesCollection<%(type)s>::GetRawScriptSource(int index) {
+%(get_raw_script_source_cases)s\
     return Vector<const char>("", 0);
   }
 
   template <>
   Vector<const char> NativesCollection<%(type)s>::GetScriptName(int index) {
 %(get_script_name_cases)s\
     return Vector<const char>("", 0);
   }
 
+  template <>
+  Vector<const byte> NativesCollection<%(type)s>::GetScriptSource(int index) {
+%(get_script_source_cases)s\
+    static const byte empty[] = { 0 };
+    return Vector<const byte>(empty, 0);
+  }
+
+  template <>
+  void NativesCollection<%(type)s>::SetRawScriptSource(int index, Vector<const char> raw_source) {
+%(set_raw_script_source_cases)s\
+  }
+
 }  // internal
 }  // v8
 """
 
 
 SOURCE_DECLARATION = """\
-  static const char %(id)s[] = { %(data)s };
+  static const byte %(id)s[] = { %(data)s };
 """
 
 
-GET_DEBUGGER_INDEX_CASE = """\
+RAW_SOURCE_COMPRESSION_DECLARATION = """\
+  static const char* %(raw_id)s = NULL;
+"""
+
+
+RAW_SOURCE_DECLARATION = """\
+  static const char* %(raw_id)s = reinterpret_cast<const char*>(%(id)s);
+"""
+
+
+GET_INDEX_CASE = """\
     if (strcmp(name, "%(id)s") == 0) return %(i)i;
 """
 
 
-GET_DEBUGGER_SCRIPT_SOURCE_CASE = """\
-    if (index == %(i)i) return Vector<const char>(%(id)s, %(length)i);
+GET_RAW_SCRIPT_SIZE_CASE = """\
+    if (index == %(i)i) return %(raw_length)i;
 """
 
 
-GET_DEBUGGER_SCRIPT_NAME_CASE = """\
+GET_RAW_SCRIPT_SOURCE_CASE = """\
+    if (index == %(i)i) return Vector<const char>(%(raw_id)s, %(raw_length)i);
+"""
+
+
+GET_SCRIPT_NAME_CASE = """\
     if (index == %(i)i) return Vector<const char>("%(name)s", %(length)i);
 """
 
+
+GET_SCRIPT_SOURCE_CASE = """\
+    if (index == %(i)i) return Vector<const byte>(%(id)s, %(length)i);
+"""
+
+
+SET_RAW_SOURCE_CASE = """\
+    if (index == %(i)i) { ASSERT(%(raw_length)i == raw_source.length()); %(raw_id)s = raw_source.start(); }
+"""
+
 def JS2C(source, target, env):
   ids = []
   debugger_ids = []
   modules = []
   # Locate the macros file name.
   consts = []
   macros = []
   for s in source:
     if 'macros.py' == (os.path.split(str(s))[1]):
       (consts, macros) = ReadMacros(ReadLines(str(s)))
     else:
       modules.append(s)
 
   # Build source code lines
   source_lines = [ ]
+  raw_source_lines = [ ]
 
   minifier = jsmin.JavaScriptMinifier()
 
   for module in modules:
     filename = str(module)
     debugger = filename.endswith('-debugger.js')
     lines = ReadFile(filename)
     lines = ExpandConstants(lines, consts)
     lines = ExpandMacros(lines, macros)
     Validate(lines, filename)
     lines = minifier.JSMinify(lines)
-    data = ToCArray(lines)
+    raw_length = len(lines)
+    if env['COMPRESSION'] == 'bz2':

[Vitaly Repeshko, 2011/06/06 10:08:23]

I think we can get better compression by gluing together all sources.

[mnaganov (inactive), 2011/06/06 13:40:18]

Good idea! This also simplifies code.

+      lines = bz2.compress(lines, 9)

[Vitaly Repeshko, 2011/06/06 10:08:23]

9 seems to be the default compression level so we can avoid the magic number.

[mnaganov (inactive), 2011/06/06 13:40:18]

Done.

+    length = len(lines)
+    if env['COMPRESSION'] == 'off':
+      data = ToCAsciiArray(lines)
+    else:
+      data = ToCArray(lines)
     id = (os.path.split(filename)[1])[:-3]
     if debugger: id = id[:-9]
     if debugger:
-      debugger_ids.append((id, len(lines)))
+      debugger_ids.append((id, raw_length, length))
     else:
-      ids.append((id, len(lines)))
+      ids.append((id, raw_length, length))
+    raw_id = 'raw_' + id
     source_lines.append(SOURCE_DECLARATION % { 'id': id, 'data': data })
+    if env['COMPRESSION'] == 'off':
+      raw_source_lines.append(RAW_SOURCE_DECLARATION % { 'raw_id': raw_id, 'id': id })

[Vitaly Repeshko, 2011/06/06 10:08:23]

Let's at least fit the code lines in 80 cols.

[mnaganov (inactive), 2011/06/06 13:40:18]

Done.

+    else:
+      raw_source_lines.append(RAW_SOURCE_COMPRESSION_DECLARATION % { 'raw_id': raw_id, 'id': id })
 
   # Build debugger support functions
   get_index_cases = [ ]
+  get_raw_script_size_cases = [ ]
+  get_raw_script_source_cases = [ ]
+  get_script_name_cases = [ ]
   get_script_source_cases = [ ]
-  get_script_name_cases = [ ]
+  set_raw_script_source_cases = [ ]
 
   i = 0
-  for (id, length) in debugger_ids:
+  for (id, raw_length, length) in debugger_ids:
     native_name = "native %s.js" % id
-    get_index_cases.append(GET_DEBUGGER_INDEX_CASE % { 'id': id, 'i': i })
-    get_script_source_cases.append(GET_DEBUGGER_SCRIPT_SOURCE_CASE % {
-      'id': id,
-      'length': length,
-      'i': i
-    })
-    get_script_name_cases.append(GET_DEBUGGER_SCRIPT_NAME_CASE % {
-      'name': native_name,
-      'length': len(native_name),
-      'i': i
-    });
+    raw_id = 'raw_' + id
+    get_index_cases.append(GET_INDEX_CASE % { 'id': id, 'i': i })
+    get_raw_script_size_cases.append(GET_RAW_SCRIPT_SIZE_CASE % {
+        'raw_length': raw_length,
+        'i': i
+        })
+    get_raw_script_source_cases.append(GET_RAW_SCRIPT_SOURCE_CASE % {
+        'raw_id': raw_id,
+        'raw_length': raw_length,
+        'i': i
+        })
+    get_script_name_cases.append(GET_SCRIPT_NAME_CASE % {
+        'name': native_name,
+        'length': len(native_name),
+        'i': i
+        })
+    get_script_source_cases.append(GET_SCRIPT_SOURCE_CASE % {
+        'id': id,
+        'length': length,
+        'i': i
+        })
+    set_raw_script_source_cases.append(SET_RAW_SOURCE_CASE % {
+        'raw_id': raw_id,
+        'raw_length': raw_length,
+        'i': i
+        })
     i = i + 1
 
-  for (id, length) in ids:
+  for (id, raw_length, length) in ids:
     native_name = "native %s.js" % id
-    get_index_cases.append(GET_DEBUGGER_INDEX_CASE % { 'id': id, 'i': i })
-    get_script_source_cases.append(GET_DEBUGGER_SCRIPT_SOURCE_CASE % {
-      'id': id,
-      'length': length,
-      'i': i
-    })
-    get_script_name_cases.append(GET_DEBUGGER_SCRIPT_NAME_CASE % {
-      'name': native_name,
-      'length': len(native_name),
-      'i': i
-    });
+    raw_id = 'raw_' + id
+    get_index_cases.append(GET_INDEX_CASE % { 'id': id, 'i': i })
+    get_raw_script_size_cases.append(GET_RAW_SCRIPT_SIZE_CASE % {
+        'raw_length': raw_length,
+        'i': i
+        })
+    get_raw_script_source_cases.append(GET_RAW_SCRIPT_SOURCE_CASE % {
+        'raw_id': raw_id,
+        'raw_length': raw_length,
+        'i': i
+        })
+    get_script_name_cases.append(GET_SCRIPT_NAME_CASE % {
+        'name': native_name,
+        'length': len(native_name),
+        'i': i
+        })
+    get_script_source_cases.append(GET_SCRIPT_SOURCE_CASE % {
+        'id': id,
+        'length': length,
+        'i': i
+        })
+    set_raw_script_source_cases.append(SET_RAW_SOURCE_CASE % {
+        'raw_id': raw_id,
+        'raw_length': raw_length,
+        'i': i
+        })
     i = i + 1
 
   # Emit result
   output = open(str(target[0]), "w")
   output.write(HEADER_TEMPLATE % {
     'builtin_count': len(ids) + len(debugger_ids),
     'debugger_count': len(debugger_ids),
     'source_lines': "\n".join(source_lines),
+    'raw_source_lines': "\n".join(raw_source_lines),
     'get_index_cases': "".join(get_index_cases),
+    'get_raw_script_size_cases': "".join(get_raw_script_size_cases),
+    'get_raw_script_source_cases': "".join(get_raw_script_source_cases),
+    'get_script_name_cases': "".join(get_script_name_cases),
     'get_script_source_cases': "".join(get_script_source_cases),
-    'get_script_name_cases': "".join(get_script_name_cases),
+    'set_raw_script_source_cases': "".join(set_raw_script_source_cases),
     'type': env['TYPE']
   })
   output.close()
 
 def main():
   natives = sys.argv[1]
   type = sys.argv[2]
-  source_files = sys.argv[3:]
-  JS2C(source_files, [natives], { 'TYPE': type })
+  compression = sys.argv[3]
+  source_files = sys.argv[4:]
+  JS2C(source_files, [natives], { 'TYPE': type, 'COMPRESSION': compression })
 
 if __name__ == "__main__":
   main()