Merge up to 8597 to experimental/gc from the bleeding edge.

tools/js2c.py

Index: tools/js2c.py
===================================================================
--- tools/js2c.py	(revision 8618)
+++ tools/js2c.py	(working copy)
@@ -33,18 +33,25 @@
 
 import os, re, sys, string
 import jsmin
+import bz2
 
 
-def ToCArray(lines):
+def ToCAsciiArray(lines):
   result = []
   for chr in lines:
     value = ord(chr)
     assert value < 128
     result.append(str(value))
-  result.append("0")
   return ", ".join(result)
 
 
+def ToCArray(lines):
+  result = []
+  for chr in lines:
+    result.append(str(ord(chr)))
+  return ", ".join(result)
+
+
 def RemoveCommentsAndTrailingWhitespace(lines):
   lines = re.sub(r'//.*\n', '\n', lines) # end-of-line comments
   lines = re.sub(re.compile(r'/\*.*?\*/', re.DOTALL), '', lines) # comments.
@@ -87,8 +94,8 @@
     return string
 
 
-EVAL_PATTERN = re.compile(r'\beval\s*\(');
-WITH_PATTERN = re.compile(r'\bwith\s*\(');
+EVAL_PATTERN = re.compile(r'\beval\s*\(')
+WITH_PATTERN = re.compile(r'\bwith\s*\(')
 
 
 def Validate(lines, file):
@@ -212,12 +219,15 @@
 
 #include "v8.h"
 #include "natives.h"
+#include "utils.h"
 
 namespace v8 {
 namespace internal {
 
-%(source_lines)s\
+  static const byte sources[] = { %(sources_data)s };
 
+%(raw_sources_declaration)s\
+
   template <>
   int NativesCollection<%(type)s>::GetBuiltinsCount() {
     return %(builtin_count)i;
@@ -235,8 +245,13 @@
   }
 
   template <>
-  Vector<const char> NativesCollection<%(type)s>::GetScriptSource(int index) {
-%(get_script_source_cases)s\
+  int NativesCollection<%(type)s>::GetRawScriptsSize() {
+    return %(raw_total_length)i;
+  }
+
+  template <>
+  Vector<const char> NativesCollection<%(type)s>::GetRawScriptSource(int index) {
+%(get_raw_script_source_cases)s\
     return Vector<const char>("", 0);
   }
 
@@ -246,27 +261,43 @@
     return Vector<const char>("", 0);
   }
 
+  template <>
+  Vector<const byte> NativesCollection<%(type)s>::GetScriptsSource() {
+    return Vector<const byte>(sources, %(total_length)i);
+  }
+
+  template <>
+  void NativesCollection<%(type)s>::SetRawScriptsSource(Vector<const char> raw_source) {
+    ASSERT(%(raw_total_length)i == raw_source.length());
+    raw_sources = raw_source.start();
+  }
+
 }  // internal
 }  // v8
 """
 
 
-SOURCE_DECLARATION = """\
-  static const char %(id)s[] = { %(data)s };
+RAW_SOURCES_COMPRESSION_DECLARATION = """\
+  static const char* raw_sources = NULL;
 """
 
 
-GET_DEBUGGER_INDEX_CASE = """\
+RAW_SOURCES_DECLARATION = """\
+  static const char* raw_sources = reinterpret_cast<const char*>(sources);
+"""
+
+
+GET_INDEX_CASE = """\
     if (strcmp(name, "%(id)s") == 0) return %(i)i;
 """
 
 
-GET_DEBUGGER_SCRIPT_SOURCE_CASE = """\
-    if (index == %(i)i) return Vector<const char>(%(id)s, %(length)i);
+GET_RAW_SCRIPT_SOURCE_CASE = """\
+    if (index == %(i)i) return Vector<const char>(raw_sources + %(offset)i, %(raw_length)i);
 """
 
 
-GET_DEBUGGER_SCRIPT_NAME_CASE = """\
+GET_SCRIPT_NAME_CASE = """\
     if (index == %(i)i) return Vector<const char>("%(name)s", %(length)i);
 """
 
@@ -283,11 +314,10 @@
     else:
       modules.append(s)
 
-  # Build source code lines
-  source_lines = [ ]
-
   minifier = jsmin.JavaScriptMinifier()
 
+  module_offset = 0
+  all_sources = []
   for module in modules:
     filename = str(module)
     debugger = filename.endswith('-debugger.js')
@@ -296,59 +326,59 @@
     lines = ExpandMacros(lines, macros)
     Validate(lines, filename)
     lines = minifier.JSMinify(lines)
-    data = ToCArray(lines)
     id = (os.path.split(filename)[1])[:-3]
     if debugger: id = id[:-9]
+    raw_length = len(lines)
     if debugger:
-      debugger_ids.append((id, len(lines)))
+      debugger_ids.append((id, raw_length, module_offset))
     else:
-      ids.append((id, len(lines)))
-    source_lines.append(SOURCE_DECLARATION % { 'id': id, 'data': data })
+      ids.append((id, raw_length, module_offset))
+    all_sources.append(lines)
+    module_offset += raw_length
+  total_length = raw_total_length = module_offset
 
+  if env['COMPRESSION'] == 'off':
+    raw_sources_declaration = RAW_SOURCES_DECLARATION
+    sources_data = ToCAsciiArray("".join(all_sources))
+  else:
+    raw_sources_declaration = RAW_SOURCES_COMPRESSION_DECLARATION
+    if env['COMPRESSION'] == 'bz2':
+      all_sources = bz2.compress("".join(all_sources))
+    total_length = len(all_sources)
+    sources_data = ToCArray(all_sources)
+
   # Build debugger support functions
   get_index_cases = [ ]
-  get_script_source_cases = [ ]
+  get_raw_script_source_cases = [ ]
   get_script_name_cases = [ ]
 
   i = 0
-  for (id, length) in debugger_ids:
+  for (id, raw_length, module_offset) in debugger_ids + ids:
     native_name = "native %s.js" % id
-    get_index_cases.append(GET_DEBUGGER_INDEX_CASE % { 'id': id, 'i': i })
-    get_script_source_cases.append(GET_DEBUGGER_SCRIPT_SOURCE_CASE % {
-      'id': id,
-      'length': length,
-      'i': i
-    })
-    get_script_name_cases.append(GET_DEBUGGER_SCRIPT_NAME_CASE % {
-      'name': native_name,
-      'length': len(native_name),
-      'i': i
-    });
+    get_index_cases.append(GET_INDEX_CASE % { 'id': id, 'i': i })
+    get_raw_script_source_cases.append(GET_RAW_SCRIPT_SOURCE_CASE % {
+        'offset': module_offset,
+        'raw_length': raw_length,
+        'i': i
+        })
+    get_script_name_cases.append(GET_SCRIPT_NAME_CASE % {
+        'name': native_name,
+        'length': len(native_name),
+        'i': i
+        })
     i = i + 1
 
-  for (id, length) in ids:
-    native_name = "native %s.js" % id
-    get_index_cases.append(GET_DEBUGGER_INDEX_CASE % { 'id': id, 'i': i })
-    get_script_source_cases.append(GET_DEBUGGER_SCRIPT_SOURCE_CASE % {
-      'id': id,
-      'length': length,
-      'i': i
-    })
-    get_script_name_cases.append(GET_DEBUGGER_SCRIPT_NAME_CASE % {
-      'name': native_name,
-      'length': len(native_name),
-      'i': i
-    });
-    i = i + 1
-
   # Emit result
   output = open(str(target[0]), "w")
   output.write(HEADER_TEMPLATE % {
     'builtin_count': len(ids) + len(debugger_ids),
     'debugger_count': len(debugger_ids),
-    'source_lines': "\n".join(source_lines),
+    'sources_data': sources_data,
+    'raw_sources_declaration': raw_sources_declaration,
+    'raw_total_length': raw_total_length,
+    'total_length': total_length,
     'get_index_cases': "".join(get_index_cases),
-    'get_script_source_cases': "".join(get_script_source_cases),
+    'get_raw_script_source_cases': "".join(get_raw_script_source_cases),
     'get_script_name_cases': "".join(get_script_name_cases),
     'type': env['TYPE']
   })
@@ -357,8 +387,9 @@
 def main():
   natives = sys.argv[1]
   type = sys.argv[2]
-  source_files = sys.argv[3:]
-  JS2C(source_files, [natives], { 'TYPE': type })
+  compression = sys.argv[3]
+  source_files = sys.argv[4:]
+  JS2C(source_files, [natives], { 'TYPE': type, 'COMPRESSION': compression })
 
 if __name__ == "__main__":
   main()