Change scanner buffers to not use utf-8.

src/preparse-data.h

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 57 matching lines @@
   ParserRecorder() { }
   virtual ~ParserRecorder() { }
 
   // Logs the scope and some details of a function literal in the source.
   virtual void LogFunction(int start,
                            int end,
                            int literals,
                            int properties) = 0;
 
   // Logs a symbol creation of a literal or identifier.
-  virtual void LogSymbol(int start, const char* symbol, int length) = 0;
+  virtual void LogAsciiSymbol(int start, Vector<const char> literal) { }
+  virtual void LogUC16Symbol(int start, Vector<const uc16> literal) { }
 
   // Logs an error message and marks the log as containing an error.
   // Further logging will be ignored, and ExtractData will return a vector
   // representing the error only.
   virtual void LogMessage(int start,
                           int end,
                           const char* message,
                           const char* argument_opt) = 0;
 
   virtual int function_position() = 0;
@@ skipping 69 matching lines @@
 #endif
 };
 
 
 // ----------------------------------------------------------------------------
 // PartialParserRecorder - Record only function entries
 
 class PartialParserRecorder : public FunctionLoggingParserRecorder {
  public:
   PartialParserRecorder() : FunctionLoggingParserRecorder() { }
-  virtual void LogSymbol(int start, const char* symbol, int length) { }
+  virtual void LogAsciiSymbol(int start, Vector<const char> literal) { }
+  virtual void LogUC16Symbol(int start, Vector<const uc16> literal) { }
   virtual ~PartialParserRecorder() { }
   virtual Vector<unsigned> ExtractData();
   virtual int symbol_position() { return 0; }
   virtual int symbol_ids() { return 0; }
 };
 
 
 // ----------------------------------------------------------------------------
 // CompleteParserRecorder -  Record both function entries and symbols.
 
 class CompleteParserRecorder: public FunctionLoggingParserRecorder {
  public:
   CompleteParserRecorder();
   virtual ~CompleteParserRecorder() { }
 
-  virtual void LogSymbol(int start, const char* symbol, int length);
+  virtual void LogAsciiSymbol(int start, Vector<const char> literal) {
+    if (!is_recording_) return;
+    int hash = vector_hash(literal);
+    LogSymbol(start, hash, true, Vector<const byte>::cast(literal));
+  }
+
+  virtual void LogUC16Symbol(int start, Vector<const uc16> literal) {
+    if (!is_recording_) return;
+    int hash = vector_hash(literal);
+    LogSymbol(start, hash, false, Vector<const byte>::cast(literal));
+  }
 
   virtual Vector<unsigned> ExtractData();
 
   virtual int symbol_position() { return symbol_store_.size(); }
   virtual int symbol_ids() { return symbol_id_; }
 
  private:
-  static int vector_hash(Vector<const char> string) {
+  struct Key {
+    bool is_ascii;
+    Vector<const byte> literal_bytes;
+  };
+
+  virtual void LogSymbol(int start,
+                         int hash,
+                         bool is_ascii,
+                         Vector<const byte> literal);
+
+  template <typename Char>
+  static int vector_hash(Vector<const Char> string) {
     int hash = 0;
     for (int i = 0; i < string.length(); i++) {
-      int c = string[i];
+      int c = static_cast<int>(string[i]);
       hash += c;
       hash += (hash << 10);
       hash ^= (hash >> 6);
     }
     return hash;
   }
 
   static bool vector_compare(void* a, void* b) {
-    Vector<const char>* string1 = reinterpret_cast<Vector<const char>* >(a);
-    Vector<const char>* string2 = reinterpret_cast<Vector<const char>* >(b);
-    int length = string1->length();
-    if (string2->length() != length) return false;
-    return memcmp(string1->start(), string2->start(), length) == 0;
+    Key* string1 = reinterpret_cast<Key*>(a);
+    Key* string2 = reinterpret_cast<Key*>(b);
+    if (string1->is_ascii != string2->is_ascii) return false;
+    int length = string1->literal_bytes.length();
+    if (string2->literal_bytes.length() != length) return false;
+    return memcmp(string1->literal_bytes.start(),
+                  string2->literal_bytes.start(), length) == 0;
   }
 
   // Write a non-negative number to the symbol store.
   void WriteNumber(int number);
 
+  Collector<byte> literal_chars_;
   Collector<byte> symbol_store_;
-  Collector<Vector<const char> > symbol_entries_;
+  Collector<Key> symbol_keys_;
   HashMap symbol_table_;
   int symbol_id_;
 };
 
 
 } }  // namespace v8::internal.
 
 #endif  // V8_PREPARSER_DATA_H_