Change scanner buffers to not use utf-8.

src/preparser.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1103 matching lines @@
   if (scanner_->has_line_terminator_before_next() ||
       tok == i::Token::RBRACE ||
       tok == i::Token::EOS) {
     return;
   }
   Expect(i::Token::SEMICOLON, ok);
 }
 
 
 PreParser::Identifier PreParser::GetIdentifierSymbol() {
-  const char* literal_chars = scanner_->literal_string();
-  int literal_length = scanner_->literal_length();
   int identifier_pos = scanner_->location().beg_pos;
-
-  log_->LogSymbol(identifier_pos, literal_chars, literal_length);
-
-  return kUnknownExpression;
+  if (scanner_->is_literal_ascii()) {
+    log_->LogAsciiSymbol(identifier_pos, scanner_->literal_ascii_string());
+  } else {
+    log_->LogUC16Symbol(identifier_pos, scanner_->literal_uc16_string());
+  }
+  return kUnknownIdentifier;
 }
 
 
 PreParser::Expression PreParser::GetStringSymbol() {
-  const char* literal_chars = scanner_->literal_string();
-  int literal_length = scanner_->literal_length();
-
-  int literal_position = scanner_->location().beg_pos;
-  log_->LogSymbol(literal_position, literal_chars, literal_length);
-
+  int identifier_pos = scanner_->location().beg_pos;
+  if (scanner_->is_literal_ascii()) {
+    log_->LogAsciiSymbol(identifier_pos, scanner_->literal_ascii_string());
+  } else {
+    log_->LogUC16Symbol(identifier_pos, scanner_->literal_uc16_string());
+  }
   return kUnknownExpression;
 }
 
 
 PreParser::Identifier PreParser::ParseIdentifier(bool* ok) {
   Expect(i::Token::IDENTIFIER, ok);
   if (!*ok) return kUnknownIdentifier;
   return GetIdentifierSymbol();
 }
 
 
 PreParser::Identifier PreParser::ParseIdentifierName(bool* ok) {
   i::Token::Value next = Next();
   if (i::Token::IsKeyword(next)) {
     int pos = scanner_->location().beg_pos;
     const char* keyword = i::Token::String(next);
-    log_->LogSymbol(pos, keyword, i::StrLength(keyword));
+    log_->LogAsciiSymbol(pos, i::Vector<const char>(keyword,
+                                                    i::StrLength(keyword)));
     return kUnknownExpression;
   }
   if (next == i::Token::IDENTIFIER) {
     return GetIdentifierSymbol();
   }
   *ok = false;
   return kUnknownIdentifier;
 }
 
 
 // This function reads an identifier and determines whether or not it
 // is 'get' or 'set'.  The reason for not using ParseIdentifier and
 // checking on the output is that this involves heap allocation which
 // we can't do during preparsing.
 PreParser::Identifier PreParser::ParseIdentifierOrGetOrSet(bool* is_get,
                                                            bool* is_set,
                                                            bool* ok) {
   Expect(i::Token::IDENTIFIER, CHECK_OK);
-  if (scanner_->literal_length() == 3) {
-    const char* token = scanner_->literal_string();
+  if (scanner_->is_literal_ascii() && scanner_->literal_length() == 3) {
+    const char* token = scanner_->literal_ascii_string().start();
     *is_get = strncmp(token, "get", 3) == 0;
     *is_set = !*is_get && strncmp(token, "set", 3) == 0;
   }
   return GetIdentifierSymbol();
 }
 
 #undef CHECK_OK
 } }  // v8::preparser