[Isolates] Merge from bleeding_edge, revisions 5934-6100.

src/parser.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 593 matching lines @@
 FunctionLiteral* Parser::ParseProgram(Handle<String> source,
                                       bool in_global_context) {
   CompilationZoneScope zone_scope(DONT_DELETE_ON_EXIT);
 
   HistogramTimerScope timer(COUNTERS->parse());
   COUNTERS->total_parse_size()->Increment(source->length());
   fni_ = new FuncNameInferrer();
 
   // Initialize parser state.
   source->TryFlatten();
-  scanner_.Initialize(source);
+  if (source->IsExternalTwoByteString()) {
+    // Notice that the stream is destroyed at the end of the branch block.
+    // The last line of the blocks can't be moved outside, even though they're
+    // identical calls.
+    ExternalTwoByteStringUC16CharacterStream stream(
+        Handle<ExternalTwoByteString>::cast(source), 0, source->length());
+    scanner_.Initialize(&stream, JavaScriptScanner::kAllLiterals);
+    return DoParseProgram(source, in_global_context, &zone_scope);
+  } else {
+    GenericStringUC16CharacterStream stream(source, 0, source->length());
+    scanner_.Initialize(&stream, JavaScriptScanner::kAllLiterals);
+    return DoParseProgram(source, in_global_context, &zone_scope);
+  }
+}
+
+
+FunctionLiteral* Parser::DoParseProgram(Handle<String> source,
+                                        bool in_global_context,
+                                        ZoneScope* zone_scope) {
   ASSERT(target_stack_ == NULL);
   if (pre_data_ != NULL) pre_data_->Initialize();
 
   // Compute the parsing mode.
   mode_ = FLAG_lazy ? PARSE_LAZILY : PARSE_EAGERLY;
   if (allow_natives_syntax_ || extension_ != NULL) mode_ = PARSE_EAGERLY;
 
   Scope::Type type =
     in_global_context
       ? Scope::GLOBAL_SCOPE
@@ skipping 25 matching lines @@
     } else if (stack_overflow_) {
       isolate()->StackOverflow();
     }
   }
 
   // Make sure the target stack is empty.
   ASSERT(target_stack_ == NULL);
 
   // If there was a syntax error we have to get rid of the AST
   // and it is not safe to do so before the scope has been deleted.
-  if (result == NULL) zone_scope.DeleteOnExit();
+  if (result == NULL) zone_scope->DeleteOnExit();
   return result;
 }
 
-
 FunctionLiteral* Parser::ParseLazy(Handle<SharedFunctionInfo> info) {
   CompilationZoneScope zone_scope(DONT_DELETE_ON_EXIT);
   HistogramTimerScope timer(COUNTERS->parse_lazy());
   Handle<String> source(String::cast(script_->source()));
   COUNTERS->total_parse_size()->Increment(source->length());
 
+  // Initialize parser state.
+  source->TryFlatten();
+  if (source->IsExternalTwoByteString()) {
+    ExternalTwoByteStringUC16CharacterStream stream(
+        Handle<ExternalTwoByteString>::cast(source),
+        info->start_position(),
+        info->end_position());
+    FunctionLiteral* result = ParseLazy(info, &stream, &zone_scope);
+    return result;
+  } else {
+    GenericStringUC16CharacterStream stream(source,
+                                            info->start_position(),
+                                            info->end_position());
+    FunctionLiteral* result = ParseLazy(info, &stream, &zone_scope);
+    return result;
+  }
+}
+
+
+FunctionLiteral* Parser::ParseLazy(Handle<SharedFunctionInfo> info,
+                                   UC16CharacterStream* source,
+                                   ZoneScope* zone_scope) {
+  scanner_.Initialize(source, JavaScriptScanner::kAllLiterals);
+  ASSERT(target_stack_ == NULL);
+
   Handle<String> name(String::cast(info->name()));
   fni_ = new FuncNameInferrer();
   fni_->PushEnclosingName(name);
 
-  // Initialize parser state.
-  source->TryFlatten();
-  scanner_.Initialize(source, info->start_position(), info->end_position());
-  ASSERT(target_stack_ == NULL);
   mode_ = PARSE_EAGERLY;
 
   // Place holder for the result.
   FunctionLiteral* result = NULL;
 
   {
     // Parse the function literal.
     Handle<String> no_name = isolate()->factory()->empty_symbol();
     Scope* scope =
         NewScope(top_scope_, Scope::GLOBAL_SCOPE, inside_with());
@@ skipping 11 matching lines @@
     ASSERT(ok || stack_overflow_);
   }
 
   // Make sure the target stack is empty.
   ASSERT(target_stack_ == NULL);
 
   // If there was a stack overflow we have to get rid of AST and it is
   // not safe to do before scope has been deleted.
   if (result == NULL) {
     isolate()->StackOverflow();
-    zone_scope.DeleteOnExit();
+    zone_scope->DeleteOnExit();
   } else {
     Handle<String> inferred_name(info->inferred_name());
     result->set_inferred_name(inferred_name);
   }
   return result;
 }
 
 
 Handle<String> Parser::GetSymbol(bool* ok) {
   int symbol_id = -1;
   if (pre_data() != NULL) {
     symbol_id = pre_data()->GetSymbolIdentifier();
   }
-  return LookupSymbol(symbol_id, scanner_.literal());
+  return LookupSymbol(symbol_id, scanner().literal());
 }
 
 
 void Parser::ReportMessage(const char* type, Vector<const char*> args) {
-  Scanner::Location source_location = scanner_.location();
+  Scanner::Location source_location = scanner().location();
   ReportMessageAt(source_location, type, args);
 }
 
 
 void Parser::ReportMessageAt(Scanner::Location source_location,
                              const char* type,
                              Vector<const char*> args) {
   MessageLocation location(script_,
                            source_location.beg_pos, source_location.end_pos);
   Handle<JSArray> array = isolate()->factory()->NewJSArray(args.length());
@@ skipping 897 matching lines @@
 }
 
 
 Statement* Parser::ParseContinueStatement(bool* ok) {
   // ContinueStatement ::
   //   'continue' Identifier? ';'
 
   Expect(Token::CONTINUE, CHECK_OK);
   Handle<String> label = Handle<String>::null();
   Token::Value tok = peek();
-  if (!scanner_.has_line_terminator_before_next() &&
+  if (!scanner().has_line_terminator_before_next() &&
       tok != Token::SEMICOLON && tok != Token::RBRACE && tok != Token::EOS) {
     label = ParseIdentifier(CHECK_OK);
   }
   IterationStatement* target = NULL;
   target = LookupContinueTarget(label, CHECK_OK);
   if (target == NULL) {
     // Illegal continue statement.  To be consistent with KJS we delay
     // reporting of the syntax error until runtime.
     Handle<String> error_type = isolate()->factory()->illegal_continue_symbol();
     if (!label.is_null()) {
       error_type = isolate()->factory()->unknown_label_symbol();
     }
     Expression* throw_error = NewThrowSyntaxError(error_type, label);
     return new ExpressionStatement(throw_error);
   }
   ExpectSemicolon(CHECK_OK);
   return new ContinueStatement(target);
 }
 
 
 Statement* Parser::ParseBreakStatement(ZoneStringList* labels, bool* ok) {
   // BreakStatement ::
   //   'break' Identifier? ';'
 
   Expect(Token::BREAK, CHECK_OK);
   Handle<String> label;
   Token::Value tok = peek();
-  if (!scanner_.has_line_terminator_before_next() &&
+  if (!scanner().has_line_terminator_before_next() &&
       tok != Token::SEMICOLON && tok != Token::RBRACE && tok != Token::EOS) {
     label = ParseIdentifier(CHECK_OK);
   }
   // Parse labeled break statements that target themselves into
   // empty statements, e.g. 'l1: l2: l3: break l2;'
   if (!label.is_null() && ContainsLabel(labels, label)) {
     return EmptyStatement();
   }
   BreakableStatement* target = NULL;
   target = LookupBreakTarget(label, CHECK_OK);
@@ skipping 26 matching lines @@
   // function. See ECMA-262, section 12.9, page 67.
   //
   // To be consistent with KJS we report the syntax error at runtime.
   if (!top_scope_->is_function_scope()) {
     Handle<String> type = isolate()->factory()->illegal_return_symbol();
     Expression* throw_error = NewThrowSyntaxError(type, Handle<Object>::null());
     return new ExpressionStatement(throw_error);
   }
 
   Token::Value tok = peek();
-  if (scanner_.has_line_terminator_before_next() ||
+  if (scanner().has_line_terminator_before_next() ||
       tok == Token::SEMICOLON ||
       tok == Token::RBRACE ||
       tok == Token::EOS) {
     ExpectSemicolon(CHECK_OK);
     return new ReturnStatement(GetLiteralUndefined());
   }
 
   Expression* expr = ParseExpression(true, CHECK_OK);
   ExpectSemicolon(CHECK_OK);
   return new ReturnStatement(expr);
@@ skipping 111 matching lines @@
   return statement;
 }
 
 
 Statement* Parser::ParseThrowStatement(bool* ok) {
   // ThrowStatement ::
   //   'throw' Expression ';'
 
   Expect(Token::THROW, CHECK_OK);
   int pos = scanner().location().beg_pos;
-  if (scanner_.has_line_terminator_before_next()) {
+  if (scanner().has_line_terminator_before_next()) {
     ReportMessage("newline_after_throw", Vector<const char*>::empty());
     *ok = false;
     return NULL;
   }
   Expression* exception = ParseExpression(true, CHECK_OK);
   ExpectSemicolon(CHECK_OK);
 
   return new ExpressionStatement(new Throw(exception, pos));
 }
 
@@ skipping 547 matching lines @@
     return ParsePostfixExpression(ok);
   }
 }
 
 
 Expression* Parser::ParsePostfixExpression(bool* ok) {
   // PostfixExpression ::
   //   LeftHandSideExpression ('++' | '--')?
 
   Expression* expression = ParseLeftHandSideExpression(CHECK_OK);
-  if (!scanner_.has_line_terminator_before_next() && Token::IsCountOp(peek())) {
+  if (!scanner().has_line_terminator_before_next() &&
+      Token::IsCountOp(peek())) {
     // Signal a reference error if the expression is an invalid
     // left-hand side expression.  We could report this as a syntax
     // error here but for compatibility with JSC we choose to report the
     // error at runtime.
     if (expression == NULL || !expression->IsValidLeftHandSide()) {
       Handle<String> type =
           isolate()->factory()->invalid_lhs_in_postfix_op_symbol();
       expression = NewThrowReferenceError(type);
     }
     Token::Value next = Next();
@@ skipping 250 matching lines @@
     case Token::IDENTIFIER: {
       Handle<String> name = ParseIdentifier(CHECK_OK);
       if (fni_ != NULL) fni_->PushVariableName(name);
       result = top_scope_->NewUnresolved(name, inside_with());
       break;
     }
 
     case Token::NUMBER: {
       Consume(Token::NUMBER);
       double value =
-        StringToDouble(scanner_.literal(), ALLOW_HEX | ALLOW_OCTALS);
+        StringToDouble(scanner().literal(), ALLOW_HEX | ALLOW_OCTALS);
       result = NewNumberLiteral(value);
       break;
     }
 
     case Token::STRING: {
       Consume(Token::STRING);
       Handle<String> symbol = GetSymbol(CHECK_OK);
       result = new Literal(symbol);
       if (fni_ != NULL) fni_->PushLiteralName(symbol);
       break;
@@ skipping 330 matching lines @@
         if (!string.is_null() && string->AsArrayIndex(&index)) {
           key = NewNumberLiteral(index);
           break;
         }
         key = new Literal(string);
         break;
       }
       case Token::NUMBER: {
         Consume(Token::NUMBER);
         double value =
-          StringToDouble(scanner_.literal(), ALLOW_HEX | ALLOW_OCTALS);
+          StringToDouble(scanner().literal(), ALLOW_HEX | ALLOW_OCTALS);
         key = NewNumberLiteral(value);
         break;
       }
       default:
         if (Token::IsKeyword(next)) {
           Consume(next);
           Handle<String> string = GetSymbol(CHECK_OK);
           key = new Literal(string);
         } else {
           // Unexpected token.
@@ skipping 40 matching lines @@
   return new ObjectLiteral(constant_properties,
                            properties,
                            literal_index,
                            is_simple,
                            fast_elements,
                            depth);
 }
 
 
 Expression* Parser::ParseRegExpLiteral(bool seen_equal, bool* ok) {
-  if (!scanner_.ScanRegExpPattern(seen_equal)) {
+  if (!scanner().ScanRegExpPattern(seen_equal)) {
     Next();
     ReportMessage("unterminated_regexp", Vector<const char*>::empty());
     *ok = false;
     return NULL;
   }
 
   int literal_index = temp_scope_->NextMaterializedLiteralIndex();
 
   Handle<String> js_pattern =
-      isolate()->factory()->NewStringFromUtf8(scanner_.next_literal(), TENURED);
-  scanner_.ScanRegExpFlags();
+      isolate_->factory()->NewStringFromUtf8(scanner().next_literal(), TENURED);
+  scanner().ScanRegExpFlags();
   Handle<String> js_flags =
-      isolate()->factory()->NewStringFromUtf8(scanner_.next_literal(), TENURED);
+      isolate_->factory()->NewStringFromUtf8(scanner().next_literal(), TENURED);
   Next();
 
   return new RegExpLiteral(js_pattern, js_flags, literal_index);
 }
 
 
 ZoneList<Expression*>* Parser::ParseArguments(bool* ok) {
   // Arguments ::
   //   '(' (AssignmentExpression)*[','] ')'
 
@@ skipping 36 matching lines @@
   { Scope* scope =
         NewScope(top_scope_, Scope::FUNCTION_SCOPE, inside_with());
     LexicalScope lexical_scope(&this->top_scope_, &this->with_nesting_level_,
                                scope);
     TemporaryScope temp_scope(&this->temp_scope_);
     top_scope_->SetScopeName(name);
 
     //  FormalParameterList ::
     //    '(' (Identifier)*[','] ')'
     Expect(Token::LPAREN, CHECK_OK);
-    int start_pos = scanner_.location().beg_pos;
+    int start_pos = scanner().location().beg_pos;
     bool done = (peek() == Token::RPAREN);
     while (!done) {
       Handle<String> param_name = ParseIdentifier(CHECK_OK);
       top_scope_->AddParameter(top_scope_->DeclareLocal(param_name,
                                                         Variable::VAR));
       num_parameters++;
       done = (peek() == Token::RPAREN);
       if (!done) Expect(Token::COMMA, CHECK_OK);
     }
     Expect(Token::RPAREN, CHECK_OK);
@@ skipping 16 matching lines @@
                     new Assignment(Token::INIT_CONST, fproxy,
                                    new ThisFunction(),
                                    RelocInfo::kNoPosition)));
     }
 
     // Determine if the function will be lazily compiled. The mode can
     // only be PARSE_LAZILY if the --lazy flag is true.
     bool is_lazily_compiled =
         mode() == PARSE_LAZILY && top_scope_->HasTrivialOuterContext();
 
-    int function_block_pos = scanner_.location().beg_pos;
+    int function_block_pos = scanner().location().beg_pos;
     int materialized_literal_count;
     int expected_property_count;
     int end_pos;
     bool only_simple_this_property_assignments;
     Handle<FixedArray> this_property_assignments;
     if (is_lazily_compiled && pre_data() != NULL) {
       FunctionEntry entry = pre_data()->GetFunctionEntry(function_block_pos);
       if (!entry.is_valid()) {
         ReportInvalidPreparseData(name, CHECK_OK);
       }
       end_pos = entry.end_pos();
       if (end_pos <= function_block_pos) {
         // End position greater than end of stream is safe, and hard to check.
         ReportInvalidPreparseData(name, CHECK_OK);
       }
       COUNTERS->total_preparse_skipped()->Increment(
           end_pos - function_block_pos);
-      scanner_.SeekForward(end_pos);
+      // Seek to position just before terminal '}'.
+      scanner().SeekForward(end_pos - 1);
       materialized_literal_count = entry.literal_count();
       expected_property_count = entry.property_count();
       only_simple_this_property_assignments = false;
       this_property_assignments = isolate()->factory()->empty_fixed_array();
       Expect(Token::RBRACE, CHECK_OK);
     } else {
       ParseSourceElements(body, Token::RBRACE, CHECK_OK);
 
       materialized_literal_count = temp_scope.materialized_literal_count();
       expected_property_count = temp_scope.expected_property_count();
       only_simple_this_property_assignments =
           temp_scope.only_simple_this_property_assignments();
       this_property_assignments = temp_scope.this_property_assignments();
 
       Expect(Token::RBRACE, CHECK_OK);
-      end_pos = scanner_.location().end_pos;
+      end_pos = scanner().location().end_pos;
     }
 
     FunctionLiteral* function_literal =
         new FunctionLiteral(name,
                             top_scope_,
                             body,
                             materialized_literal_count,
                             expected_property_count,
                             only_simple_this_property_assignments,
                             this_property_assignments,
@@ skipping 83 matching lines @@
 
 
 void Parser::ExpectSemicolon(bool* ok) {
   // Check for automatic semicolon insertion according to
   // the rules given in ECMA-262, section 7.9, page 21.
   Token::Value tok = peek();
   if (tok == Token::SEMICOLON) {
     Next();
     return;
   }
-  if (scanner_.has_line_terminator_before_next() ||
+  if (scanner().has_line_terminator_before_next() ||
       tok == Token::RBRACE ||
       tok == Token::EOS) {
     return;
   }
   Expect(Token::SEMICOLON, ok);
 }
 
 
 Literal* Parser::GetLiteralUndefined() {
   return new Literal(isolate()->factory()->undefined_value());
@@ skipping 30 matching lines @@
 
 // This function reads an identifier and determines whether or not it
 // is 'get' or 'set'.  The reason for not using ParseIdentifier and
 // checking on the output is that this involves heap allocation which
 // we can't do during preparsing.
 Handle<String> Parser::ParseIdentifierOrGetOrSet(bool* is_get,
                                                  bool* is_set,
                                                  bool* ok) {
   Expect(Token::IDENTIFIER, ok);
   if (!*ok) return Handle<String>();
-  if (scanner_.literal_length() == 3) {
-    const char* token = scanner_.literal_string();
+  if (scanner().literal_length() == 3) {
+    const char* token = scanner().literal_string();
     *is_get = strcmp(token, "get") == 0;
     *is_set = !*is_get && strcmp(token, "set") == 0;
   }
   return GetSymbol(ok);
 }
 
 
 // ----------------------------------------------------------------------------
 // Parser support
 
@@ skipping 100 matching lines @@
   ZoneList<Expression*>* args = new ZoneList<Expression*>(2);
   args->Add(new Literal(type));
   args->Add(new Literal(array));
   return new Throw(new CallRuntime(constructor, NULL, args),
                    scanner().location().beg_pos);
 }
 
 // ----------------------------------------------------------------------------
 // JSON
 
-Handle<Object> JsonParser::ParseJson(Handle<String> source) {
-  source->TryFlatten();
+Handle<Object> JsonParser::ParseJson(Handle<String> script,
+                                     UC16CharacterStream* source) {
   scanner_.Initialize(source);
   stack_overflow_ = false;
   Handle<Object> result = ParseJsonValue();
   if (result.is_null() || scanner_.Next() != Token::EOS) {
     if (stack_overflow_) {
       // Scanner failed.
       isolate()->StackOverflow();
     } else {
       // Parse failed. Scanner's current token is the unexpected token.
       Token::Value token = scanner_.current_token();
@@ skipping 15 matching lines @@
           message = "unexpected_token_identifier";
           break;
         default:
           message = "unexpected_token";
           name_opt = Token::String(token);
           ASSERT(name_opt != NULL);
           break;
       }
 
       Scanner::Location source_location = scanner_.location();
-      MessageLocation location(isolate()->factory()->NewScript(source),
+      MessageLocation location(isolate()->factory()->NewScript(script),
                                source_location.beg_pos,
                                source_location.end_pos);
       int argc = (name_opt == NULL) ? 0 : 1;
       Handle<JSArray> array = isolate()->factory()->NewJSArray(argc);
       if (name_opt != NULL) {
         SetElement(
             array,
             0,
             isolate()->factory()->NewStringFromUtf8(CStrVector(name_opt)));
       }
@@ skipping 133 matching lines @@
       in_(in),
       current_(kEndMarker),
       next_pos_(0),
       capture_count_(0),
       has_more_(true),
       multiline_(multiline),
       simple_(false),
       contains_anchor_(false),
       is_scanned_for_captures_(false),
       failed_(false) {
-  Advance(1);
+  Advance();
 }
 
 
 uc32 RegExpParser::Next() {
   if (has_next()) {
     return in()->Get(next_pos_);
   } else {
     return kEndMarker;
   }
 }
@@ skipping 17 matching lines @@
 }
 
 
 void RegExpParser::Reset(int pos) {
   next_pos_ = pos;
   Advance();
 }
 
 
 void RegExpParser::Advance(int dist) {
-  for (int i = 0; i < dist; i++)
-    Advance();
+  next_pos_ += dist - 1;
+  Advance();
 }
 
 
 bool RegExpParser::simple() {
   return simple_;
 }
 
 RegExpTree* RegExpParser::ReportError(Vector<const char> message) {
   failed_ = true;
   *error_ = isolate()->factory()->NewStringFromAscii(message, NOT_TENURED);
@@ skipping 635 matching lines @@
         uc32 c = ParseClassCharacterEscape(CHECK_FAILED);
         return CharacterRange::Singleton(c);
     }
   } else {
     Advance();
     return CharacterRange::Singleton(first);
   }
 }
 
 
+static const uc16 kNoCharClass = 0;
+
+// Adds range or pre-defined character class to character ranges.
+// If char_class is not kInvalidClass, it's interpreted as a class
+// escape (i.e., 's' means whitespace, from '\s').
+static inline void AddRangeOrEscape(ZoneList<CharacterRange>* ranges,
+                                    uc16 char_class,
+                                    CharacterRange range) {
+  if (char_class != kNoCharClass) {
+    CharacterRange::AddClassEscape(char_class, ranges);
+  } else {
+    ranges->Add(range);
+  }
+}
+
+
 RegExpTree* RegExpParser::ParseCharacterClass() {
   static const char* kUnterminated = "Unterminated character class";
   static const char* kRangeOutOfOrder = "Range out of order in character class";
-  static const char* kInvalidRange = "Invalid character range";
 
   ASSERT_EQ(current(), '[');
   Advance();
   bool is_negated = false;
   if (current() == '^') {
     is_negated = true;
     Advance();
   }
-  // A CharacterClass is a sequence of single characters, character class
-  // escapes or ranges. Ranges are on the form "x-y" where x and y are
-  // single characters (and not character class escapes like \s).
-  // A "-" may occur at the start or end of the character class (just after
-  // "[" or "[^", or just before "]") without being considered part of a
-  // range. A "-" may also appear as the beginning or end of a range.
-  // I.e., [--+] is valid, so is [!--].
-
   ZoneList<CharacterRange>* ranges = new ZoneList<CharacterRange>(2);
   while (has_more() && current() != ']') {
-    uc16 char_class = 0;
+    uc16 char_class = kNoCharClass;
     CharacterRange first = ParseClassAtom(&char_class CHECK_FAILED);
-    if (char_class) {
-      CharacterRange::AddClassEscape(char_class, ranges);
-      if (current() == '-') {
-        Advance();
-        ranges->Add(CharacterRange::Singleton('-'));
-        if (current() != ']') {
-          ReportError(CStrVector(kInvalidRange) CHECK_FAILED);
-        }
-        break;
-      }
-      continue;
-    }
     if (current() == '-') {
       Advance();
       if (current() == kEndMarker) {
         // If we reach the end we break out of the loop and let the
         // following code report an error.
         break;
       } else if (current() == ']') {
-        ranges->Add(first);
+        AddRangeOrEscape(ranges, char_class, first);
         ranges->Add(CharacterRange::Singleton('-'));
         break;
       }
-      CharacterRange next = ParseClassAtom(&char_class CHECK_FAILED);
-      if (char_class) {
-        ReportError(CStrVector(kInvalidRange) CHECK_FAILED);
+      uc16 char_class_2 = kNoCharClass;
+      CharacterRange next = ParseClassAtom(&char_class_2 CHECK_FAILED);
+      if (char_class != kNoCharClass || char_class_2 != kNoCharClass) {
+        // Either end is an escaped character class. Treat the '-' verbatim.
+        AddRangeOrEscape(ranges, char_class, first);
+        ranges->Add(CharacterRange::Singleton('-'));
+        AddRangeOrEscape(ranges, char_class_2, next);
+        continue;
       }
       if (first.from() > next.to()) {
         return ReportError(CStrVector(kRangeOutOfOrder) CHECK_FAILED);
       }
       ranges->Add(CharacterRange::Range(first.from(), next.to()));
     } else {
-      ranges->Add(first);
+      AddRangeOrEscape(ranges, char_class, first);
     }
   }
   if (!has_more()) {
     return ReportError(CStrVector(kUnterminated) CHECK_FAILED);
   }
   Advance();
   if (ranges->length() == 0) {
     ranges->Add(CharacterRange::Everything());
     is_negated = !is_negated;
   }
@@ skipping 69 matching lines @@
     input = *data;
     result = (result << 7) | (input & 0x7f);
     data++;
   }
   *source = data;
   return result;
 }
 
 
 // Create a Scanner for the preparser to use as input, and preparse the source.
-static ScriptDataImpl* DoPreParse(Handle<String> source,
-                                  unibrow::CharacterStream* stream,
+static ScriptDataImpl* DoPreParse(UC16CharacterStream* source,
                                   bool allow_lazy,
                                   ParserRecorder* recorder,
                                   int literal_flags) {
   Isolate* isolate = Isolate::Current();
   V8JavaScriptScanner scanner(isolate);
-  scanner.Initialize(source, stream, literal_flags);
+  scanner.Initialize(source, literal_flags);
   intptr_t stack_limit = isolate->stack_guard()->real_climit();
   if (!preparser::PreParser::PreParseProgram(&scanner,
                                              recorder,
                                              allow_lazy,
                                              stack_limit)) {
     isolate->StackOverflow();
     return NULL;
   }
 
   // Extract the accumulated data from the recorder as a single
   // contiguous vector that we are responsible for disposing.
   Vector<unsigned> store = recorder->ExtractData();
   return new ScriptDataImpl(store);
 }
 
 
 // Preparse, but only collect data that is immediately useful,
 // even if the preparser data is only used once.
-ScriptDataImpl* ParserApi::PartialPreParse(Handle<String> source,
-                                           unibrow::CharacterStream* stream,
+ScriptDataImpl* ParserApi::PartialPreParse(UC16CharacterStream* source,
                                            v8::Extension* extension) {
   bool allow_lazy = FLAG_lazy && (extension == NULL);
   if (!allow_lazy) {
     // Partial preparsing is only about lazily compiled functions.
     // If we don't allow lazy compilation, the log data will be empty.
     return NULL;
   }
   PartialParserRecorder recorder;
-
-  return DoPreParse(source, stream, allow_lazy, &recorder,
+  return DoPreParse(source, allow_lazy, &recorder,
                     JavaScriptScanner::kNoLiterals);
 }
 
 
-ScriptDataImpl* ParserApi::PreParse(Handle<String> source,
-                                    unibrow::CharacterStream* stream,
+ScriptDataImpl* ParserApi::PreParse(UC16CharacterStream* source,
                                     v8::Extension* extension) {
   Handle<Script> no_script;
   bool allow_lazy = FLAG_lazy && (extension == NULL);
   CompleteParserRecorder recorder;
   int kPreParseLiteralsFlags =
       JavaScriptScanner::kLiteralString | JavaScriptScanner::kLiteralIdentifier;
-  return DoPreParse(source, stream, allow_lazy,
-                    &recorder, kPreParseLiteralsFlags);
+  return DoPreParse(source, allow_lazy, &recorder, kPreParseLiteralsFlags);
 }
 
 
 bool RegExpParser::ParseRegExp(FlatStringReader* input,
                                bool multiline,
                                RegExpCompileData* result) {
   ASSERT(result != NULL);
   RegExpParser parser(input, &result->error, multiline);
   RegExpTree* tree = parser.ParsePattern();
   if (parser.failed()) {
@@ skipping 40 matching lines @@
       Handle<String> source = Handle<String>(String::cast(script->source()));
       result = parser.ParseProgram(source, info->is_global());
     }
   }
 
   info->SetFunction(result);
   return (result != NULL);
 }
 
 } }  // namespace v8::internal