Add literal handling to experimental scanner.

src/preparser.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 86 matching lines @@
 // That means that contextual checks (like a label being declared where
 // it is used) are generally omitted.
 
 void PreParser::ReportUnexpectedToken(Token::Value token) {
   // We don't report stack overflows here, to avoid increasing the
   // stack depth even further.  Instead we report it after parsing is
   // over, in ParseProgram.
   if (token == Token::ILLEGAL && stack_overflow()) {
     return;
   }
-  Scanner::Location source_location = scanner()->location();
+  ScannerBase::Location source_location = scanner()->location();
 
   // Four of the tokens are treated specially
   switch (token) {
   case Token::EOS:
     return ReportMessageAt(source_location, "unexpected_eos", NULL);
   case Token::NUMBER:
     return ReportMessageAt(source_location, "unexpected_token_number", NULL);
   case Token::STRING:
     return ReportMessageAt(source_location, "unexpected_token_string", NULL);
   case Token::IDENTIFIER:
@@ skipping 137 matching lines @@
     case Token::SWITCH:
       return ParseSwitchStatement(ok);
 
     case Token::THROW:
       return ParseThrowStatement(ok);
 
     case Token::TRY:
       return ParseTryStatement(ok);
 
     case Token::FUNCTION: {
-      Scanner::Location start_location = scanner()->peek_location();
+      ScannerBase::Location start_location = scanner()->peek_location();
       Statement statement = ParseFunctionDeclaration(CHECK_OK);
-      Scanner::Location end_location = scanner()->location();
+      ScannerBase::Location end_location = scanner()->location();
       if (!is_classic_mode()) {
         ReportMessageAt(start_location.beg_pos, end_location.end_pos,
                         "strict_function", NULL);
         *ok = false;
         return Statement::Default();
       } else {
         return statement;
       }
     }
 
     case Token::DEBUGGER:
       return ParseDebuggerStatement(ok);
 
     default:
       return ParseExpressionOrLabelledStatement(ok);
   }
 }
 
 
 PreParser::Statement PreParser::ParseFunctionDeclaration(bool* ok) {
   // FunctionDeclaration ::
   //   'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}'
   // GeneratorDeclaration ::
   //   'function' '*' Identifier '(' FormalParameterListopt ')'
   //      '{' FunctionBody '}'
   Expect(Token::FUNCTION, CHECK_OK);
 
   bool is_generator = allow_generators() && Check(Token::MUL);
   Identifier identifier = ParseIdentifier(CHECK_OK);
-  Scanner::Location location = scanner()->location();
+  ScannerBase::Location location = scanner()->location();
 
   Expression function_value = ParseFunctionLiteral(is_generator, CHECK_OK);
 
   if (function_value.IsStrictFunction() &&
       !identifier.IsValidStrictVariable()) {
     // Strict mode violation, using either reserved word or eval/arguments
     // as name of strict function.
     const char* type = "strict_function_name";
     if (identifier.IsFutureStrictReserved() || identifier.IsYield()) {
       type = "strict_reserved_word";
@@ skipping 75 matching lines @@
     //   contained in extended code.
     //
     // However disallowing const in classic mode will break compatibility with
     // existing pages. Therefore we keep allowing const with the old
     // non-harmony semantics in classic mode.
     Consume(Token::CONST);
     switch (language_mode()) {
       case CLASSIC_MODE:
         break;
       case STRICT_MODE: {
-        Scanner::Location location = scanner()->peek_location();
+        ScannerBase::Location location = scanner()->peek_location();
         ReportMessageAt(location, "strict_const", NULL);
         *ok = false;
         return Statement::Default();
       }
       case EXTENDED_MODE:
         if (var_context != kSourceElement &&
             var_context != kForStatement) {
-          Scanner::Location location = scanner()->peek_location();
+          ScannerBase::Location location = scanner()->peek_location();
           ReportMessageAt(location.beg_pos, location.end_pos,
                           "unprotected_const", NULL);
           *ok = false;
           return Statement::Default();
         }
         require_initializer = true;
         break;
     }
   } else if (peek() == Token::LET) {
     // ES6 Draft Rev4 section 12.2.1:
     //
     // LetDeclaration : let LetBindingList ;
     //
     // * It is a Syntax Error if the code that matches this production is not
     //   contained in extended code.
     if (!is_extended_mode()) {
-      Scanner::Location location = scanner()->peek_location();
+      ScannerBase::Location location = scanner()->peek_location();
       ReportMessageAt(location.beg_pos, location.end_pos,
                       "illegal_let", NULL);
       *ok = false;
       return Statement::Default();
     }
     Consume(Token::LET);
     if (var_context != kSourceElement &&
         var_context != kForStatement) {
-      Scanner::Location location = scanner()->peek_location();
+      ScannerBase::Location location = scanner()->peek_location();
       ReportMessageAt(location.beg_pos, location.end_pos,
                       "unprotected_let", NULL);
       *ok = false;
       return Statement::Default();
     }
   } else {
     *ok = false;
     return Statement::Default();
   }
 
@@ skipping 126 matching lines @@
   ExpectSemicolon(CHECK_OK);
   return Statement::Default();
 }
 
 
 PreParser::Statement PreParser::ParseWithStatement(bool* ok) {
   // WithStatement ::
   //   'with' '(' Expression ')' Statement
   Expect(Token::WITH, CHECK_OK);
   if (!is_classic_mode()) {
-    Scanner::Location location = scanner()->location();
+    ScannerBase::Location location = scanner()->location();
     ReportMessageAt(location, "strict_mode_with", NULL);
     *ok = false;
     return Statement::Default();
   }
   Expect(Token::LPAREN, CHECK_OK);
   ParseExpression(true, CHECK_OK);
   Expect(Token::RPAREN, CHECK_OK);
 
   Scope::InsideWith iw(scope_);
   ParseStatement(CHECK_OK);
@@ skipping 124 matching lines @@
   return Statement::Default();
 }
 
 
 PreParser::Statement PreParser::ParseThrowStatement(bool* ok) {
   // ThrowStatement ::
   //   'throw' [no line terminator] Expression ';'
 
   Expect(Token::THROW, CHECK_OK);
   if (scanner()->HasAnyLineTerminatorBeforeNext()) {
-    Scanner::Location pos = scanner()->location();
+    ScannerBase::Location pos = scanner()->location();
     ReportMessageAt(pos, "newline_after_throw", NULL);
     *ok = false;
     return Statement::Default();
   }
   ParseExpression(true, CHECK_OK);
   ExpectSemicolon(ok);
   return Statement::Default();
 }
 
 
@@ skipping 88 matching lines @@
                                                            bool* ok) {
   // AssignmentExpression ::
   //   ConditionalExpression
   //   YieldExpression
   //   LeftHandSideExpression AssignmentOperator AssignmentExpression
 
   if (scope_->is_generator() && peek() == Token::YIELD) {
     return ParseYieldExpression(ok);
   }
 
-  Scanner::Location before = scanner()->peek_location();
+  ScannerBase::Location before = scanner()->peek_location();
   Expression expression = ParseConditionalExpression(accept_IN, CHECK_OK);
 
   if (!Token::IsAssignmentOp(peek())) {
     // Parsed conditional expression only (no assignment).
     return expression;
   }
 
   if (!is_classic_mode() &&
       expression.IsIdentifier() &&
       expression.AsIdentifier().IsEvalOrArguments()) {
-    Scanner::Location after = scanner()->location();
+    ScannerBase::Location after = scanner()->location();
     ReportMessageAt(before.beg_pos, after.end_pos,
                     "strict_lhs_assignment", NULL);
     *ok = false;
     return Expression::Default();
   }
 
   Token::Value op = Next();  // Get assignment operator.
   ParseAssignmentExpression(accept_IN, CHECK_OK);
 
   if ((op == Token::ASSIGN) && expression.IsThisProperty()) {
@@ skipping 68 matching lines @@
   //   '~' UnaryExpression
   //   '!' UnaryExpression
 
   Token::Value op = peek();
   if (Token::IsUnaryOp(op)) {
     op = Next();
     ParseUnaryExpression(ok);
     return Expression::Default();
   } else if (Token::IsCountOp(op)) {
     op = Next();
-    Scanner::Location before = scanner()->peek_location();
+    ScannerBase::Location before = scanner()->peek_location();
     Expression expression = ParseUnaryExpression(CHECK_OK);
     if (!is_classic_mode() &&
         expression.IsIdentifier() &&
         expression.AsIdentifier().IsEvalOrArguments()) {
-      Scanner::Location after = scanner()->location();
+      ScannerBase::Location after = scanner()->location();
       ReportMessageAt(before.beg_pos, after.end_pos,
                       "strict_lhs_prefix", NULL);
       *ok = false;
     }
     return Expression::Default();
   } else {
     return ParsePostfixExpression(ok);
   }
 }
 
 
 PreParser::Expression PreParser::ParsePostfixExpression(bool* ok) {
   // PostfixExpression ::
   //   LeftHandSideExpression ('++' | '--')?
 
-  Scanner::Location before = scanner()->peek_location();
+  ScannerBase::Location before = scanner()->peek_location();
   Expression expression = ParseLeftHandSideExpression(CHECK_OK);
   if (!scanner()->HasAnyLineTerminatorBeforeNext() &&
       Token::IsCountOp(peek())) {
     if (!is_classic_mode() &&
         expression.IsIdentifier() &&
         expression.AsIdentifier().IsEvalOrArguments()) {
-      Scanner::Location after = scanner()->location();
+      ScannerBase::Location after = scanner()->location();
       ReportMessageAt(before.beg_pos, after.end_pos,
                       "strict_lhs_postfix", NULL);
       *ok = false;
       return Expression::Default();
     }
     Next();
     return Expression::Default();
   }
   return expression;
 }
@@ skipping 519 matching lines @@
     }
   }
   return Identifier::Default();
 }
 
 
 PreParser::Identifier PreParser::ParseIdentifier(bool* ok) {
   Token::Value next = Next();
   switch (next) {
     case Token::FUTURE_RESERVED_WORD: {
-      Scanner::Location location = scanner()->location();
+      ScannerBase::Location location = scanner()->location();
       ReportMessageAt(location.beg_pos, location.end_pos,
                       "reserved_word", NULL);
       *ok = false;
       return GetIdentifierSymbol();
     }
     case Token::YIELD:
       if (scope_->is_generator()) {
         // 'yield' in a generator is only valid as part of a YieldExpression.
         ReportMessageAt(scanner()->location(), "unexpected_token", "yield");
         *ok = false;
         return Identifier::Yield();
       }
       // FALLTHROUGH
     case Token::FUTURE_STRICT_RESERVED_WORD:
       if (!is_classic_mode()) {
-        Scanner::Location location = scanner()->location();
+        ScannerBase::Location location = scanner()->location();
         ReportMessageAt(location.beg_pos, location.end_pos,
                         "strict_reserved_word", NULL);
         *ok = false;
       }
       // FALLTHROUGH
     case Token::IDENTIFIER:
       return GetIdentifierSymbol();
     default:
       *ok = false;
       return Identifier::Default();
   }
 }
 
 
-void PreParser::SetStrictModeViolation(Scanner::Location location,
+void PreParser::SetStrictModeViolation(ScannerBase::Location location,
                                        const char* type,
                                        bool* ok) {
   if (!is_classic_mode()) {
     ReportMessageAt(location, type, NULL);
     *ok = false;
     return;
   }
   // Delay report in case this later turns out to be strict code
   // (i.e., for function names and parameters prior to a "use strict"
   // directive).
   // It's safe to overwrite an existing violation.
   // It's either from a function that turned out to be non-strict,
   // or it's in the current function (and we just need to report
   // one error), or it's in a unclosed nesting function that wasn't
   // strict (otherwise we would already be in strict mode).
   strict_mode_violation_location_ = location;
   strict_mode_violation_type_ = type;
 }
 
 
 void PreParser::CheckDelayedStrictModeViolation(int beg_pos,
                                                 int end_pos,
                                                 bool* ok) {
-  Scanner::Location location = strict_mode_violation_location_;
+  ScannerBase::Location location = strict_mode_violation_location_;
   if (location.IsValid() &&
       location.beg_pos > beg_pos && location.end_pos < end_pos) {
     ReportMessageAt(location, strict_mode_violation_type_, NULL);
     *ok = false;
   }
 }
 
 
-void PreParser::StrictModeIdentifierViolation(Scanner::Location location,
+void PreParser::StrictModeIdentifierViolation(ScannerBase::Location location,
                                               const char* eval_args_type,
                                               Identifier identifier,
                                               bool* ok) {
   const char* type = eval_args_type;
   if (identifier.IsFutureReserved()) {
     type = "reserved_word";
   } else if (identifier.IsFutureStrictReserved() || identifier.IsYield()) {
     type = "strict_reserved_word";
   }
   if (!is_classic_mode()) {
@@ skipping 69 matching lines @@
       ASSERT(IsAccessorAccessorConflict(old_type, type));
       // Both accessors of the same type.
       parser()->ReportMessageAt(scanner()->location(),
                                "accessor_get_set");
     }
     *ok = false;
   }
 }
 
 } }  // v8::internal