Merge preparser Scanner with main JavaScript scanner.

src/scanner-base.h

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 309 matching lines @@
 
   // One Unicode character look-ahead; c0_ < 0 at the end of the input.
   uc32 c0_;
 };
 
 // ----------------------------------------------------------------------------
 // JavaScriptScanner - base logic for JavaScript scanning.
 
 class JavaScriptScanner : public Scanner {
  public:
+
+  enum LiteralType {
+    kLiteralNumber = 1,
+    kLiteralIdentifier = 2,
+    kLiteralString = 4,
+    kLiteralRegExp = 8,
+    kLiteralRegExpFlags = 16
+  };
+
+  // More specialized literal scope.

[Søren Thygesen Gjesse, 2010/11/19 08:17:21]

More specialized - in what sense?

[Lasse Reichstein, 2010/11/19 08:51:24]

Elaborated.

+  class LiteralScope {
+   public:
+    LiteralScope(JavaScriptScanner* self, LiteralType type)
+        : scanner_(self), complete_(false) {
+      if (scanner_->RecordsLiteral(type)) {
+        scanner_->StartLiteral();
+      }
+    }
+     ~LiteralScope() {
+       if (!complete_) scanner_->DropLiteral();
+     }
+    void Complete() {
+      scanner_->TerminateLiteral();
+      complete_ = true;
+    }
+
+   private:
+    JavaScriptScanner* scanner_;
+    bool complete_;
+  };
+
   JavaScriptScanner();
 
   // Returns the next token.
   Token::Value Next();
 
   // Returns true if there was a line terminator before the peek'ed token.
   bool has_line_terminator_before_next() const {
     return has_line_terminator_before_next_;
   }
 
   // Scans the input as a regular expression pattern, previous
   // character(s) must be /(=). Returns true if a pattern is scanned.
   bool ScanRegExpPattern(bool seen_equal);
   // Returns true if regexp flags are scanned (always since flags can
   // be empty).
   bool ScanRegExpFlags();
 
   // Tells whether the buffer contains an identifier (no escapes).
   // Used for checking if a property name is an identifier.
   static bool IsIdentifier(unibrow::CharacterStream* buffer);
 
   // Seek forward to the given position.  This operation does not
   // work in general, for instance when there are pushed back
   // characters, but works for seeking forward until simple delimiter
   // tokens, which is what it is used for.
   void SeekForward(int pos);
 
+  // Whether this scanner records the given literal type or not.
+  bool RecordsLiteral(LiteralType type) {
+    return (literal_flags_ & type) != 0;
+  }
+
  protected:
   bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
   Token::Value SkipMultiLineComment();
 
   // Scans a single JavaScript token.
   void Scan();
 
   void ScanDecimalDigits();
   Token::Value ScanNumber(bool seen_period);
-  Token::Value ScanIdentifier();
+  Token::Value ScanIdentifierOrKeyword();
+  Token::Value ScanIdentifierSuffix(LiteralScope* literal);
 
   void ScanEscape();
   Token::Value ScanString();
 
   // Scans a possible HTML comment -- begins with '<!'.
   Token::Value ScanHtmlComment();
 
   // Decodes a unicode escape-sequence which is part of an identifier.
   // If the escape sequence cannot be decoded the result is kBadChar.
   uc32 ScanIdentifierUnicodeEscape();
 
+  int literal_flags_;
   bool has_line_terminator_before_next_;
 };
 
 
 // ----------------------------------------------------------------------------
 // Keyword matching state machine.
 
 class KeywordMatcher {
 //  Incrementally recognize keywords.
 //
 //  Recognized keywords:
 //      break case catch const* continue debugger* default delete do else
 //      finally false for function if in instanceof native* new null
 //      return switch this throw true try typeof var void while with
 //
 //  *: Actually "future reserved keywords". These are the only ones we
 //     recognize, the remaining are allowed as identifiers.
 //     In ES5 strict mode, we should disallow all reserved keywords.
  public:
   KeywordMatcher()
       : state_(INITIAL),
         token_(Token::IDENTIFIER),
         keyword_(NULL),
         counter_(0),
         keyword_token_(Token::ILLEGAL) {}
 
   Token::Value token() { return token_; }
 
-  inline void AddChar(unibrow::uchar input) {
+  inline bool AddChar(unibrow::uchar input) {
     if (state_ != UNMATCHABLE) {
       Step(input);
     }
+    return state_ != UNMATCHABLE;
   }
 
   void Fail() {
     token_ = Token::IDENTIFIER;
     state_ = UNMATCHABLE;
   }
 
  private:
   enum State {
     UNMATCHABLE,
@@ skipping 30 matching lines @@
       kFirstCharRangeMax - kFirstCharRangeMin + 1;
   // State map for first keyword character range.
   static FirstState first_states_[kFirstCharRangeLength];
 
   // If input equals keyword's character at position, continue matching keyword
   // from that position.
   inline bool MatchKeywordStart(unibrow::uchar input,
                                 const char* keyword,
                                 int position,
                                 Token::Value token_if_match) {
-    if (input == static_cast<unibrow::uchar>(keyword[position])) {
-      state_ = KEYWORD_PREFIX;
-      this->keyword_ = keyword;
-      this->counter_ = position + 1;
-      this->keyword_token_ = token_if_match;
-      return true;
+    if (input != static_cast<unibrow::uchar>(keyword[position])) {
+      return false;
     }
-    return false;
+    state_ = KEYWORD_PREFIX;
+    this->keyword_ = keyword;
+    this->counter_ = position + 1;
+    this->keyword_token_ = token_if_match;
+    return true;
   }
 
   // If input equals match character, transition to new state and return true.
   inline bool MatchState(unibrow::uchar input, char match, State new_state) {
-    if (input == static_cast<unibrow::uchar>(match)) {
-      state_ = new_state;
-      return true;
+    if (input != static_cast<unibrow::uchar>(match)) {
+      return false;
     }
-    return false;
+    state_ = new_state;
+    return true;
   }
 
   inline bool MatchKeyword(unibrow::uchar input,
                            char match,
-                           State new_state,
                            Token::Value keyword_token) {
     if (input != static_cast<unibrow::uchar>(match)) {
       return false;
     }
-    state_ = new_state;
+    state_ = KEYWORD_MATCHED;
     token_ = keyword_token;
     return true;
   }
 
   void Step(unibrow::uchar input);
 
   // Current state.
   State state_;
   // Token for currently added characters.
   Token::Value token_;
 
   // Matching a specific keyword string (there is only one possible valid
   // keyword with the current prefix).
   const char* keyword_;
   int counter_;
   Token::Value keyword_token_;
 };
 
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_BASE_H_