Merge preparser Scanner with main JavaScript scanner.

src/scanner-base.h

Index: src/scanner-base.h
diff --git a/src/scanner-base.h b/src/scanner-base.h
index 61b1fddda9331837f122028eda31972b80b6177c..19bb225ce16be189f158dfa1beddb600050c87f2 100644
--- a/src/scanner-base.h
+++ b/src/scanner-base.h
@@ -327,6 +327,37 @@ class Scanner {
 
 class JavaScriptScanner : public Scanner {
  public:
+
+  enum LiteralType {
+    kLiteralNumber = 1,
+    kLiteralIdentifier = 2,
+    kLiteralString = 4,
+    kLiteralRegExp = 8,
+    kLiteralRegExpFlags = 16
+  };
+
+  // More specialized literal scope.

[Søren Thygesen Gjesse, 2010/11/19 08:17:21]

More specialized - in what sense?

[Lasse Reichstein, 2010/11/19 08:51:24]

Elaborated.

+  class LiteralScope {
+   public:
+    LiteralScope(JavaScriptScanner* self, LiteralType type)
+        : scanner_(self), complete_(false) {
+      if (scanner_->RecordsLiteral(type)) {
+        scanner_->StartLiteral();
+      }
+    }
+     ~LiteralScope() {
+       if (!complete_) scanner_->DropLiteral();
+     }
+    void Complete() {
+      scanner_->TerminateLiteral();
+      complete_ = true;
+    }
+
+   private:
+    JavaScriptScanner* scanner_;
+    bool complete_;
+  };
+
   JavaScriptScanner();
 
   // Returns the next token.
@@ -354,6 +385,11 @@ class JavaScriptScanner : public Scanner {
   // tokens, which is what it is used for.
   void SeekForward(int pos);
 
+  // Whether this scanner records the given literal type or not.
+  bool RecordsLiteral(LiteralType type) {
+    return (literal_flags_ & type) != 0;
+  }
+
  protected:
   bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
@@ -364,7 +400,8 @@ class JavaScriptScanner : public Scanner {
 
   void ScanDecimalDigits();
   Token::Value ScanNumber(bool seen_period);
-  Token::Value ScanIdentifier();
+  Token::Value ScanIdentifierOrKeyword();
+  Token::Value ScanIdentifierSuffix(LiteralScope* literal);
 
   void ScanEscape();
   Token::Value ScanString();
@@ -376,6 +413,7 @@ class JavaScriptScanner : public Scanner {
   // If the escape sequence cannot be decoded the result is kBadChar.
   uc32 ScanIdentifierUnicodeEscape();
 
+  int literal_flags_;
   bool has_line_terminator_before_next_;
 };
 
@@ -404,10 +442,11 @@ class KeywordMatcher {
 
   Token::Value token() { return token_; }
 
-  inline void AddChar(unibrow::uchar input) {
+  inline bool AddChar(unibrow::uchar input) {
     if (state_ != UNMATCHABLE) {
       Step(input);
     }
+    return state_ != UNMATCHABLE;
   }
 
   void Fail() {
@@ -458,33 +497,32 @@ class KeywordMatcher {
                                 const char* keyword,
                                 int position,
                                 Token::Value token_if_match) {
-    if (input == static_cast<unibrow::uchar>(keyword[position])) {
-      state_ = KEYWORD_PREFIX;
-      this->keyword_ = keyword;
-      this->counter_ = position + 1;
-      this->keyword_token_ = token_if_match;
-      return true;
+    if (input != static_cast<unibrow::uchar>(keyword[position])) {
+      return false;
     }
-    return false;
+    state_ = KEYWORD_PREFIX;
+    this->keyword_ = keyword;
+    this->counter_ = position + 1;
+    this->keyword_token_ = token_if_match;
+    return true;
   }
 
   // If input equals match character, transition to new state and return true.
   inline bool MatchState(unibrow::uchar input, char match, State new_state) {
-    if (input == static_cast<unibrow::uchar>(match)) {
-      state_ = new_state;
-      return true;
+    if (input != static_cast<unibrow::uchar>(match)) {
+      return false;
     }
-    return false;
+    state_ = new_state;
+    return true;
   }
 
   inline bool MatchKeyword(unibrow::uchar input,
                            char match,
-                           State new_state,
                            Token::Value keyword_token) {
     if (input != static_cast<unibrow::uchar>(match)) {
       return false;
     }
-    state_ = new_state;
+    state_ = KEYWORD_MATCHED;
     token_ = keyword_token;
     return true;
   }