Implement handling of arrow functions in the parser

src/scanner.h

Index: src/scanner.h
diff --git a/src/scanner.h b/src/scanner.h
index 7d8b2ab4f222a428559b340f0cac1d16c872f714..7affb229e889d57b0f7222199f3a341c0d38414a 100644
--- a/src/scanner.h
+++ b/src/scanner.h
@@ -189,6 +189,54 @@ class DuplicateFinder {
 
 
 // ----------------------------------------------------------------------------
+// ParamListFinder discovers sequences of tokens which form a valid function
+// parameter list.
+class Scanner;
+
+class ParamListFinder {
+ public:
+  explicit ParamListFinder(UnicodeCache* unicode_cache)
+      : state_(Invalid)
+      , start_pos_(-1)
+      , identifier_pos_(-1)
+      , duplicate_pos_(-1)
+      , duplicate_finder_(unicode_cache) { }
+
+  V8_INLINE void Update(Scanner* scanner);
+
+  bool IsValid(int pos) const {
+    return (state_ == Valid && start_pos_ == pos)
+        || (identifier_pos_ == pos);
+  }
+
+  bool HasDuplicateIdentifiers() const {
+    return duplicate_pos_ > start_pos_;
+  }
+
+  int FirstDuplicatePosition() const {
+    return duplicate_pos_;
+  }
+
+ private:
+  enum State {
+    Invalid,
+    Valid,
+    LeftParen,
+    Identifier,
+    Comma
+  };
+
+  V8_INLINE void AddIdentifier(Scanner* scanner);
+
+  State state_;
+  int start_pos_;
+  int identifier_pos_;
+  int duplicate_pos_;
+  DuplicateFinder duplicate_finder_;
+};
+
+
+// ----------------------------------------------------------------------------
 // LiteralBuffer -  Collector of chars of literals.
 
 class LiteralBuffer {
@@ -403,6 +451,10 @@ class Scanner {
     }
   }
 
+  ParamListFinder* parameter_list() {
+      return &param_list_finder_;
+  }
+
   int FindNumber(DuplicateFinder* finder, int value);
   int FindSymbol(DuplicateFinder* finder, int value);
 
@@ -451,6 +503,8 @@ class Scanner {
   // be empty).
   bool ScanRegExpFlags();
 
+  bool IdentifierIsFutureStrictReserved(const AstString* string) const;
+
  private:
   // The current and look-ahead token.
   struct TokenDesc {
@@ -606,6 +660,8 @@ class Scanner {
   LiteralBuffer literal_buffer1_;
   LiteralBuffer literal_buffer2_;
 
+  ParamListFinder param_list_finder_;
+
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
@@ -634,6 +690,95 @@ class Scanner {
   bool harmony_numeric_literals_;
 };
 
+
+void ParamListFinder::Update(Scanner* scanner) {

[marja, 2014/06/17 11:47:38]

So you have this state machine which is updated for each token when scanning,
and then there is ParserTraits::ParameterListFromExpression too, for sort of
getting the same data "after the fact". This looks pretty duplicated to me.

Also, you're adding this to be executed after each token during scanning, and
I'm worried about the overhead.

Would it be feasible to
1) only do the parameter list dance if we have some indicator that we are inside
the parameter list (seen "(", for example, and quit as soon as we realize it's
not a valid parameter list)
2) do it either when reading the parameter list, or after, but not both?

And I'm not sure if Scanner is the right place for the parameter list logic...

What's the reason why the "after the fact" processing is not sufficient? Is it
because of PreParser? Did you try out the "PreParserExpression keeps track of
the "this is a comma-separated identifier name list" property" approach, and why
didn't it fly?

Btw, how much does this regress the scanning performance? There's this tool
lexer-shell which you can use for finding out..

+  const Token::Value token = scanner->current_token();
+
+  switch (token) {
+    case Token::IDENTIFIER:
+    case Token::YIELD:
+    case Token::FUTURE_STRICT_RESERVED_WORD:
+      identifier_pos_ = scanner->location().beg_pos;
+      AddIdentifier(scanner);
+      break;
+    default:
+      identifier_pos_ = -1;
+  }
+
+  switch (state_) {
+    case Valid:
+      state_ = Invalid;
+      start_pos_ = -1;
+      // Fall-through.
+    case Invalid:
+      switch (token) {
+        case Token::LPAREN:
+          state_ = LeftParen;
+          start_pos_ = scanner->location().beg_pos;
+          break;
+        default:
+          // Stay in Invalid state.
+          break;
+      }
+      break;
+
+    case LeftParen:
+      switch (token) {
+        case Token::LPAREN:
+          start_pos_ = scanner->location().beg_pos;
+          break;
+        case Token::RPAREN:
+          state_ = Valid;
+          break;
+        case Token::YIELD:
+        case Token::IDENTIFIER:
+        case Token::FUTURE_STRICT_RESERVED_WORD:
+          state_ = Identifier;
+          break;
+        default:
+          state_ = Invalid;
+      }
+      break;
+
+    case Identifier:
+      switch (token) {
+        case Token::RPAREN:
+          state_ = Valid;
+          break;
+        case Token::COMMA:
+          state_ = Comma;
+          break;
+        default:
+          state_ = Invalid;
+      }
+      break;
+
+    case Comma:
+      switch (token) {
+        case Token::YIELD:
+        case Token::IDENTIFIER:
+        case Token::FUTURE_STRICT_RESERVED_WORD:
+          state_ = Identifier;
+          break;
+        default:
+          state_ = Invalid;
+      }
+      break;
+  }
+}
+
+
+void ParamListFinder::AddIdentifier(Scanner* scanner) {
+  // A duplicate has already been found.
+  if (duplicate_pos_ >= start_pos_)
+    return;
+
+  int prev_value = scanner->FindSymbol(&duplicate_finder_, 1);
+  if (prev_value != 0) {
+    duplicate_pos_ = scanner->location().beg_pos;
+  }
+}
+
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_