Implement handling of arrow functions in the parser

src/scanner.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #ifndef V8_SCANNER_H_
 #define V8_SCANNER_H_
 
 #include "src/allocation.h"
@@ skipping 171 matching lines @@
   UnicodeCache* unicode_constants_;
   // Backing store used to store strings used as hashmap keys.
   SequenceCollector<unsigned char> backing_store_;
   HashMap map_;
   // Buffer used for string->number->canonical string conversions.
   char number_buffer_[kBufferSize];
 };
 
 
 // ----------------------------------------------------------------------------
+// ParamListFinder discovers sequences of tokens which form a valid function
+// parameter list.
+class Scanner;
+
+class ParamListFinder {
+ public:
+  explicit ParamListFinder(UnicodeCache* unicode_cache)
+      : state_(Invalid)
+      , start_pos_(-1)
+      , identifier_pos_(-1)
+      , duplicate_pos_(-1)
+      , duplicate_finder_(unicode_cache) { }
+
+  V8_INLINE void Update(Scanner* scanner);
+
+  bool IsValid(int pos) const {
+    return (state_ == Valid && start_pos_ == pos)
+        || (identifier_pos_ == pos);
+  }
+
+  bool HasDuplicateIdentifiers() const {
+    return duplicate_pos_ > start_pos_;
+  }
+
+  int FirstDuplicatePosition() const {
+    return duplicate_pos_;
+  }
+
+ private:
+  enum State {
+    Invalid,
+    Valid,
+    LeftParen,
+    Identifier,
+    Comma
+  };
+
+  V8_INLINE void AddIdentifier(Scanner* scanner);
+
+  State state_;
+  int start_pos_;
+  int identifier_pos_;
+  int duplicate_pos_;
+  DuplicateFinder duplicate_finder_;
+};
+
+
+// ----------------------------------------------------------------------------
 // LiteralBuffer -  Collector of chars of literals.
 
 class LiteralBuffer {
  public:
   LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() { }
 
   ~LiteralBuffer() {
     if (backing_store_.length() > 0) {
       backing_store_.Dispose();
     }
@@ skipping 194 matching lines @@
     if (is_literal_one_byte() &&
         literal_length() == 3 &&
         !literal_contains_escapes()) {
       const char* token =
           reinterpret_cast<const char*>(literal_one_byte_string().start());
       *is_get = strncmp(token, "get", 3) == 0;
       *is_set = !*is_get && strncmp(token, "set", 3) == 0;
     }
   }
 
+  ParamListFinder* parameter_list() {
+      return &param_list_finder_;
+  }
+
   int FindNumber(DuplicateFinder* finder, int value);
   int FindSymbol(DuplicateFinder* finder, int value);
 
   UnicodeCache* unicode_cache() { return unicode_cache_; }
 
   // Returns the location of the last seen octal literal.
   Location octal_position() const { return octal_pos_; }
   void clear_octal_position() { octal_pos_ = Location::invalid(); }
 
   // Seek forward to the given position.  This operation does not
@@ skipping 28 matching lines @@
            has_multiline_comment_before_next_;
   }
 
   // Scans the input as a regular expression pattern, previous
   // character(s) must be /(=). Returns true if a pattern is scanned.
   bool ScanRegExpPattern(bool seen_equal);
   // Returns true if regexp flags are scanned (always since flags can
   // be empty).
   bool ScanRegExpFlags();
 
+  bool IdentifierIsFutureStrictReserved(const AstString* string) const;
+
  private:
   // The current and look-ahead token.
   struct TokenDesc {
     Token::Value token;
     Location location;
     LiteralBuffer* literal_chars;
   };
 
   static const int kCharacterLookaheadBufferSize = 1;
 
@@ skipping 135 matching lines @@
   int source_pos() {
     return source_->pos() - kCharacterLookaheadBufferSize;
   }
 
   UnicodeCache* unicode_cache_;
 
   // Buffers collecting literal strings, numbers, etc.
   LiteralBuffer literal_buffer1_;
   LiteralBuffer literal_buffer2_;
 
+  ParamListFinder param_list_finder_;
+
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
   // Input stream. Must be initialized to an Utf16CharacterStream.
   Utf16CharacterStream* source_;
 
 
   // Start position of the octal literal last scanned.
   Location octal_pos_;
 
   // One Unicode character look-ahead; c0_ < 0 at the end of the input.
   uc32 c0_;
 
   // Whether there is a line terminator whitespace character after
   // the current token, and  before the next. Does not count newlines
   // inside multiline comments.
   bool has_line_terminator_before_next_;
   // Whether there is a multi-line comment that contains a
   // line-terminator after the current token, and before the next.
   bool has_multiline_comment_before_next_;
   // Whether we scan 'let' as a keyword for harmony block-scoped let bindings.
   bool harmony_scoping_;
   // Whether we scan 'module', 'import', 'export' as keywords.
   bool harmony_modules_;
   // Whether we scan 0o777 and 0b111 as numbers.
   bool harmony_numeric_literals_;
 };
 
+
+void ParamListFinder::Update(Scanner* scanner) {

[marja, 2014/06/17 11:47:38]

So you have this state machine which is updated for each token when scanning,
and then there is ParserTraits::ParameterListFromExpression too, for sort of
getting the same data "after the fact". This looks pretty duplicated to me.

Also, you're adding this to be executed after each token during scanning, and
I'm worried about the overhead.

Would it be feasible to
1) only do the parameter list dance if we have some indicator that we are inside
the parameter list (seen "(", for example, and quit as soon as we realize it's
not a valid parameter list)
2) do it either when reading the parameter list, or after, but not both?

And I'm not sure if Scanner is the right place for the parameter list logic...

What's the reason why the "after the fact" processing is not sufficient? Is it
because of PreParser? Did you try out the "PreParserExpression keeps track of
the "this is a comma-separated identifier name list" property" approach, and why
didn't it fly?

Btw, how much does this regress the scanning performance? There's this tool
lexer-shell which you can use for finding out..

+  const Token::Value token = scanner->current_token();
+
+  switch (token) {
+    case Token::IDENTIFIER:
+    case Token::YIELD:
+    case Token::FUTURE_STRICT_RESERVED_WORD:
+      identifier_pos_ = scanner->location().beg_pos;
+      AddIdentifier(scanner);
+      break;
+    default:
+      identifier_pos_ = -1;
+  }
+
+  switch (state_) {
+    case Valid:
+      state_ = Invalid;
+      start_pos_ = -1;
+      // Fall-through.
+    case Invalid:
+      switch (token) {
+        case Token::LPAREN:
+          state_ = LeftParen;
+          start_pos_ = scanner->location().beg_pos;
+          break;
+        default:
+          // Stay in Invalid state.
+          break;
+      }
+      break;
+
+    case LeftParen:
+      switch (token) {
+        case Token::LPAREN:
+          start_pos_ = scanner->location().beg_pos;
+          break;
+        case Token::RPAREN:
+          state_ = Valid;
+          break;
+        case Token::YIELD:
+        case Token::IDENTIFIER:
+        case Token::FUTURE_STRICT_RESERVED_WORD:
+          state_ = Identifier;
+          break;
+        default:
+          state_ = Invalid;
+      }
+      break;
+
+    case Identifier:
+      switch (token) {
+        case Token::RPAREN:
+          state_ = Valid;
+          break;
+        case Token::COMMA:
+          state_ = Comma;
+          break;
+        default:
+          state_ = Invalid;
+      }
+      break;
+
+    case Comma:
+      switch (token) {
+        case Token::YIELD:
+        case Token::IDENTIFIER:
+        case Token::FUTURE_STRICT_RESERVED_WORD:
+          state_ = Identifier;
+          break;
+        default:
+          state_ = Invalid;
+      }
+      break;
+  }
+}
+
+
+void ParamListFinder::AddIdentifier(Scanner* scanner) {
+  // A duplicate has already been found.
+  if (duplicate_pos_ >= start_pos_)
+    return;
+
+  int prev_value = scanner->FindSymbol(&duplicate_finder_, 1);
+  if (prev_value != 0) {
+    duplicate_pos_ = scanner->location().beg_pos;
+  }
+}
+
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_