Port parser and scanner fixes from rasta branch.

pkg/dart_parser/lib/src/parser.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library dart_parser.parser;
 
+import 'package:dart_scanner/dart_scanner.dart' show
+    ErrorToken;
+
+import 'package:dart_scanner/src/recover.dart' show
+    closeBraceFor,
+    skipToEof;
+
 import 'package:dart_scanner/src/keyword.dart' show
     Keyword;
 
 import 'package:dart_scanner/src/precedence.dart' show
     ASSIGNMENT_PRECEDENCE,
     AS_INFO,
     CASCADE_PRECEDENCE,
-    EOF_INFO,
     EQUALITY_PRECEDENCE,
     GT_INFO,
     IS_INFO,
     MINUS_MINUS_INFO,
     OPEN_PAREN_INFO,
     OPEN_SQUARE_BRACKET_INFO,
     PERIOD_INFO,
     PLUS_PLUS_INFO,
     POSTFIX_PRECEDENCE,
     PrecedenceInfo,
     QUESTION_INFO,
     QUESTION_PERIOD_INFO,
     RELATIONAL_PRECEDENCE;
 
 import 'package:dart_scanner/src/token.dart' show
-    BadInputToken,
     BeginGroupToken,
-    ErrorToken,
     KeywordToken,
     SymbolToken,
     Token,
-    UnmatchedToken,
-    UnterminatedToken,
     isUserDefinableOperator;
 
 import 'package:dart_scanner/src/token_constants.dart' show
-    BAD_INPUT_TOKEN,
     COMMA_TOKEN,
     DOUBLE_TOKEN,
     EOF_TOKEN,
     EQ_TOKEN,
     FUNCTION_TOKEN,
     GT_TOKEN,
     GT_GT_TOKEN,
     HASH_TOKEN,
     HEXADECIMAL_TOKEN,
     IDENTIFIER_TOKEN,
@@ skipping 719 matching lines @@
         ++interfacesCount;
       } while (optional(',', token));
     }
     token = parseClassBody(token);
     listener.endClassDeclaration(
         interfacesCount, begin, extendsKeyword, implementsKeyword, token);
     return token.next;
   }
 
   Token parseStringPart(Token token) {
-    if (identical(token.kind, STRING_TOKEN)) {
-      listener.handleStringPart(token);
-      return token.next;
-    } else {
-      return reportUnrecoverableError(token, ErrorKind.ExpectedString);
+    if (token.kind != STRING_TOKEN) {
+      token = reportUnrecoverableError(token, ErrorKind.ExpectedString);
     }
+    listener.handleStringPart(token);
+    return token.next;
   }
 
   Token parseIdentifier(Token token) {
     if (!token.isIdentifier()) {
       token = reportUnrecoverableError(token, ErrorKind.ExpectedIdentifier);
     }
     listener.handleIdentifier(token);
     return token.next;
   }
 
@@ skipping 343 matching lines @@
     } else {
       listener.handleNoTypeVariables(token);
     }
     token = parseFormalParametersOpt(token);
     bool previousAsyncAwaitKeywordsEnabled = asyncAwaitKeywordsEnabled;
     token = parseAsyncModifier(token);
     token = parseFunctionBody(token, false, externalModifier != null);
     asyncAwaitKeywordsEnabled = previousAsyncAwaitKeywordsEnabled;
     Token endToken = token;
     token = token.next;
-    if (token.kind == BAD_INPUT_TOKEN) {
-      token = reportUnrecoverableError(token, ErrorKind.UnexpectedToken);
-    }
     listener.endTopLevelMethod(start, getOrSet, endToken);
     return token;
   }
 
   /// Looks ahead to find the name of a member. Returns a link of the modifiers,
   /// set/get, (operator) name, and either the start of the method body or the
   /// end of the declaration.
   ///
   /// Examples:
   ///
@@ skipping 18 matching lines @@
   ///
   Link<Token> findMemberName(Token token) {
     Link<Token> identifiers = const Link<Token>();
 
     // `true` if 'get' has been seen.
     bool isGetter = false;
     // `true` if an identifier has been seen after 'get'.
     bool hasName = false;
 
     while (token.kind != EOF_TOKEN) {
-      String value = token.stringValue;
-      if (value == 'get') {
+      if (optional('get', token)) {
         isGetter = true;
-      } else if (hasName && (value == 'sync' || value == 'async')) {
+      } else if (hasName &&
+                 (optional("sync", token) || optional("async", token))) {
         // Skip.
         token = token.next;
-        value = token.stringValue;
-        if (value == '*') {
+        if (optional("*", token)) {
           // Skip.
           token = token.next;
         }
         continue;
-      } else if (value == '(' || value == '{' || value == '=>') {
+      } else if (optional("(", token) || optional("{", token) ||
+                 optional("=>", token)) {
         // A method.
         identifiers = identifiers.prepend(token);
         return identifiers;
-      } else if (value == '=' || value == ';' || value == ',') {
+      } else if (optional("=", token) || optional(";", token) ||
+                 optional(",", token)) {
         // A field or abstract getter.
         identifiers = identifiers.prepend(token);
         return identifiers;
       } else if (isGetter) {
         hasName = true;
       }
       identifiers = identifiers.prepend(token);
       if (isValidTypeReference(token)) {
         // type ...
         if (optional('.', token.next)) {
           // type '.' ...
           if (token.next.next.isIdentifier()) {
             // type '.' identifier
             token = token.next.next;
           }
         }
         if (optional('<', token.next)) {
           if (token.next is BeginGroupToken) {
             BeginGroupToken beginGroup = token.next;
             if (beginGroup.endGroup == null) {
               reportUnrecoverableError(beginGroup, ErrorKind.UnmatchedToken);

[Johnni Winther, 2017/01/30 09:04:38]

Why not use the token returned from [reportUnrecoverableError]? You do that in
parseStringPart.

[ahe, 2017/01/30 13:26:22]

Because I'm not trying to change or improve error recovery in that case, I'm
just try to prevent a crash.

I'm reluctant to change these things as long as it has a chance to break
message_test from dart2js.

+            } else {
+              token = beginGroup.endGroup;
             }
-            token = beginGroup.endGroup;
           }
         }
       }
       token = token.next;
     }
     return const Link<Token>();
   }
 
   Token parseFieldInitializerOpt(Token token) {
     if (optional('=', token)) {
@@ skipping 426 matching lines @@
     listener.endFunctionName(token);
     if (getOrSet == null) {
       token = parseTypeVariablesOpt(token);
     } else {
       listener.handleNoTypeVariables(token);
     }
     token = parseFormalParametersOpt(token);
     token = parseInitializersOpt(token);
     bool previousAsyncAwaitKeywordsEnabled = asyncAwaitKeywordsEnabled;
     token = parseAsyncModifier(token);
-    if (optional('=', token)) {
-      token = parseRedirectingFactoryBody(token);
-    } else {
-      token = parseFunctionBody(token, false, true);
-    }
+    token = parseFunctionBody(token, false, true);
     asyncAwaitKeywordsEnabled = previousAsyncAwaitKeywordsEnabled;
     listener.endFunction(getOrSet, token);
     return token.next;
   }
 
   Token parseUnnamedFunction(Token token) {
     listener.beginUnnamedFunction(token);
     token = parseFormalParameters(token);
     bool previousAsyncAwaitKeywordsEnabled = asyncAwaitKeywordsEnabled;
     token = parseAsyncModifier(token);
@@ skipping 62 matching lines @@
     String value = token.stringValue;
     if (identical(value, ';')) {
       if (!allowAbstract) {
         reportRecoverableError(token, ErrorKind.ExpectedBody);
       }
       listener.handleNoFunctionBody(token);
     } else {
       if (identical(value, '=>')) {
         token = parseExpression(token.next);
         expectSemicolon(token);
-      } else if (value == '=') {
-        token = parseRedirectingFactoryBody(token);
+      } else if (identical(value, '=')) {
+        reportRecoverableError(token, ErrorKind.ExpectedBody);
+        token = parseExpression(token.next);
         expectSemicolon(token);
       } else {
         token = skipBlock(token);
       }
       listener.handleFunctionBodySkipped(token);
     }
     return token;
   }
 
   Token parseFunctionBody(Token token, bool isExpression, bool allowAbstract) {
     if (optional(';', token)) {
       if (!allowAbstract) {
         reportRecoverableError(token, ErrorKind.ExpectedBody);
       }
       listener.endFunctionBody(0, null, token);
       return token;
     } else if (optional('=>', token)) {
       Token begin = token;
       token = parseExpression(token.next);
       if (!isExpression) {
         expectSemicolon(token);
         listener.endReturnStatement(true, begin, token);
       } else {
         listener.endReturnStatement(true, begin, null);
       }
       return token;
+    } else if (optional('=', token)) {
+      Token begin = token;
+      // Recover from a bad factory method.
+      reportRecoverableError(token, ErrorKind.ExpectedBody);
+      token = parseExpression(token.next);
+      if (!isExpression) {
+        expectSemicolon(token);
+        listener.endReturnStatement(true, begin, token);
+      } else {
+        listener.endReturnStatement(true, begin, null);
+      }
+      return token;
     }
     Token begin = token;
     int statementCount = 0;
     if (!optional('{', token)) {
       token = reportUnrecoverableError(token, ErrorKind.ExpectedFunctionBody);
       listener.handleInvalidFunctionBody(token);
       return token;
     }
 
     listener.beginFunctionBody(begin);
@@ skipping 47 matching lines @@
         star = token;
         token = token.next;
       } else {
         reportRecoverableError(async, ErrorKind.InvalidSyncModifier);
       }
     }
     listener.handleAsyncModifier(async, star);
     return token;
   }
 
+  int statementDepth = 0;
   Token parseStatement(Token token) {
+    if (statementDepth++ > 500) {

[Johnni Winther, 2017/01/30 09:04:38]

When does this occur?

[ahe, 2017/01/30 13:26:22]

Deeply nested if statements. See
tests/language/deep_nesting2_negative_test.dart.

+      reportRecoverableError(
+          token, ErrorKind.Unspecified, {'text': 'Stack overflow'});
+      return skipToEof(token);
+    }
+    Token result = parseStatementX(token);
+    statementDepth--;
+    return result;
+  }
+
+  Token parseStatementX(Token token) {
     final value = token.stringValue;
     if (identical(token.kind, IDENTIFIER_TOKEN)) {
       return parseExpressionStatementOrDeclaration(token);
     } else if (identical(value, '{')) {
       return parseBlock(token);
     } else if (identical(value, 'return')) {
       return parseReturnStatement(token);
     } else if (identical(value, 'var') || identical(value, 'final')) {
       return parseVariablesDeclaration(token);
     } else if (identical(value, 'if')) {
@@ skipping 284 matching lines @@
         BeginGroupToken begin = token;
         token = (begin.endGroup != null) ? begin.endGroup : token;
       } else if (token is ErrorToken) {
         reportErrorToken(token, false);
       }
       token = token.next;
     }
     return token;
   }
 
+  int expressionDepth = 0;
   Token parseExpression(Token token) {
+    if (expressionDepth++ > 500) {
+      reportRecoverableError(

[Johnni Winther, 2017/01/30 09:04:38]

When does this occur?

[ahe, 2017/01/30 13:26:22]

Deeply nested array literals, for example, see:
tests/language/deep_nesting1_negative_test.dart

+          token, ErrorKind.Unspecified, {'text': 'Stack overflow'});
+      return token.next;
+    }
     listener.beginExpression(token);
-    return optional('throw', token)
+    Token result = optional('throw', token)
         ? parseThrowExpression(token, true)
         : parsePrecedenceExpression(token, ASSIGNMENT_PRECEDENCE, true);
+    expressionDepth--;
+    return result;
   }
 
   Token parseExpressionWithoutCascade(Token token) {
     listener.beginExpression(token);
     return optional('throw', token)
         ? parseThrowExpression(token, false)
         : parsePrecedenceExpression(token, ASSIGNMENT_PRECEDENCE, false);
   }
 
   Token parseConditionalExpressionRest(Token token) {
@@ skipping 24 matching lines @@
           token = parseCascadeExpression(token);
         } else if (identical(tokenLevel, ASSIGNMENT_PRECEDENCE)) {
           // Right associative, so we recurse at the same precedence
           // level.
           listener.beginExpression(token.next);
           token = parsePrecedenceExpression(token.next, level, allowCascades);
           listener.handleAssignmentExpression(operator);
         } else if (identical(tokenLevel, POSTFIX_PRECEDENCE)) {
           if (identical(info, PERIOD_INFO) ||
               identical(info, QUESTION_PERIOD_INFO)) {
-            // Left associative, so we recurse at the next higher
-            // precedence level. However, POSTFIX_PRECEDENCE is the
-            // highest level, so we just call parseUnaryExpression
-            // directly.
-            token = parseUnaryExpression(token.next, allowCascades);
+            // Left associative, so we recurse at the next higher precedence
+            // level. However, POSTFIX_PRECEDENCE is the highest level, so we
+            // should just call [parseUnaryExpression] directly. However, a
+            // unary expression isn't legal after a period, so we call
+            // [parsePrimary] instead.
+            token = parsePrimary(token.next);
             listener.handleBinaryExpression(operator);
           } else if ((identical(info, OPEN_PAREN_INFO)) ||
               (identical(info, OPEN_SQUARE_BRACKET_INFO))) {
             token = parseArgumentOrIndexStar(token);
           } else if ((identical(info, PLUS_PLUS_INFO)) ||
               (identical(info, MINUS_MINUS_INFO))) {
             listener.handleUnaryPostfixAssignmentExpression(token);
             token = token.next;
           } else {
             token = reportUnrecoverableError(token, ErrorKind.UnexpectedToken);
@@ skipping 1013 matching lines @@
   void reportRecoverableError(Token token, ErrorKind kind, [Map arguments]) {
     if (token is ErrorToken) {
       reportErrorToken(token, true);
     } else {
       arguments ??= {};
       listener.handleRecoverableError(token, kind, arguments);
     }
   }
 
   Token reportErrorToken(ErrorToken token, bool isRecoverable) {
-    ErrorKind kind;
+    ErrorKind kind = token.errorCode;
     Map arguments = const {};
-    if (token is BadInputToken) {
-      String hex = token.character.toRadixString(16);
-      if (hex.length < 4) {
-        String padding = "0000".substring(hex.length);
-        hex = "$padding$hex";
-      }
-      kind = ErrorKind.InvalidInputCharacter;
-      arguments = {'characterHex': hex};
-    } else if (token is UnterminatedToken) {
-      switch (token.start) {
-        case '1e':
-          kind = ErrorKind.MissingExponent;
-          break;
-        case '"':
-        case "'":
-        case '"""':
-        case "'''":
-        case 'r"':
-        case "r'":
-        case 'r"""':
-        case "r'''":
-          kind = ErrorKind.UnterminatedString;
-          arguments = {'quote': token.start};
-          break;
-        case '0x':
-          kind = ErrorKind.ExpectedHexDigit;
-          break;
-        case r'$':
-          kind = ErrorKind.MalformedStringLiteral;
-          break;
-        case '/*':
-          kind = ErrorKind.UnterminatedComment;
-          break;
-        default:
-          kind = ErrorKind.UnterminatedToken;
-          break;
-      }
-    } else if (token is UnmatchedToken) {
-      String begin = token.begin.value;
-      String end = closeBraceFor(begin);
-      kind = ErrorKind.UnmatchedToken;
-      arguments = {'begin': begin, 'end': end};
-    } else {
-      return listener.handleUnrecoverableError(
-          token, ErrorKind.Unspecified, {"text": token.assertionMessage});
+    switch (kind) {
+      case ErrorKind.NonAsciiIdentifier:
+        String hex = token.character.toRadixString(16);
+        if (hex.length < 4) {
+          String padding = "0000".substring(hex.length);
+          hex = "$padding$hex";
+        }
+        arguments = {'characterHex': hex};
+        break;
+
+      case ErrorKind.UnterminatedString:
+        arguments = {'quote': token.start};
+        break;
+
+      case ErrorKind.UnmatchedToken:
+        String begin = token.begin.value;
+        String end = closeBraceFor(begin);
+        arguments = {'begin': begin, 'end': end};
+        break;
+
+      case ErrorKind.Unspecified:
+        arguments = {"text": token.assertionMessage};
+        break;
+
+      default:
+        break;
     }
     if (isRecoverable) {
       listener.handleRecoverableError(token, kind, arguments);
       return null;
     } else {
       return listener.handleUnrecoverableError(token, kind, arguments);
     }
   }
 }
-
-String closeBraceFor(String openBrace) {
-  return const {
-    '(': ')',
-    '[': ']',
-    '{': '}',
-    '<': '>',
-    r'${': '}',
-  }[openBrace];
-}
-
-Token skipToEof(Token token) {
-  while (!identical(token.info, EOF_INFO)) {
-    token = token.next;
-  }
-  return token;
-}