Mark unsafe token access in AstBuilder

pkg/analyzer/lib/src/fasta/ast_builder.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.analyzer.ast_builder;
 
 import 'package:analyzer/analyzer.dart';
 import 'package:analyzer/dart/ast/ast_factory.dart' show AstFactory;
 import 'package:analyzer/dart/ast/standard_ast_factory.dart' as standard;
 import 'package:analyzer/dart/ast/token.dart' show Token, TokenType;
@@ skipping 376 matching lines @@
       bool hasExpression, Token beginToken, Token endToken) {
     debugEvent("ReturnStatement");
     Expression expression = hasExpression ? pop() : null;
     push(ast.returnStatement(beginToken, expression, endToken));
   }
 
   void endIfStatement(Token ifToken, Token elseToken) {
     Statement elsePart = popIfNotNull(elseToken);
     Statement thenPart = pop();
     Expression condition = pop();
-    analyzer.BeginToken leftParenthesis = ifToken.next;
+    analyzer.BeginToken leftParenthesis =
+        unsafeToken(ifToken.next, TokenType.OPEN_PAREN);
     push(ast.ifStatement(ifToken, ifToken.next, condition,
         leftParenthesis.endGroup, thenPart, elseToken, elsePart));
   }
 
   void handleNoInitializers() {
     debugEvent("NoInitializers");
     if (!isFullAst) return;
     push(NullValue.ConstructorInitializerSeparator);
     push(NullValue.ConstructorInitializers);
   }
@@ skipping 141 matching lines @@
   void endForStatement(Token forKeyword, Token leftSeparator,
       int updateExpressionCount, Token endToken) {
     debugEvent("ForStatement");
     Statement body = pop();
     List<Expression> updates = popList(updateExpressionCount);
     Statement conditionStatement = pop();
     Object initializerPart = pop();
     exitLocalScope();
     exitContinueTarget();
     exitBreakTarget();
-    analyzer.BeginToken leftParenthesis = forKeyword.next;
+    analyzer.BeginToken leftParenthesis =
+        unsafeToken(forKeyword.next, TokenType.OPEN_PAREN);
 
     VariableDeclarationList variableList;
     Expression initializer;
     if (initializerPart is VariableDeclarationStatement) {
       variableList = initializerPart.variables;
     } else {
       initializer = initializerPart as Expression;
     }
 
     Expression condition;
     Token rightSeparator;
     if (conditionStatement is ExpressionStatement) {
       condition = conditionStatement.expression;
       rightSeparator = conditionStatement.semicolon;
     } else {
       rightSeparator = (conditionStatement as EmptyStatement).semicolon;
     }
 
     push(ast.forStatement(
         forKeyword,
         leftParenthesis,
         variableList,
         initializer,
         leftSeparator,
         condition,
         rightSeparator,
         updates,
-        leftParenthesis.endGroup,
+        leftParenthesis?.endGroup,
         body));
   }
 
   void handleLiteralList(
       int count, Token beginToken, Token constKeyword, Token endToken) {
     debugEvent("LiteralList");
     List<Expression> expressions = popList(count);
     TypeArgumentList typeArguments = pop();
     push(ast.listLiteral(
         constKeyword, typeArguments, beginToken, expressions, endToken));
@@ skipping 241 matching lines @@
             returnType: typeOrFunctionTypedParameter.returnType,
             typeParameters: typeOrFunctionTypedParameter.typeParameters,
             parameters: typeOrFunctionTypedParameter.parameters);
       } else {
         node = ast.fieldFormalParameter2(
             identifier: name,
             comment: comment,
             covariantKeyword: covariantKeyword,
             type: typeOrFunctionTypedParameter.returnType,
             thisKeyword: thisKeyword,
-            period: thisKeyword.next,
+            period: unsafeToken(thisKeyword.next, TokenType.PERIOD),
             typeParameters: typeOrFunctionTypedParameter.typeParameters,
             parameters: typeOrFunctionTypedParameter.parameters);
       }
     } else {
       TypeAnnotation type = typeOrFunctionTypedParameter;
       if (thisKeyword == null) {
         node = ast.simpleFormalParameter2(
             comment: comment,
             covariantKeyword: covariantKeyword,
             keyword: keyword,
@@ skipping 193 matching lines @@
   void handleInvalidFunctionBody(Token token) {
     debugEvent("InvalidFunctionBody");
   }
 
   @override
   Token handleUnrecoverableError(Token token, Message message) {
     if (message.code == codeExpectedFunctionBody) {
       if (identical('native', token.stringValue) && parser != null) {
         Token nativeKeyword = token;
         Token semicolon = parser.parseLiteralString(token.next);
+        // TODO(brianwilkerson) Should this be using ensureSemicolon?

[danrubel, 2017/08/31 17:27:50]

Just FYI... I'm working on removing this code because it belongs in the parser.

[Brian Wilkerson, 2017/08/31 17:29:37]

Agreed.

         token = parser.expectSemicolon(semicolon);
         StringLiteral name = pop();
         pop(); // star
         pop(); // async
         push(ast.nativeFunctionBody(nativeKeyword, name, semicolon));
         return token;
       }
     } else if (message.code == codeExpectedExpression) {
       String lexeme = token.lexeme;
       if (identical('async', lexeme) || identical('yield', lexeme)) {
@@ skipping 87 matching lines @@
     checkEmpty(endToken.charOffset);
 
     push(ast.compilationUnit(
         beginToken, scriptTag, directives, declarations, endToken));
   }
 
   void endImport(Token importKeyword, Token deferredKeyword, Token asKeyword,
       Token semicolon) {
     debugEvent("Import");
     List<Combinator> combinators = pop();
-    SimpleIdentifier prefix;
-    if (asKeyword != null) prefix = pop();
+    SimpleIdentifier prefix = popIfNotNull(asKeyword);
     List<Configuration> configurations = pop();
     StringLiteral uri = pop();
     List<Annotation> metadata = pop();
     assert(metadata == null); // TODO(paulberry): fix.
     Comment comment = pop();
     directives.add(ast.importDirective(
         comment,
         metadata,
         importKeyword,
         uri,
@@ skipping 38 matching lines @@
         whileKeyword,
         condition.leftParenthesis,
         condition.expression,
         condition.rightParenthesis,
         semicolon));
   }
 
   void endConditionalUri(Token ifKeyword, Token equalitySign) {
     debugEvent("ConditionalUri");
     StringLiteral libraryUri = pop();
-    StringLiteral value;
-    if (equalitySign != null) {
-      value = pop();
-    }
+    StringLiteral value = popIfNotNull(equalitySign);
     DottedName name = pop();
     // TODO(paulberry,ahe): what if there is no `(` token due to an error in the
     // file being parsed?  It seems like we need the parser to do adequate error
     // recovery and then report both the ifKeyword and leftParen tokens to the
     // listener.
-    Token leftParen = ifKeyword.next;
+    Token leftParen = unsafeToken(ifKeyword.next, TokenType.OPEN_PAREN);
     // TODO(paulberry,ahe): the parser should report the right paren token to
     // the listener.
-    Token rightParen = name.endToken.next;
+    Token lastToken = value?.endToken ?? equalitySign ?? name?.endToken;
+    Token rightParen = unsafeToken(lastToken.next, TokenType.CLOSE_PAREN);
     push(ast.configuration(ifKeyword, leftParen, name, equalitySign, value,
         rightParen, libraryUri));
   }
 
   @override
   void endConditionalUris(int count) {
     debugEvent("ConditionalUris");
     push(popList(count) ?? NullValue.ConditionalUris);
   }
 
@@ skipping 192 matching lines @@
     var libraryNameOrUri = pop();
     LibraryIdentifier name;
     StringLiteral uri;
     if (libraryNameOrUri is StringLiteral) {
       uri = libraryNameOrUri;
     } else {
       name = ast.libraryIdentifier(libraryNameOrUri);
     }
     // TODO(paulberry,ahe): seems hacky.  It would be nice if the parser passed
     // in a reference to the "of" keyword.
-    var ofKeyword = partKeyword.next;
+    var ofKeyword = unsafeToken(partKeyword.next, analyzer.Keyword.OF);
     List<Annotation> metadata = pop();
     Comment comment = pop();
     directives.add(ast.partOfDirective(
         comment, metadata, partKeyword, ofKeyword, uri, name, semicolon));
   }
 
   @override
   void endFunctionExpression(Token beginToken, Token token) {
     // TODO(paulberry): set up scopes properly to resolve parameters and type
     // variables.  Note that this is tricky due to the handling of initializers
@@ skipping 246 matching lines @@
       declarations.add(ast.genericTypeAlias(comment, metadata, typedefKeyword,
           name, templateParameters, equals, type, endToken));
     }
   }
 
   @override
   void endEnum(Token enumKeyword, Token endBrace, int count) {
     debugEvent("Enum");
     List<EnumConstantDeclaration> constants = popList(count);
     // TODO(paulberry,ahe): the parser should pass in the openBrace token.
-    var openBrace = enumKeyword.next.next as analyzer.BeginToken;
+    var openBrace =
+        unsafeToken(enumKeyword.next.next, TokenType.OPEN_CURLY_BRACKET)
+            as analyzer.BeginToken;
     // TODO(paulberry): what if the '}' is missing and the parser has performed
     // error recovery?
-    Token closeBrace = openBrace.endGroup;
+    Token closeBrace = openBrace?.endGroup;
     SimpleIdentifier name = pop();
     List<Annotation> metadata = pop();
     Comment comment = pop();
     declarations.add(ast.enumDeclaration(comment, metadata, enumKeyword, name,
         openBrace, constants, closeBrace));
   }
 
   @override
   void endTypeArguments(int count, Token beginToken, Token endToken) {
     debugEvent("TypeArguments");
@@ skipping 213 matching lines @@
         } else {
           errorReporter?.reportErrorForOffset(ParserErrorCode.UNEXPECTED_TOKEN,
               charOffset, text.length, [text]);
         }
         return;
       default:
       // fall through
     }
     library.addCompileTimeError(message, charOffset, uri);
   }
+
+  /// A marker method used to mark locations where a token is being located in
+  /// an unsafe way. In all such cases the parser needs to be fixed to pass in
+  /// the token.
+  Token unsafeToken(Token token, TokenType tokenType) {
+    // TODO(brianwilkerson) Eliminate the need for this method.
+    return token.type == tokenType ? token : null;
+  }
 }
 
 /// Data structure placed on the stack to represent a class body.
 ///
 /// This is needed because analyzer has no separate AST representation of a
 /// class body; it simply stores all of the relevant data in the
 /// [ClassDeclaration] object.
 class _ClassBody {
   final Token beginToken;
 
@@ skipping 87 matching lines @@
       } else if (identical('var', s)) {
         finalConstOrVarKeyword = token;
       } else if (identical('covariant', s)) {
         covariantKeyword = token;
       } else {
         unhandled("$s", "modifier", token.charOffset, null);
       }
     }
   }
 }