Move scanner tests into the front_end package.

pkg/front_end/test/scanner_test.dart

-// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-library analyzer.test.generated.scanner_test;
-
-import 'package:analyzer/dart/ast/token.dart';
-import 'package:analyzer/error/error.dart';
-import 'package:analyzer/error/listener.dart';
-import 'package:analyzer/src/dart/ast/token.dart';
-import 'package:analyzer/src/dart/scanner/reader.dart';
-import 'package:analyzer/src/dart/scanner/scanner.dart';
-import 'package:analyzer/src/generated/source.dart';
+import 'package:front_end/src/base/errors.dart';
+import 'package:front_end/src/base/jenkins_smi_hash.dart';
+import 'package:front_end/src/scanner/errors.dart';
+import 'package:front_end/src/scanner/reader.dart';
+import 'package:front_end/src/scanner/scanner.dart';
+import 'package:front_end/src/scanner/token.dart';
 import 'package:test/test.dart';
 import 'package:test_reflective_loader/test_reflective_loader.dart';
 
-import 'test_support.dart';
-
 main() {
   defineReflectiveSuite(() {
     defineReflectiveTests(CharSequenceReaderTest);
     defineReflectiveTests(KeywordStateTest);
     defineReflectiveTests(ScannerTest);
     defineReflectiveTests(TokenTypeTest);
   });
 }
 
-class CharacterRangeReaderTest extends EngineTestCase {
-  void test_advance() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 4);
-    expect(reader.advance(), 0x79);
-    expect(reader.advance(), 0x80);
-    expect(reader.advance(), 0x80);
-    expect(reader.advance(), -1);
-    expect(reader.advance(), -1);
-  }
-
-  void test_creation() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 4);
-    expect(reader, isNotNull);
-  }
-
-  void test_getOffset() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 2);
-    expect(reader.offset, 1);
-    reader.advance();
-    expect(reader.offset, 2);
-    reader.advance();
-    expect(reader.offset, 2);
-  }
-
-  void test_getString() {
-    CharSequenceReader baseReader = new CharSequenceReader("__xyzzy__");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 2, 7);
-    reader.offset = 5;
-    expect(reader.getString(3, 0), "yzz");
-    expect(reader.getString(4, 1), "zzy");
-  }
-
-  void test_peek() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 3);
-    expect(reader.peek(), 0x79);
-    expect(reader.peek(), 0x79);
-    reader.advance();
-    expect(reader.peek(), 0x80);
-    expect(reader.peek(), 0x80);
-    reader.advance();
-    expect(reader.peek(), -1);
-    expect(reader.peek(), -1);
-  }
-
-  void test_setOffset() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 4);
-    reader.offset = 2;
-    expect(reader.offset, 2);
-  }
-}
-
 @reflectiveTest
 class CharSequenceReaderTest {
   void test_advance() {
     CharSequenceReader reader = new CharSequenceReader("x");
     expect(reader.advance(), 0x78);
     expect(reader.advance(), -1);
     expect(reader.advance(), -1);
   }
 
   void test_creation() {
@@ skipping 166 matching lines @@
 
   void test_colon() {
     _assertToken(TokenType.COLON, ":");
   }
 
   void test_comma() {
     _assertToken(TokenType.COMMA, ",");
   }
 
   void test_comment_disabled_multi() {
-    Scanner scanner = new Scanner(
-        null,
-        new CharSequenceReader("/* comment */ "),
-        AnalysisErrorListener.NULL_LISTENER);
+    Scanner scanner =
+        new _TestScanner(new CharSequenceReader("/* comment */ "));
     scanner.preserveComments = false;
     Token token = scanner.tokenize();
     expect(token, isNotNull);
     expect(token.precedingComments, isNull);
   }
 
   void test_comment_generic_method_type_assign() {
     _assertComment(TokenType.MULTI_LINE_COMMENT, "/*=comment*/");
     _assertComment(TokenType.GENERIC_METHOD_TYPE_ASSIGN, "/*=comment*/",
         genericMethodComments: true);
   }
 
   void test_comment_generic_method_type_list() {
     _assertComment(TokenType.MULTI_LINE_COMMENT, "/*<comment>*/");
     _assertComment(TokenType.GENERIC_METHOD_TYPE_LIST, "/*<comment>*/",
         genericMethodComments: true);
   }
 
   void test_comment_multi() {
     _assertComment(TokenType.MULTI_LINE_COMMENT, "/* comment */");
   }
 
   void test_comment_multi_lineEnds() {
     String code = r'''
 /**
  * aa
  * bbb
  * c
  */''';
-    GatheringErrorListener listener = new GatheringErrorListener();
-    Scanner scanner = new Scanner(null, new CharSequenceReader(code), listener);
+    _ErrorListener listener = new _ErrorListener();
+    Scanner scanner = new _TestScanner(new CharSequenceReader(code), listener);
     scanner.tokenize();
     expect(
         scanner.lineStarts,
         equals(<int>[
           code.indexOf('/**'),
           code.indexOf(' * aa'),
           code.indexOf(' * bbb'),
           code.indexOf(' * c'),
           code.indexOf(' */')
         ]));
@@ skipping 78 matching lines @@
 
   void test_hexidecimal_missingDigit() {
     _assertError(ScannerErrorCode.MISSING_HEX_DIGIT, 1, "0x");
   }
 
   void test_identifier() {
     _assertToken(TokenType.IDENTIFIER, "result");
   }
 
   void test_illegalChar_cyrillicLetter_middle() {
-    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 5, "Shche\u0433lov");
+    _assertError(
+        ScannerErrorCode.ILLEGAL_CHARACTER, 5, "Shche\u0433lov", [0x433]);
   }
 
   void test_illegalChar_cyrillicLetter_start() {
-    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0429");
+    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0429", [0x429]);
   }
 
   void test_illegalChar_nbsp() {
-    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u00A0");
+    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u00A0", [0xa0]);
   }
 
   void test_illegalChar_notLetter() {
-    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0312");
+    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0312", [0x312]);
   }
 
   void test_index() {
     _assertToken(TokenType.INDEX, "[]");
   }
 
   void test_index_eq() {
     _assertToken(TokenType.INDEX_EQ, "[]=");
   }
 
@@ skipping 190 matching lines @@
   }
 
   void test_keyword_while() {
     _assertKeywordToken("while");
   }
 
   void test_keyword_with() {
     _assertKeywordToken("with");
   }
 
-  void test_lineInfo_multilineComment() {
-    String source = "/*\r *\r */";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(4, 2, 2),
-      new ScannerTest_ExpectedLocation(source.length - 1, 3, 3)
-    ]);
-  }
-
-  void test_lineInfo_multilineString() {
-    String source = "'''a\r\nbc\r\nd'''";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(7, 2, 2),
-      new ScannerTest_ExpectedLocation(source.length - 1, 3, 4)
-    ]);
-  }
-
-  void test_lineInfo_multilineString_raw() {
-    String source = "var a = r'''\nblah\n''';\n\nfoo";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(14, 2, 2),
-      new ScannerTest_ExpectedLocation(source.length - 2, 5, 2)
-    ]);
-  }
-
-  void test_lineInfo_simpleClass() {
-    String source =
-        "class Test {\r\n    String s = '...';\r\n    int get x => s.MISSING_GETTER;\r\n}";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(source.indexOf("MISSING_GETTER"), 3, 20),
-      new ScannerTest_ExpectedLocation(source.length - 1, 4, 1)
-    ]);
-  }
-
-  void test_lineInfo_slashN() {
-    String source = "class Test {\n}";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(source.indexOf("}"), 2, 1)
-    ]);
-  }
-
   void test_lt() {
     _assertToken(TokenType.LT, "<");
   }
 
   void test_lt_eq() {
     _assertToken(TokenType.LT_EQ, "<=");
   }
 
   void test_lt_lt() {
     _assertToken(TokenType.LT_LT, "<<");
@@ skipping 110 matching lines @@
   void test_scriptTag_withSpace() {
     _assertToken(TokenType.SCRIPT_TAG, "#! /bin/dart");
   }
 
   void test_semicolon() {
     _assertToken(TokenType.SEMICOLON, ";");
   }
 
   void test_setSourceStart() {
     int offsetDelta = 42;
-    GatheringErrorListener listener = new GatheringErrorListener();
+    _ErrorListener listener = new _ErrorListener();
     Scanner scanner =
-        new Scanner(null, new SubSequenceReader("a", offsetDelta), listener);
+        new _TestScanner(new SubSequenceReader("a", offsetDelta), listener);
     scanner.setSourceStart(3, 9);
     scanner.tokenize();
     List<int> lineStarts = scanner.lineStarts;
     expect(lineStarts, isNotNull);
     expect(lineStarts.length, 3);
     expect(lineStarts[2], 33);
   }
 
   void test_slash() {
     _assertToken(TokenType.SLASH, "/");
@@ skipping 248 matching lines @@
 
   void test_tilde_slash() {
     _assertToken(TokenType.TILDE_SLASH, "~/");
   }
 
   void test_tilde_slash_eq() {
     _assertToken(TokenType.TILDE_SLASH_EQ, "~/=");
   }
 
   void test_unclosedPairInInterpolation() {
-    GatheringErrorListener listener = new GatheringErrorListener();
+    _ErrorListener listener = new _ErrorListener();
     _scanWithListener("'\${(}'", listener);
   }
 
   void _assertComment(TokenType commentType, String source,
       {bool genericMethodComments: false}) {
     //
     // Test without a trailing end-of-line marker
     //
     Token token = _scan(source, genericMethodComments: genericMethodComments);
     expect(token, isNotNull);
@@ skipping 20 matching lines @@
 
   /**
    * Assert that scanning the given [source] produces an error with the given
    * code.
    *
    * [expectedError] the error that should be produced
    * [expectedOffset] the string offset that should be associated with the error
    * [source] the source to be scanned to produce the error
    */
   void _assertError(
-      ScannerErrorCode expectedError, int expectedOffset, String source) {
-    GatheringErrorListener listener = new GatheringErrorListener();
+      ScannerErrorCode expectedError, int expectedOffset, String source,
+      [List<Object> arguments]) {
+    _ErrorListener listener = new _ErrorListener();
     _scanWithListener(source, listener);
-    listener.assertErrors([
-      new AnalysisError(null, expectedOffset, 1, expectedError,
-          [source.codeUnitAt(expectedOffset)])
-    ]);
+    listener.assertErrors(
+        [new _TestError(expectedOffset, 1, expectedError, arguments)]);
   }
 
   /**
    * Assert that scanning the given [source] produces an error with the given
    * code, and also produces the given tokens.
    *
    * [expectedError] the error that should be produced
    * [expectedOffset] the string offset that should be associated with the error
    * [source] the source to be scanned to produce the error
    * [expectedTokens] the tokens that are expected to be in the source
    */
   void _assertErrorAndTokens(ScannerErrorCode expectedError, int expectedOffset,
       String source, List<Token> expectedTokens) {
-    GatheringErrorListener listener = new GatheringErrorListener();
+    _ErrorListener listener = new _ErrorListener();
     Token token = _scanWithListener(source, listener);
-    listener.assertErrors([
-      new AnalysisError(null, expectedOffset, 1, expectedError,
-          [source.codeUnitAt(expectedOffset)])
-    ]);
+    listener
+        .assertErrors([new _TestError(expectedOffset, 1, expectedError, null)]);
     _checkTokens(token, expectedTokens);
   }
 
   /**
    * Assert that when scanned the given [source] contains a single keyword token
    * with the same lexeme as the original source.
    */
   void _assertKeywordToken(String source) {
     Token token = _scan(source);
     expect(token, isNotNull);
     expect(token.type, TokenType.KEYWORD);
     expect(token.offset, 0);
     expect(token.length, source.length);
     expect(token.lexeme, source);
     Object value = token.value();
     expect(value is Keyword, isTrue);
     expect((value as Keyword).syntax, source);
     token = _scan(" $source ");
     expect(token, isNotNull);
     expect(token.type, TokenType.KEYWORD);
     expect(token.offset, 1);
     expect(token.length, source.length);
     expect(token.lexeme, source);
     value = token.value();
     expect(value is Keyword, isTrue);
     expect((value as Keyword).syntax, source);
     expect(token.next.type, TokenType.EOF);
   }
 
-  void _assertLineInfo(
-      String source, List<ScannerTest_ExpectedLocation> expectedLocations) {
-    GatheringErrorListener listener = new GatheringErrorListener();
-    _scanWithListener(source, listener);
-    listener.assertNoErrors();
-    LineInfo info = listener.getLineInfo(new TestSource());
-    expect(info, isNotNull);
-    int count = expectedLocations.length;
-    for (int i = 0; i < count; i++) {
-      ScannerTest_ExpectedLocation expectedLocation = expectedLocations[i];
-      LineInfo_Location location = info.getLocation(expectedLocation._offset);
-      expect(location.lineNumber, expectedLocation._lineNumber,
-          reason: 'Line number in location $i');
-      expect(location.columnNumber, expectedLocation._columnNumber,
-          reason: 'Column number in location $i');
-    }
-  }
-
   /**
    * Assert that the token scanned from the given [source] has the
    * [expectedType].
    */
   Token _assertToken(TokenType expectedType, String source,
       {bool lazyAssignmentOperators: false}) {
     Token originalToken =
         _scan(source, lazyAssignmentOperators: lazyAssignmentOperators);
     expect(originalToken, isNotNull);
     expect(originalToken.type, expectedType);
@@ skipping 65 matching lines @@
           reason: "Wrong lexeme for token $i");
       token = token.next;
       expect(token, isNotNull);
     }
     expect(token.type, TokenType.EOF);
   }
 
   Token _scan(String source,
       {bool genericMethodComments: false,
       bool lazyAssignmentOperators: false}) {
-    GatheringErrorListener listener = new GatheringErrorListener();
+    _ErrorListener listener = new _ErrorListener();
     Token token = _scanWithListener(source, listener,
         genericMethodComments: genericMethodComments,
         lazyAssignmentOperators: lazyAssignmentOperators);
     listener.assertNoErrors();
     return token;
   }
 
-  Token _scanWithListener(String source, GatheringErrorListener listener,
+  Token _scanWithListener(String source, _ErrorListener listener,
       {bool genericMethodComments: false,
       bool lazyAssignmentOperators: false}) {
     Scanner scanner =
-        new Scanner(null, new CharSequenceReader(source), listener);
+        new _TestScanner(new CharSequenceReader(source), listener);
     scanner.scanGenericMethodComments = genericMethodComments;
     scanner.scanLazyAssignmentOperators = lazyAssignmentOperators;
-    Token result = scanner.tokenize();
-    listener.setLineInfo(new TestSource(), scanner.lineStarts);
-    return result;
-  }
-}
-
-/**
- * An `ExpectedLocation` encodes information about the expected location of a
- * given offset in source code.
- */
-class ScannerTest_ExpectedLocation {
-  final int _offset;
-
-  final int _lineNumber;
-
-  final int _columnNumber;
-
-  ScannerTest_ExpectedLocation(
-      this._offset, this._lineNumber, this._columnNumber);
-}
-
-/**
- * A `TokenStreamValidator` is used to validate the correct construction of a
- * stream of tokens.
- */
-class TokenStreamValidator {
-  /**
-   * Validate that the stream of tokens that starts with the given [token] is
-   * correct.
-   */
-  void validate(Token token) {
-    StringBuffer buffer = new StringBuffer();
-    _validateStream(buffer, token);
-    if (buffer.length > 0) {
-      fail(buffer.toString());
-    }
-  }
-
-  void _validateStream(StringBuffer buffer, Token token) {
-    if (token == null) {
-      return;
-    }
-    Token previousToken = null;
-    int previousEnd = -1;
-    Token currentToken = token;
-    while (currentToken != null && currentToken.type != TokenType.EOF) {
-      _validateStream(buffer, currentToken.precedingComments);
-      TokenType type = currentToken.type;
-      if (type == TokenType.OPEN_CURLY_BRACKET ||
-          type == TokenType.OPEN_PAREN ||
-          type == TokenType.OPEN_SQUARE_BRACKET ||
-          type == TokenType.STRING_INTERPOLATION_EXPRESSION) {
-        if (currentToken is! BeginToken) {
-          buffer.write("\r\nExpected BeginToken, found ");
-          buffer.write(currentToken.runtimeType.toString());
-          buffer.write(" ");
-          _writeToken(buffer, currentToken);
-        }
-      }
-      int currentStart = currentToken.offset;
-      int currentLength = currentToken.length;
-      int currentEnd = currentStart + currentLength - 1;
-      if (currentStart <= previousEnd) {
-        buffer.write("\r\nInvalid token sequence: ");
-        _writeToken(buffer, previousToken);
-        buffer.write(" followed by ");
-        _writeToken(buffer, currentToken);
-      }
-      previousEnd = currentEnd;
-      previousToken = currentToken;
-      currentToken = currentToken.next;
-    }
-  }
-
-  void _writeToken(StringBuffer buffer, Token token) {
-    buffer.write("[");
-    buffer.write(token.type);
-    buffer.write(", '");
-    buffer.write(token.lexeme);
-    buffer.write("', ");
-    buffer.write(token.offset);
-    buffer.write(", ");
-    buffer.write(token.length);
-    buffer.write("]");
+    return scanner.tokenize();
   }
 }
 
 @reflectiveTest
-class TokenTypeTest extends EngineTestCase {
+class TokenTypeTest {
   void test_isOperator() {
     expect(TokenType.AMPERSAND.isOperator, isTrue);
     expect(TokenType.AMPERSAND_AMPERSAND.isOperator, isTrue);
     expect(TokenType.AMPERSAND_EQ.isOperator, isTrue);
     expect(TokenType.BANG.isOperator, isTrue);
     expect(TokenType.BANG_EQ.isOperator, isTrue);
     expect(TokenType.BAR.isOperator, isTrue);
     expect(TokenType.BAR_BAR.isOperator, isTrue);
     expect(TokenType.BAR_EQ.isOperator, isTrue);
     expect(TokenType.CARET.isOperator, isTrue);
@@ skipping 45 matching lines @@
     expect(TokenType.LT_LT.isUserDefinableOperator, isTrue);
     expect(TokenType.MINUS.isUserDefinableOperator, isTrue);
     expect(TokenType.PERCENT.isUserDefinableOperator, isTrue);
     expect(TokenType.PLUS.isUserDefinableOperator, isTrue);
     expect(TokenType.SLASH.isUserDefinableOperator, isTrue);
     expect(TokenType.STAR.isUserDefinableOperator, isTrue);
     expect(TokenType.TILDE.isUserDefinableOperator, isTrue);
     expect(TokenType.TILDE_SLASH.isUserDefinableOperator, isTrue);
   }
 }
+
+class _ErrorListener {
+  final errors = <_TestError>[];
+
+  void assertErrors(List<_TestError> expectedErrors) {
+    expect(errors, unorderedEquals(expectedErrors));
+  }
+
+  void assertNoErrors() {
+    assertErrors([]);
+  }
+}
+
+class _TestError {
+  final int offset;
+  final int length;
+  final ErrorCode errorCode;
+  final List<Object> arguments;
+
+  _TestError(this.offset, this.length, this.errorCode, this.arguments);
+
+  @override
+  get hashCode {
+    var h = new JenkinsSmiHash()..add(offset)..add(length)..add(errorCode);
+    if (arguments != null) {
+      for (Object argument in arguments) {
+        h.add(argument);
+      }
+    }
+    return h.hashCode;
+  }
+
+  @override
+  operator ==(Object other) {
+    if (other is _TestError &&
+        offset == other.offset &&
+        length == other.length &&
+        errorCode == other.errorCode) {
+      if (arguments == null) return other.arguments == null;
+      if (other.arguments == null) return false;
+      if (arguments.length != other.arguments.length) return false;
+      for (int i = 0; i < arguments.length; i++) {
+        if (arguments[i] != other.arguments[i]) return false;
+      }
+      return true;
+    }
+    return false;
+  }
+
+  @override
+  toString() {
+    var end = offset + length;
+    var argString = arguments == null ? '' : '(${arguments.join(', ')})';
+    return 'Error($offset..$end, $errorCode$argString)';
+  }
+}
+
+class _TestScanner extends Scanner {
+  final _ErrorListener listener;
+
+  _TestScanner(CharacterReader reader, [this.listener]) : super(reader);
+
+  @override
+  void reportError(
+      ScannerErrorCode errorCode, int offset, List<Object> arguments) {
+    if (listener != null) {
+      listener.errors.add(new _TestError(offset, 1, errorCode, arguments));
+    }
+  }
+}