Move scanner tests into the front_end package.

pkg/front_end/test/scanner_test.dart

Index: pkg/front_end/test/scanner_test.dart
diff --git a/pkg/analyzer/test/generated/scanner_test.dart b/pkg/front_end/test/scanner_test.dart
similarity index 80%
copy from pkg/analyzer/test/generated/scanner_test.dart
copy to pkg/front_end/test/scanner_test.dart
index 748bab49f09f603ccf8c6dcbd1f873a03f329150..69e802d3008e3b898096106cdaafabf806b56901 100644
--- a/pkg/analyzer/test/generated/scanner_test.dart
+++ b/pkg/front_end/test/scanner_test.dart
@@ -1,21 +1,16 @@
-// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-library analyzer.test.generated.scanner_test;
-
-import 'package:analyzer/dart/ast/token.dart';
-import 'package:analyzer/error/error.dart';
-import 'package:analyzer/error/listener.dart';
-import 'package:analyzer/src/dart/ast/token.dart';
-import 'package:analyzer/src/dart/scanner/reader.dart';
-import 'package:analyzer/src/dart/scanner/scanner.dart';
-import 'package:analyzer/src/generated/source.dart';
+import 'package:front_end/src/base/errors.dart';
+import 'package:front_end/src/base/jenkins_smi_hash.dart';
+import 'package:front_end/src/scanner/errors.dart';
+import 'package:front_end/src/scanner/reader.dart';
+import 'package:front_end/src/scanner/scanner.dart';
+import 'package:front_end/src/scanner/token.dart';
 import 'package:test/test.dart';
 import 'package:test_reflective_loader/test_reflective_loader.dart';
 
-import 'test_support.dart';
-
 main() {
   defineReflectiveSuite(() {
     defineReflectiveTests(CharSequenceReaderTest);
@@ -25,62 +20,6 @@ main() {
   });
 }
 
-class CharacterRangeReaderTest extends EngineTestCase {
-  void test_advance() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 4);
-    expect(reader.advance(), 0x79);
-    expect(reader.advance(), 0x80);
-    expect(reader.advance(), 0x80);
-    expect(reader.advance(), -1);
-    expect(reader.advance(), -1);
-  }
-
-  void test_creation() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 4);
-    expect(reader, isNotNull);
-  }
-
-  void test_getOffset() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 2);
-    expect(reader.offset, 1);
-    reader.advance();
-    expect(reader.offset, 2);
-    reader.advance();
-    expect(reader.offset, 2);
-  }
-
-  void test_getString() {
-    CharSequenceReader baseReader = new CharSequenceReader("__xyzzy__");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 2, 7);
-    reader.offset = 5;
-    expect(reader.getString(3, 0), "yzz");
-    expect(reader.getString(4, 1), "zzy");
-  }
-
-  void test_peek() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 3);
-    expect(reader.peek(), 0x79);
-    expect(reader.peek(), 0x79);
-    reader.advance();
-    expect(reader.peek(), 0x80);
-    expect(reader.peek(), 0x80);
-    reader.advance();
-    expect(reader.peek(), -1);
-    expect(reader.peek(), -1);
-  }
-
-  void test_setOffset() {
-    CharSequenceReader baseReader = new CharSequenceReader("xyzzy");
-    CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 4);
-    reader.offset = 2;
-    expect(reader.offset, 2);
-  }
-}
-
 @reflectiveTest
 class CharSequenceReaderTest {
   void test_advance() {
@@ -267,10 +206,8 @@ class ScannerTest {
   }
 
   void test_comment_disabled_multi() {
-    Scanner scanner = new Scanner(
-        null,
-        new CharSequenceReader("/* comment */ "),
-        AnalysisErrorListener.NULL_LISTENER);
+    Scanner scanner =
+        new _TestScanner(new CharSequenceReader("/* comment */ "));
     scanner.preserveComments = false;
     Token token = scanner.tokenize();
     expect(token, isNotNull);
@@ -300,8 +237,8 @@ class ScannerTest {
  * bbb
  * c
  */''';
-    GatheringErrorListener listener = new GatheringErrorListener();
-    Scanner scanner = new Scanner(null, new CharSequenceReader(code), listener);
+    _ErrorListener listener = new _ErrorListener();
+    Scanner scanner = new _TestScanner(new CharSequenceReader(code), listener);
     scanner.tokenize();
     expect(
         scanner.lineStarts,
@@ -400,19 +337,20 @@ class ScannerTest {
   }
 
   void test_illegalChar_cyrillicLetter_middle() {
-    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 5, "Shche\u0433lov");
+    _assertError(
+        ScannerErrorCode.ILLEGAL_CHARACTER, 5, "Shche\u0433lov", [0x433]);
   }
 
   void test_illegalChar_cyrillicLetter_start() {
-    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0429");
+    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0429", [0x429]);
   }
 
   void test_illegalChar_nbsp() {
-    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u00A0");
+    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u00A0", [0xa0]);
   }
 
   void test_illegalChar_notLetter() {
-    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0312");
+    _assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0312", [0x312]);
   }
 
   void test_index() {
@@ -623,51 +561,6 @@ class ScannerTest {
     _assertKeywordToken("with");
   }
 
-  void test_lineInfo_multilineComment() {
-    String source = "/*\r *\r */";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(4, 2, 2),
-      new ScannerTest_ExpectedLocation(source.length - 1, 3, 3)
-    ]);
-  }
-
-  void test_lineInfo_multilineString() {
-    String source = "'''a\r\nbc\r\nd'''";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(7, 2, 2),
-      new ScannerTest_ExpectedLocation(source.length - 1, 3, 4)
-    ]);
-  }
-
-  void test_lineInfo_multilineString_raw() {
-    String source = "var a = r'''\nblah\n''';\n\nfoo";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(14, 2, 2),
-      new ScannerTest_ExpectedLocation(source.length - 2, 5, 2)
-    ]);
-  }
-
-  void test_lineInfo_simpleClass() {
-    String source =
-        "class Test {\r\n    String s = '...';\r\n    int get x => s.MISSING_GETTER;\r\n}";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(source.indexOf("MISSING_GETTER"), 3, 20),
-      new ScannerTest_ExpectedLocation(source.length - 1, 4, 1)
-    ]);
-  }
-
-  void test_lineInfo_slashN() {
-    String source = "class Test {\n}";
-    _assertLineInfo(source, [
-      new ScannerTest_ExpectedLocation(0, 1, 1),
-      new ScannerTest_ExpectedLocation(source.indexOf("}"), 2, 1)
-    ]);
-  }
-
   void test_lt() {
     _assertToken(TokenType.LT, "<");
   }
@@ -798,9 +691,9 @@ class ScannerTest {
 
   void test_setSourceStart() {
     int offsetDelta = 42;
-    GatheringErrorListener listener = new GatheringErrorListener();
+    _ErrorListener listener = new _ErrorListener();
     Scanner scanner =
-        new Scanner(null, new SubSequenceReader("a", offsetDelta), listener);
+        new _TestScanner(new SubSequenceReader("a", offsetDelta), listener);
     scanner.setSourceStart(3, 9);
     scanner.tokenize();
     List<int> lineStarts = scanner.lineStarts;
@@ -1069,7 +962,7 @@ class ScannerTest {
   }
 
   void test_unclosedPairInInterpolation() {
-    GatheringErrorListener listener = new GatheringErrorListener();
+    _ErrorListener listener = new _ErrorListener();
     _scanWithListener("'\${(}'", listener);
   }
 
@@ -1110,13 +1003,12 @@ class ScannerTest {
    * [source] the source to be scanned to produce the error
    */
   void _assertError(
-      ScannerErrorCode expectedError, int expectedOffset, String source) {
-    GatheringErrorListener listener = new GatheringErrorListener();
+      ScannerErrorCode expectedError, int expectedOffset, String source,
+      [List<Object> arguments]) {
+    _ErrorListener listener = new _ErrorListener();
     _scanWithListener(source, listener);
-    listener.assertErrors([
-      new AnalysisError(null, expectedOffset, 1, expectedError,
-          [source.codeUnitAt(expectedOffset)])
-    ]);
+    listener.assertErrors(
+        [new _TestError(expectedOffset, 1, expectedError, arguments)]);
   }
 
   /**
@@ -1130,12 +1022,10 @@ class ScannerTest {
    */
   void _assertErrorAndTokens(ScannerErrorCode expectedError, int expectedOffset,
       String source, List<Token> expectedTokens) {
-    GatheringErrorListener listener = new GatheringErrorListener();
+    _ErrorListener listener = new _ErrorListener();
     Token token = _scanWithListener(source, listener);
-    listener.assertErrors([
-      new AnalysisError(null, expectedOffset, 1, expectedError,
-          [source.codeUnitAt(expectedOffset)])
-    ]);
+    listener
+        .assertErrors([new _TestError(expectedOffset, 1, expectedError, null)]);
     _checkTokens(token, expectedTokens);
   }
 
@@ -1165,24 +1055,6 @@ class ScannerTest {
     expect(token.next.type, TokenType.EOF);
   }
 
-  void _assertLineInfo(
-      String source, List<ScannerTest_ExpectedLocation> expectedLocations) {
-    GatheringErrorListener listener = new GatheringErrorListener();
-    _scanWithListener(source, listener);
-    listener.assertNoErrors();
-    LineInfo info = listener.getLineInfo(new TestSource());
-    expect(info, isNotNull);
-    int count = expectedLocations.length;
-    for (int i = 0; i < count; i++) {
-      ScannerTest_ExpectedLocation expectedLocation = expectedLocations[i];
-      LineInfo_Location location = info.getLocation(expectedLocation._offset);
-      expect(location.lineNumber, expectedLocation._lineNumber,
-          reason: 'Line number in location $i');
-      expect(location.columnNumber, expectedLocation._columnNumber,
-          reason: 'Column number in location $i');
-    }
-  }
-
   /**
    * Assert that the token scanned from the given [source] has the
    * [expectedType].
@@ -1268,7 +1140,7 @@ class ScannerTest {
   Token _scan(String source,
       {bool genericMethodComments: false,
       bool lazyAssignmentOperators: false}) {
-    GatheringErrorListener listener = new GatheringErrorListener();
+    _ErrorListener listener = new _ErrorListener();
     Token token = _scanWithListener(source, listener,
         genericMethodComments: genericMethodComments,
         lazyAssignmentOperators: lazyAssignmentOperators);
@@ -1276,102 +1148,19 @@ class ScannerTest {
     return token;
   }
 
-  Token _scanWithListener(String source, GatheringErrorListener listener,
+  Token _scanWithListener(String source, _ErrorListener listener,
       {bool genericMethodComments: false,
       bool lazyAssignmentOperators: false}) {
     Scanner scanner =
-        new Scanner(null, new CharSequenceReader(source), listener);
+        new _TestScanner(new CharSequenceReader(source), listener);
     scanner.scanGenericMethodComments = genericMethodComments;
     scanner.scanLazyAssignmentOperators = lazyAssignmentOperators;
-    Token result = scanner.tokenize();
-    listener.setLineInfo(new TestSource(), scanner.lineStarts);
-    return result;
-  }
-}
-
-/**
- * An `ExpectedLocation` encodes information about the expected location of a
- * given offset in source code.
- */
-class ScannerTest_ExpectedLocation {
-  final int _offset;
-
-  final int _lineNumber;
-
-  final int _columnNumber;
-
-  ScannerTest_ExpectedLocation(
-      this._offset, this._lineNumber, this._columnNumber);
-}
-
-/**
- * A `TokenStreamValidator` is used to validate the correct construction of a
- * stream of tokens.
- */
-class TokenStreamValidator {
-  /**
-   * Validate that the stream of tokens that starts with the given [token] is
-   * correct.
-   */
-  void validate(Token token) {
-    StringBuffer buffer = new StringBuffer();
-    _validateStream(buffer, token);
-    if (buffer.length > 0) {
-      fail(buffer.toString());
-    }
-  }
-
-  void _validateStream(StringBuffer buffer, Token token) {
-    if (token == null) {
-      return;
-    }
-    Token previousToken = null;
-    int previousEnd = -1;
-    Token currentToken = token;
-    while (currentToken != null && currentToken.type != TokenType.EOF) {
-      _validateStream(buffer, currentToken.precedingComments);
-      TokenType type = currentToken.type;
-      if (type == TokenType.OPEN_CURLY_BRACKET ||
-          type == TokenType.OPEN_PAREN ||
-          type == TokenType.OPEN_SQUARE_BRACKET ||
-          type == TokenType.STRING_INTERPOLATION_EXPRESSION) {
-        if (currentToken is! BeginToken) {
-          buffer.write("\r\nExpected BeginToken, found ");
-          buffer.write(currentToken.runtimeType.toString());
-          buffer.write(" ");
-          _writeToken(buffer, currentToken);
-        }
-      }
-      int currentStart = currentToken.offset;
-      int currentLength = currentToken.length;
-      int currentEnd = currentStart + currentLength - 1;
-      if (currentStart <= previousEnd) {
-        buffer.write("\r\nInvalid token sequence: ");
-        _writeToken(buffer, previousToken);
-        buffer.write(" followed by ");
-        _writeToken(buffer, currentToken);
-      }
-      previousEnd = currentEnd;
-      previousToken = currentToken;
-      currentToken = currentToken.next;
-    }
-  }
-
-  void _writeToken(StringBuffer buffer, Token token) {
-    buffer.write("[");
-    buffer.write(token.type);
-    buffer.write(", '");
-    buffer.write(token.lexeme);
-    buffer.write("', ");
-    buffer.write(token.offset);
-    buffer.write(", ");
-    buffer.write(token.length);
-    buffer.write("]");
+    return scanner.tokenize();
   }
 }
 
 @reflectiveTest
-class TokenTypeTest extends EngineTestCase {
+class TokenTypeTest {
   void test_isOperator() {
     expect(TokenType.AMPERSAND.isOperator, isTrue);
     expect(TokenType.AMPERSAND_AMPERSAND.isOperator, isTrue);
@@ -1437,3 +1226,73 @@ class TokenTypeTest extends EngineTestCase {
     expect(TokenType.TILDE_SLASH.isUserDefinableOperator, isTrue);
   }
 }
+
+class _ErrorListener {
+  final errors = <_TestError>[];
+
+  void assertErrors(List<_TestError> expectedErrors) {
+    expect(errors, unorderedEquals(expectedErrors));
+  }
+
+  void assertNoErrors() {
+    assertErrors([]);
+  }
+}
+
+class _TestError {
+  final int offset;
+  final int length;
+  final ErrorCode errorCode;
+  final List<Object> arguments;
+
+  _TestError(this.offset, this.length, this.errorCode, this.arguments);
+
+  @override
+  get hashCode {
+    var h = new JenkinsSmiHash()..add(offset)..add(length)..add(errorCode);
+    if (arguments != null) {
+      for (Object argument in arguments) {
+        h.add(argument);
+      }
+    }
+    return h.hashCode;
+  }
+
+  @override
+  operator ==(Object other) {
+    if (other is _TestError &&
+        offset == other.offset &&
+        length == other.length &&
+        errorCode == other.errorCode) {
+      if (arguments == null) return other.arguments == null;
+      if (other.arguments == null) return false;
+      if (arguments.length != other.arguments.length) return false;
+      for (int i = 0; i < arguments.length; i++) {
+        if (arguments[i] != other.arguments[i]) return false;
+      }
+      return true;
+    }
+    return false;
+  }
+
+  @override
+  toString() {
+    var end = offset + length;
+    var argString = arguments == null ? '' : '(${arguments.join(', ')})';
+    return 'Error($offset..$end, $errorCode$argString)';
+  }
+}
+
+class _TestScanner extends Scanner {
+  final _ErrorListener listener;
+
+  _TestScanner(CharacterReader reader, [this.listener]) : super(reader);
+
+  @override
+  void reportError(
+      ScannerErrorCode errorCode, int offset, List<Object> arguments) {
+    if (listener != null) {
+      listener.errors.add(new _TestError(offset, 1, errorCode, arguments));
+    }
+  }
+}