Formatter sanity-checking (via token stream verification).

pkg/analyzer_experimental/test/services/formatter_test.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:unittest/unittest.dart';
 
 import 'package:analyzer_experimental/src/generated/scanner.dart';
 import 'package:analyzer_experimental/src/services/formatter_impl.dart';
 import 'package:analyzer_experimental/src/services/writer.dart';
 
@@ skipping 392 matching lines @@
           '}\n',
           '/// Copyright info\n'
           '\n'
           'library foo;\n'
           '/// Class comment\n'
           '//TODO: implement\n'
           'class C {\n'
           '}\n'
       );
     });
-    
+
     test('CU - mixed comments', () {
       expectCUFormatsTo(
           'library foo;\n'
           '\n'
           '\n'
           '/* Comment 1 */\n'
           '\n'
           '// Comment 2\n'
           '\n'
           '/* Comment 3 */',
           'library foo;\n'
           '\n'
           '\n'
           '/* Comment 1 */\n'
           '\n'
           '// Comment 2\n'
           '\n'
           '/* Comment 3 */'
         );
     });
-    
+
     test('CU - comments (EOF)', () {
       expectCUFormatsTo(
           'library foo; //zamm',
           'library foo; //zamm\n' //<-- note extra NEWLINE
         );
-    });  
-    
+    });
+
     test('CU - comments (0)', () {
       expectCUFormatsTo(
           'library foo; //zamm\n'
           '\n'
           'class A {\n'
           '}\n',
           'library foo; //zamm\n'
           '\n'
           'class A {\n'
           '}\n'
         );
     });
 
     test('CU - comments (1)', () {
       expectCUFormatsTo(
           '/* foo */ /* bar */\n',
           '/* foo */ /* bar */\n'
       );
     });
-    
+
     test('CU - comments (2)', () {
       expectCUFormatsTo(
           '/** foo */ /** bar */\n',
-          '/** foo */\n' 
+          '/** foo */\n'
           '/** bar */\n'
       );
     });
 
     test('CU - comments (3)', () {
       expectCUFormatsTo(
           'var x;   //x\n',
           'var x; //x\n'
       );
     });
 
     test('CU - comments (4)', () {
       expectCUFormatsTo(
           'class X { //X!\n'
           '}',
           'class X { //X!\n'
           '}'
       );
     });
-    
+
     test('CU - comments (5)', () {
       expectCUFormatsTo(
           '//comment one\n\n'
           '//comment two\n\n',
           '//comment one\n\n'
           '//comment two\n\n'
       );
-    });    
-    
+    });
+
     test('CU - comments (6)', () {
       expectCUFormatsTo(
           'var x;   //x\n',
           'var x; //x\n'
       );
-    });    
+    });
 
     test('CU - comments (6)', () {
       expectCUFormatsTo(
           'var /* int */ x; //x\n',
           'var /* int */ x; //x\n'
       );
-    });    
-    
+    });
+
     test('CU - comments (7)', () {
       expectCUFormatsTo(
           'library foo;\n'
           '\n'
           '/// Docs\n'
           '/// spanning\n'
           '/// lines.\n'
           'class A {\n'
           '}\n'
           '\n'
           '/// ... and\n'
           '\n'
           '/// Dangling ones too\n'
           'int x;\n',
           'library foo;\n'
           '\n'
           '/// Docs\n'
           '/// spanning\n'
           '/// lines.\n'
           'class A {\n'
           '}\n'
           '\n'
           '/// ... and\n'
           '\n'
           '/// Dangling ones too\n'
           'int x;\n'
         );
     });
-    
-    
+
+
     test('CU - constructor', () {
       expectCUFormatsTo(
           'class A {\n'
           '  const _a;\n'
           '  A();\n'
           '  int a() => _a;\n'
           '}\n',
           'class A {\n'
           '  const _a;\n'
           '  A();\n'
@@ skipping 194 matching lines @@
     test('initialIndent', () {
       var formatter = new CodeFormatter(
           new FormatterOptions(initialIndentationLevel: 2));
       var formattedSource = formatter.format(CodeKind.STATEMENT, 'var x;');
       expect(formattedSource, startsWith('    '));
     });
 
   });
 
 
+  /// Token streams
+  group('token streams', () {
+
+    test('string tokens', () {
+      expectTokenizedEqual('class A{}', 'class A{ }');

[Brian Wilkerson, 2013/09/09 22:21:47]

In all of the equal cases the strings are identical. It would be good to test
with strings that have whitespace differences.

[pquitslund, 2013/09/10 18:24:13]

Added some.  Thanks!

+    });
+
+    test('string tokens - w/ comments', () {
+      expectTokenizedEqual('//foo\nint bar;', '//foo\nint bar;');
+      expectTokenizedNotEqual('int bar;', '//foo\nint bar;');
+      expectTokenizedNotEqual('//foo\nint bar;', 'int bar;');
+    });
+
+    test('INDEX', () {
+      /// '[' ']' => '[]'
+      var t1 = openSqBracket()..setNext(closeSqBracket()..setNext(eof()));
+      var t2 = index()..setNext(eof());
+      expectStreamsEqual(t1, t2);
+    });
+
+    test('GT_GT', () {
+      /// '>' '>' => '>>'
+      var t1 = gt()..setNext(gt()..setNext(eof()));
+      var t2 = gt_gt()..setNext(eof());
+      expectStreamsEqual(t1, t2);
+    });
+
+    test('t1 < t2', () {
+      var t1 = string('foo')..setNext(eof());
+      var t2 = string('foo')..setNext(string('bar')..setNext(eof()));
+      expectStreamsNotEqual(t1, t2);
+    });
+
+    test('t1 > t2', () {
+      var t1 = string('foo')..setNext(string('bar')..setNext(eof()));
+      var t2 = string('foo')..setNext(eof());
+      expectStreamsNotEqual(t1, t2);
+    });
+
+  });
+
+
   /// Line tests
   group('line', () {
 
     test('space', () {
       var line = new Line(indent: 0);
       line.addSpaces(2);
       expect(line.toString(), equals('  '));
     });
 
     test('initial indent', () {
@@ skipping 83 matching lines @@
     test('repeat', () {
       expect(repeat('x', 0), '');
       expect(repeat('x', 1), 'x');
       expect(repeat('x', 4), 'xxxx');
     });
 
   });
 
 }
 
-Token classKeyword(int offset) =>
-    new KeywordToken(Keyword.CLASS, offset);
+Token closeSqBracket() => new Token(TokenType.CLOSE_SQUARE_BRACKET, 0);
+
+Token eof() => new Token(TokenType.EOF, 0);
+
+Token gt() => new Token(TokenType.GT, 0);
+
+Token gt_gt() => new Token(TokenType.GT_GT, 0);
+
+Token index() => new Token(TokenType.INDEX, 0);
+
+Token openSqBracket() => new BeginToken(TokenType.OPEN_SQUARE_BRACKET, 0);
+
+Token string(String lexeme) => new StringToken(TokenType.STRING, lexeme, 0);

[Brian Wilkerson, 2013/09/09 22:21:47]

There is a TokenFactory class in the engine tests that might be useful.

[pquitslund, 2013/09/10 18:24:13]

I'll look.  Thanks!

+
+Token classKeyword(int offset) => new KeywordToken(Keyword.CLASS, offset);
 
 Token identifier(String value, int offset) =>
     new StringToken(TokenType.IDENTIFIER, value, offset);
 
 Token openParen(int offset) =>
     new StringToken(TokenType.OPEN_PAREN, '{', offset);
 
 Token closeParen(int offset) =>
     new StringToken(TokenType.CLOSE_PAREN, '}', offset);
 
 Token chain(List<Token> tokens) {
   for (var i = 0; i < tokens.length - 1; ++i) {
     tokens[i].setNext(tokens[i + 1]);
   }
   return tokens[0];
 }
 
 String formatCU(src, {options: const FormatterOptions()}) =>
     new CodeFormatter(options).format(CodeKind.COMPILATION_UNIT, src);
 
 String formatStatement(src, {options: const FormatterOptions()}) =>
     new CodeFormatter(options).format(CodeKind.STATEMENT, src);
 
+Token tokenize(String str) => new StringScanner(null, str, null).tokenize();
+
+
+expectTokenizedEqual(String s1, String s2) =>
+    expectStreamsEqual(tokenize(s1), tokenize(s2));
+
+expectTokenizedNotEqual(String s1, String s2) =>
+    expect(()=> expectStreamsEqual(tokenize(s1), tokenize(s2)),
+    throwsA(new isInstanceOf<FormatterException>()));
+
+expectStreamsEqual(Token t1, Token t2) =>
+    new TokenStreamComparator(null, t1, t2).verifyEquals();
+
+expectStreamsNotEqual(Token t1, Token t2) =>
+    expect(() => new TokenStreamComparator(null, t1, t2).verifyEquals(),
+    throwsA(new isInstanceOf<FormatterException>()));
+
 expectCUFormatsTo(src, expected) => expect(formatCU(src), equals(expected));
 
 expectStmtFormatsTo(src, expected) => expect(formatStatement(src),
     equals(expected));