Formatter sanity-checking (via token stream verification).

pkg/analyzer_experimental/test/services/formatter_test.dart

Index: pkg/analyzer_experimental/test/services/formatter_test.dart
===================================================================
--- pkg/analyzer_experimental/test/services/formatter_test.dart	(revision 27307)
+++ pkg/analyzer_experimental/test/services/formatter_test.dart	(working copy)
@@ -410,7 +410,7 @@
           '}\n'
       );
     });
-    
+
     test('CU - mixed comments', () {
       expectCUFormatsTo(
           'library foo;\n'
@@ -431,14 +431,14 @@
           '/* Comment 3 */'
         );
     });
-    
+
     test('CU - comments (EOF)', () {
       expectCUFormatsTo(
           'library foo; //zamm',
           'library foo; //zamm\n' //<-- note extra NEWLINE
         );
-    });  
-    
+    });
+
     test('CU - comments (0)', () {
       expectCUFormatsTo(
           'library foo; //zamm\n'
@@ -458,11 +458,11 @@
           '/* foo */ /* bar */\n'
       );
     });
-    
+
     test('CU - comments (2)', () {
       expectCUFormatsTo(
           '/** foo */ /** bar */\n',
-          '/** foo */\n' 
+          '/** foo */\n'
           '/** bar */\n'
       );
     });
@@ -482,7 +482,7 @@
           '}'
       );
     });
-    
+
     test('CU - comments (5)', () {
       expectCUFormatsTo(
           '//comment one\n\n'
@@ -490,22 +490,22 @@
           '//comment one\n\n'
           '//comment two\n\n'
       );
-    });    
-    
+    });
+
     test('CU - comments (6)', () {
       expectCUFormatsTo(
           'var x;   //x\n',
           'var x; //x\n'
       );
-    });    
+    });
 
     test('CU - comments (6)', () {
       expectCUFormatsTo(
           'var /* int */ x; //x\n',
           'var /* int */ x; //x\n'
       );
-    });    
-    
+    });
+
     test('CU - comments (7)', () {
       expectCUFormatsTo(
           'library foo;\n'
@@ -534,8 +534,8 @@
           'int x;\n'
         );
     });
-    
-    
+
+
     test('CU - constructor', () {
       expectCUFormatsTo(
           'class A {\n'
@@ -750,6 +750,48 @@
   });
 
 
+  /// Token streams
+  group('token streams', () {
+
+    test('string tokens', () {
+      expectTokenizedEqual('class A{}', 'class A{ }');

[Brian Wilkerson, 2013/09/09 22:21:47]

In all of the equal cases the strings are identical. It would be good to test
with strings that have whitespace differences.

[pquitslund, 2013/09/10 18:24:13]

Added some.  Thanks!

+    });
+
+    test('string tokens - w/ comments', () {
+      expectTokenizedEqual('//foo\nint bar;', '//foo\nint bar;');
+      expectTokenizedNotEqual('int bar;', '//foo\nint bar;');
+      expectTokenizedNotEqual('//foo\nint bar;', 'int bar;');
+    });
+
+    test('INDEX', () {
+      /// '[' ']' => '[]'
+      var t1 = openSqBracket()..setNext(closeSqBracket()..setNext(eof()));
+      var t2 = index()..setNext(eof());
+      expectStreamsEqual(t1, t2);
+    });
+
+    test('GT_GT', () {
+      /// '>' '>' => '>>'
+      var t1 = gt()..setNext(gt()..setNext(eof()));
+      var t2 = gt_gt()..setNext(eof());
+      expectStreamsEqual(t1, t2);
+    });
+
+    test('t1 < t2', () {
+      var t1 = string('foo')..setNext(eof());
+      var t2 = string('foo')..setNext(string('bar')..setNext(eof()));
+      expectStreamsNotEqual(t1, t2);
+    });
+
+    test('t1 > t2', () {
+      var t1 = string('foo')..setNext(string('bar')..setNext(eof()));
+      var t2 = string('foo')..setNext(eof());
+      expectStreamsNotEqual(t1, t2);
+    });
+
+  });
+
+
   /// Line tests
   group('line', () {
 
@@ -853,9 +895,22 @@
 
 }
 
-Token classKeyword(int offset) =>
-    new KeywordToken(Keyword.CLASS, offset);
+Token closeSqBracket() => new Token(TokenType.CLOSE_SQUARE_BRACKET, 0);
 
+Token eof() => new Token(TokenType.EOF, 0);
+
+Token gt() => new Token(TokenType.GT, 0);
+
+Token gt_gt() => new Token(TokenType.GT_GT, 0);
+
+Token index() => new Token(TokenType.INDEX, 0);
+
+Token openSqBracket() => new BeginToken(TokenType.OPEN_SQUARE_BRACKET, 0);
+
+Token string(String lexeme) => new StringToken(TokenType.STRING, lexeme, 0);

[Brian Wilkerson, 2013/09/09 22:21:47]

There is a TokenFactory class in the engine tests that might be useful.

[pquitslund, 2013/09/10 18:24:13]

I'll look.  Thanks!

+
+Token classKeyword(int offset) => new KeywordToken(Keyword.CLASS, offset);
+
 Token identifier(String value, int offset) =>
     new StringToken(TokenType.IDENTIFIER, value, offset);
 
@@ -878,6 +933,23 @@
 String formatStatement(src, {options: const FormatterOptions()}) =>
     new CodeFormatter(options).format(CodeKind.STATEMENT, src);
 
+Token tokenize(String str) => new StringScanner(null, str, null).tokenize();
+
+
+expectTokenizedEqual(String s1, String s2) =>
+    expectStreamsEqual(tokenize(s1), tokenize(s2));
+
+expectTokenizedNotEqual(String s1, String s2) =>
+    expect(()=> expectStreamsEqual(tokenize(s1), tokenize(s2)),
+    throwsA(new isInstanceOf<FormatterException>()));
+
+expectStreamsEqual(Token t1, Token t2) =>
+    new TokenStreamComparator(null, t1, t2).verifyEquals();
+
+expectStreamsNotEqual(Token t1, Token t2) =>
+    expect(() => new TokenStreamComparator(null, t1, t2).verifyEquals(),
+    throwsA(new isInstanceOf<FormatterException>()));
+
 expectCUFormatsTo(src, expected) => expect(formatCU(src), equals(expected));
 
 expectStmtFormatsTo(src, expected) => expect(formatStatement(src),