Line pre-chunking.

pkg/analyzer/test/services/formatter_test.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:io';
 
 import 'package:path/path.dart';
 import 'package:unittest/unittest.dart';
 
 import 'package:analyzer/src/generated/java_core.dart' show CharSequence;
@@ skipping 1219 matching lines @@
       var line = new Line();
       tokens.forEach((t) =>
           line.addToken(t is LineToken ? t : new LineToken(t)));
       return line;
     }
 
     expectTextsEqual(List<Chunk> chunks, List<String> texts) {
       expect(chunks.map((chunk) => chunk.toString()), orderedEquals(texts));
     }
 
+    expectTokensEqual(List<LineToken> tokens, List<String> texts) {
+      expect(tokens.map((token) => token.toString()), orderedEquals(texts));
+    }
+
+
     final SP_1 = new SpaceToken(1, breakWeight: 1);
 
     // 'foo|1|bar|1|baz|1|foo|1|bar|1|baz'
     final LINE_1 = line(['foo', SP_1, 'bar', SP_1, 'baz', SP_1,
                          'foo', SP_1, 'bar', SP_1, 'baz']);
 
     // '  foo|1|bar|1|baz|1|foo|1|bar|1|baz'
     final LINE_2 = line(['  foo', SP_1, 'bar', SP_1, 'baz', SP_1,
                          'foo', SP_1, 'bar', SP_1, 'baz']);
 
@@ skipping 53 matching lines @@
       expect(isWhitespace('foo  '), false);
       expect(isWhitespace(' foo '), false);
       expect(isWhitespace(' '), true);
       expect(isWhitespace('  '), true);
       expect(isWhitespace('\t'), true);
       expect(isWhitespace('\t\t'), true);
       expect(isWhitespace('\n'), true);
       expect(isWhitespace('\r'), true);
     });
 
+    test('preprocess - 1', () {
+      var tokens = line(['f', 'o', 'o', SP_1, 'b', 'a', 'r']).tokens;
+      var processed = SimpleLineBreaker.preprocess(tokens);
+      expectTokensEqual(processed, ['foo', ' ', 'bar']);
+    });
+
+    test('preprocess - 2', () {
+      var tokens = line(['f', 'o', 'o', SP_1, SP_1, 'b', 'a', 'r']).tokens;
+      var processed = SimpleLineBreaker.preprocess(tokens);
+      expectTokensEqual(processed, ['foo', ' ', ' ', 'bar']);
+    });
+
+    test('preprocess - 3', () {
+      var tokens = line(['f', 'o', 'o', SP_1, 'b', 'a', 'r', SP_1]).tokens;
+      var processed = SimpleLineBreaker.preprocess(tokens);
+      expectTokensEqual(processed, ['foo', ' ', 'bar', ' ']);
+    });
+
   });
 
   /// Helper method tests
   group('helpers', () {
 
     test('indentString', () {
       expect(getIndentString(0), '');
       expect(getIndentString(1), '  ');
       expect(getIndentString(4), '        ');
     });
@@ skipping 102 matching lines @@
       input += lines[i++] + '\n';
     }
     while(++i < lines.length && !lines[i].startsWith('>>>')) {
       expectedOutput += lines[i] + '\n';
     }
     test('test - (${testIndex++})', () {
       expectClause(input, expectedOutput);
     });
   }
 }