Line pre-chunking.

pkg/analyzer/lib/src/services/writer.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library source_writer;
 
 
 /// DEBUG flag indicating whether to use the experimental line-breaker
 const _USE_LINE_BREAKER = false;
 
@@ skipping 77 matching lines @@
       }
     }
     return buf.toString();
   }
 
   String indent(Chunk chunk, int level) =>
       '\n' + indenter(level + 2) + chunk.toString();
 
   List<Chunk> breakLine(Line line) {
 
+    var tokens = preprocess(line.tokens);
+
     var chunks = <Chunk>[];
 
     // The current unbroken line
     var current = new Chunk(maxLength: maxLength);
 
     // A tentative working chunk that will either start a new line or get
     // absorbed into 'current'
     var work = new Chunk(maxLength: maxLength);
 
-    line.tokens.forEach((tok) {
+    tokens.forEach((tok) {
 
       if (goodStart(tok, work)) {
         if (current.fits(work)) {
           current.add(work);
         } else {
           if (current.length > 0) {
             chunks.add(current);
           }
           current = work;
         }
@@ skipping 16 matching lines @@
 
     });
 
     current.add(work);
     if (current.length > 0) {
       chunks.add(current);
     }
     return chunks;
   }
 
+  static List<LineToken> preprocess(List<LineToken> tok) {
+
+    var tokens = <LineToken>[];
+    var curr;
+
+    tok.forEach((token){
+      if (token is! SpaceToken) {
+        if (curr == null) {
+          curr = token;
+        } else {
+          curr = merge(curr, token);
+        }
+      } else {
+        if (curr != null) {
+          tokens.add(curr);
+          curr = null;
+        }
+        tokens.add(token);
+      }
+    });
+
+    if (curr != null) {
+      tokens.add(curr);
+    }
+
+    return tokens;
+  }
+
+  static LineToken merge(LineToken first, LineToken second) =>
+      new LineToken(first.value + second.value);
+
   bool isAllWhitespace(Chunk chunk) => isWhitespace(chunk.buffer.toString());
 
   /// Test whether this token is a good start for a new working chunk
   bool goodStart(LineToken tok, Chunk workingChunk) =>
       tok is SpaceToken && tok.breakWeight >= workingChunk.start.breakWeight;
 
 }
 
 /// Test if this [string] contains only whitespace characters
 bool isWhitespace(String string) => string.codeUnits.every(
@@ skipping 220 matching lines @@
 
 String getTabs(int n) => n < TABS.length ? TABS[n] : repeat('\t', n);
 
 String repeat(String ch, int times) {
   var sb = new StringBuffer();
   for (var i = 0; i < times; ++i) {
     sb.write(ch);
   }
   return sb.toString();
 }