remove most string concat, fixes #7

lib/src/tokenizer.dart

 library tokenizer;
 
 import 'dart:collection';
 import 'package:html/parser.dart' show HtmlParser;
 import 'constants.dart';
 import 'inputstream.dart';
 import 'token.dart';
 import 'utils.dart';
 
 // Group entities by their first character, for faster lookups
@@ skipping 37 matching lines @@
   final Queue<Token> tokenQueue;
 
   /// Holds the token that is currently being processed.
   Token currentToken;
 
   /// Holds a reference to the method to be invoked for the next parser state.
   // TODO(jmesserly): the type should be "Predicate" but a dart2js checked mode
   // bug prevents us from doing that. See http://dartbug.com/12465
   Function state;
 
-  String temporaryBuffer;
+  final StringBuffer _buffer = new StringBuffer();
 
   int _lastOffset;
 
   // TODO(jmesserly): ideally this would be a LinkedHashMap and we wouldn't add
   // an item until it's ready. But the code doesn't have a clear notion of when
   // it's "done" with the attribute.
   List<TagAttribute> _attributes;
   Set<String> _attributeNames;
 
   HtmlTokenizer(doc, {String encoding, bool parseMeta: true,
       this.lowercaseElementName: true, this.lowercaseAttrName: true,
       bool generateSpans: false, String sourceUrl, this.attributeSpans: false})
       : stream = new HtmlInputStream(
           doc, encoding, parseMeta, generateSpans, sourceUrl),
         tokenQueue = new Queue(),
         generateSpans = generateSpans {
     reset();
   }
 
   TagToken get currentTagToken => currentToken;
   DoctypeToken get currentDoctypeToken => currentToken;
   StringToken get currentStringToken => currentToken;
 
   Token _current;
   Token get current => _current;
 
-  String get _attributeName => _attributes.last.name;
-  set _attributeName(String value) {
-    _attributes.last.name = value;
-  }
-
-  String get _attributeValue => _attributes.last.value;
-  set _attributeValue(String value) {
-    _attributes.last.value = value;
-  }
+  final StringBuffer _attributeName = new StringBuffer();
+  final StringBuffer _attributeValue = new StringBuffer();
 
   void _markAttributeEnd(int offset) {
+    _attributes.last.value = '$_attributeValue';
     if (attributeSpans) _attributes.last.end = stream.position + offset;
   }
 
   void _markAttributeValueStart(int offset) {
     if (attributeSpans) _attributes.last.startValue = stream.position + offset;
   }
 
   void _markAttributeValueEnd(int offset) {
-    if (attributeSpans) {
-      _attributes.last.endValue = stream.position + offset;
-      _markAttributeEnd(offset);
-    }
+    if (attributeSpans) _attributes.last.endValue = stream.position + offset;
+    _markAttributeEnd(offset);
   }
 
   // Note: we could track the name span here, if we need it.
   void _markAttributeNameEnd(int offset) => _markAttributeEnd(offset);
 
   void _addAttribute(String name) {
     if (_attributes == null) _attributes = [];
-    var attr = new TagAttribute(name);
+    _attributeName.clear();
+    _attributeName.write(name);
+    _attributeValue.clear();
+    var attr = new TagAttribute();
     _attributes.add(attr);
     if (attributeSpans) attr.start = stream.position - name.length;
   }
 
   /// This is where the magic happens.
   ///
   /// We do our usually processing through the states and when we have a token
   /// to return we yield the token which pauses processing until the next token
   /// is requested.
   bool moveNext() {
@@ skipping 13 matching lines @@
     }
     return true;
   }
 
   /// Resets the tokenizer state. Calling this does not reset the [stream] or
   /// the [parser].
   void reset() {
     _lastOffset = 0;
     tokenQueue.clear();
     currentToken = null;
-    temporaryBuffer = null;
+    _buffer.clear();
     _attributes = null;
     _attributeNames = null;
     state = dataState;
   }
 
   /// Adds a token to the queue. Sets the span if needed.
   void _addToken(Token token) {
     if (generateSpans && token.span == null) {
       int offset = stream.position;
       token.span = stream.fileInfo.span(_lastOffset, offset);
@@ skipping 180 matching lines @@
           stream.unget(charStack.removeLast());
           output = '${output}${slice(charStack, entityLen).join()}';
         }
       } else {
         _addToken(new ParseErrorToken("expected-named-entity"));
         stream.unget(charStack.removeLast());
         output = "&${charStack.join()}";
       }
     }
     if (fromAttribute) {
-      _attributeValue = '$_attributeValue$output';
+      _attributeValue.write(output);
     } else {
       var token;
       if (isWhitespace(output)) {
         token = new SpaceCharactersToken(output);
       } else {
         token = new CharactersToken(output);
       }
       _addToken(token);
     }
   }
@@ skipping 225 matching lines @@
       currentTagToken.name = '${currentTagToken.name}$data';
       // (Don't use charsUntil here, because tag names are
       // very short and it's faster to not do anything fancy)
     }
     return true;
   }
 
   bool rcdataLessThanSignState() {
     var data = stream.char();
     if (data == "/") {
-      temporaryBuffer = "";
+      _buffer.clear();
       state = rcdataEndTagOpenState;
     } else {
       _addToken(new CharactersToken("<"));
       stream.unget(data);
       state = rcdataState;
     }
     return true;
   }
 
   bool rcdataEndTagOpenState() {
     var data = stream.char();
     if (isLetter(data)) {
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
       state = rcdataEndTagNameState;
     } else {
       _addToken(new CharactersToken("</"));
       stream.unget(data);
       state = rcdataState;
     }
     return true;
   }
 
   bool _tokenIsAppropriate() {
+    // TODO(jmesserly): this should use case insensitive compare instead.
     return currentToken is TagToken &&
-        currentTagToken.name.toLowerCase() == temporaryBuffer.toLowerCase();
+        currentTagToken.name.toLowerCase() == '$_buffer'.toLowerCase();
   }
 
   bool rcdataEndTagNameState() {
     var appropriate = _tokenIsAppropriate();
     var data = stream.char();
     if (isWhitespace(data) && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       state = beforeAttributeNameState;
     } else if (data == "/" && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       state = selfClosingStartTagState;
     } else if (data == ">" && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       emitCurrentToken();
       state = dataState;
     } else if (isLetter(data)) {
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
     } else {
-      _addToken(new CharactersToken("</$temporaryBuffer"));
+      _addToken(new CharactersToken("</$_buffer"));
       stream.unget(data);
       state = rcdataState;
     }
     return true;
   }
 
   bool rawtextLessThanSignState() {
     var data = stream.char();
     if (data == "/") {
-      temporaryBuffer = "";
+      _buffer.clear();
       state = rawtextEndTagOpenState;
     } else {
       _addToken(new CharactersToken("<"));
       stream.unget(data);
       state = rawtextState;
     }
     return true;
   }
 
   bool rawtextEndTagOpenState() {
     var data = stream.char();
     if (isLetter(data)) {
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
       state = rawtextEndTagNameState;
     } else {
       _addToken(new CharactersToken("</"));
       stream.unget(data);
       state = rawtextState;
     }
     return true;
   }
 
   bool rawtextEndTagNameState() {
     var appropriate = _tokenIsAppropriate();
     var data = stream.char();
     if (isWhitespace(data) && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       state = beforeAttributeNameState;
     } else if (data == "/" && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       state = selfClosingStartTagState;
     } else if (data == ">" && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       emitCurrentToken();
       state = dataState;
     } else if (isLetter(data)) {
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
     } else {
-      _addToken(new CharactersToken("</$temporaryBuffer"));
+      _addToken(new CharactersToken("</$_buffer"));
       stream.unget(data);
       state = rawtextState;
     }
     return true;
   }
 
   bool scriptDataLessThanSignState() {
     var data = stream.char();
     if (data == "/") {
-      temporaryBuffer = "";
+      _buffer.clear();
       state = scriptDataEndTagOpenState;
     } else if (data == "!") {
       _addToken(new CharactersToken("<!"));
       state = scriptDataEscapeStartState;
     } else {
       _addToken(new CharactersToken("<"));
       stream.unget(data);
       state = scriptDataState;
     }
     return true;
   }
 
   bool scriptDataEndTagOpenState() {
     var data = stream.char();
     if (isLetter(data)) {
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
       state = scriptDataEndTagNameState;
     } else {
       _addToken(new CharactersToken("</"));
       stream.unget(data);
       state = scriptDataState;
     }
     return true;
   }
 
   bool scriptDataEndTagNameState() {
     var appropriate = _tokenIsAppropriate();
     var data = stream.char();
     if (isWhitespace(data) && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       state = beforeAttributeNameState;
     } else if (data == "/" && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       state = selfClosingStartTagState;
     } else if (data == ">" && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       emitCurrentToken();
       state = dataState;
     } else if (isLetter(data)) {
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
     } else {
-      _addToken(new CharactersToken("</$temporaryBuffer"));
+      _addToken(new CharactersToken("</$_buffer"));
       stream.unget(data);
       state = scriptDataState;
     }
     return true;
   }
 
   bool scriptDataEscapeStartState() {
     var data = stream.char();
     if (data == "-") {
       _addToken(new CharactersToken("-"));
@@ skipping 74 matching lines @@
     } else {
       _addToken(new CharactersToken(data));
       state = scriptDataEscapedState;
     }
     return true;
   }
 
   bool scriptDataEscapedLessThanSignState() {
     var data = stream.char();
     if (data == "/") {
-      temporaryBuffer = "";
+      _buffer.clear();
       state = scriptDataEscapedEndTagOpenState;
     } else if (isLetter(data)) {
       _addToken(new CharactersToken("<$data"));
-      temporaryBuffer = data;
+      _buffer.clear();
+      _buffer.write(data);
       state = scriptDataDoubleEscapeStartState;
     } else {
       _addToken(new CharactersToken("<"));
       stream.unget(data);
       state = scriptDataEscapedState;
     }
     return true;
   }
 
   bool scriptDataEscapedEndTagOpenState() {
     var data = stream.char();
     if (isLetter(data)) {
-      temporaryBuffer = data;
+      _buffer.clear();
+      _buffer.write(data);
       state = scriptDataEscapedEndTagNameState;
     } else {
       _addToken(new CharactersToken("</"));
       stream.unget(data);
       state = scriptDataEscapedState;
     }
     return true;
   }
 
   bool scriptDataEscapedEndTagNameState() {
     var appropriate = _tokenIsAppropriate();
     var data = stream.char();
     if (isWhitespace(data) && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       state = beforeAttributeNameState;
     } else if (data == "/" && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       state = selfClosingStartTagState;
     } else if (data == ">" && appropriate) {
-      currentToken = new EndTagToken(temporaryBuffer);
+      currentToken = new EndTagToken('$_buffer');
       emitCurrentToken();
       state = dataState;
     } else if (isLetter(data)) {
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
     } else {
-      _addToken(new CharactersToken("</$temporaryBuffer"));
+      _addToken(new CharactersToken("</$_buffer"));
       stream.unget(data);
       state = scriptDataEscapedState;
     }
     return true;
   }
 
   bool scriptDataDoubleEscapeStartState() {
     var data = stream.char();
     if (isWhitespace(data) || data == "/" || data == ">") {
       _addToken(new CharactersToken(data));
-      if (temporaryBuffer.toLowerCase() == "script") {
+      if ('$_buffer'.toLowerCase() == "script") {
         state = scriptDataDoubleEscapedState;
       } else {
         state = scriptDataEscapedState;
       }
     } else if (isLetter(data)) {
       _addToken(new CharactersToken(data));
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
     } else {
       stream.unget(data);
       state = scriptDataEscapedState;
     }
     return true;
   }
 
   bool scriptDataDoubleEscapedState() {
     var data = stream.char();
     if (data == "-") {
@@ skipping 59 matching lines @@
       _addToken(new CharactersToken(data));
       state = scriptDataDoubleEscapedState;
     }
     return true;
   }
 
   bool scriptDataDoubleEscapedLessThanSignState() {
     var data = stream.char();
     if (data == "/") {
       _addToken(new CharactersToken("/"));
-      temporaryBuffer = "";
+      _buffer.clear();
       state = scriptDataDoubleEscapeEndState;
     } else {
       stream.unget(data);
       state = scriptDataDoubleEscapedState;
     }
     return true;
   }
 
   bool scriptDataDoubleEscapeEndState() {
     var data = stream.char();
     if (isWhitespace(data) || data == "/" || data == ">") {
       _addToken(new CharactersToken(data));
-      if (temporaryBuffer.toLowerCase() == "script") {
+      if ('$_buffer'.toLowerCase() == "script") {
         state = scriptDataEscapedState;
       } else {
         state = scriptDataDoubleEscapedState;
       }
     } else if (isLetter(data)) {
       _addToken(new CharactersToken(data));
-      temporaryBuffer = '${temporaryBuffer}$data';
+      _buffer.write(data);
     } else {
       stream.unget(data);
       state = scriptDataDoubleEscapedState;
     }
     return true;
   }
 
   bool beforeAttributeNameState() {
     var data = stream.char();
     if (isWhitespace(data)) {
@@ skipping 23 matching lines @@
     return true;
   }
 
   bool attributeNameState() {
     var data = stream.char();
     bool leavingThisState = true;
     bool emitToken = false;
     if (data == "=") {
       state = beforeAttributeValueState;
     } else if (isLetter(data)) {
-      _attributeName = '$_attributeName$data'
-          '${stream.charsUntil(asciiLetters, true)}';
+      _attributeName.write(data);
+      _attributeName.write(stream.charsUntil(asciiLetters, true));
       leavingThisState = false;
     } else if (data == ">") {
       // XXX If we emit here the attributes are converted to a dict
       // without being checked and when the code below runs we error
       // because data is a dict not a list
       emitToken = true;
     } else if (isWhitespace(data)) {
       state = afterAttributeNameState;
     } else if (data == "/") {
       state = selfClosingStartTagState;
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      _attributeName = '${_attributeName}\uFFFD';
+      _attributeName.write('\uFFFD');
       leavingThisState = false;
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-attribute-name"));
       state = dataState;
     } else if ("'\"<".contains(data)) {
       _addToken(new ParseErrorToken("invalid-character-in-attribute-name"));
-      _attributeName = '$_attributeName$data';
+      _attributeName.write(data);
       leavingThisState = false;
     } else {
-      _attributeName = '$_attributeName$data';
+      _attributeName.write(data);
       leavingThisState = false;
     }
 
     if (leavingThisState) {
       _markAttributeNameEnd(-1);
 
       // Attributes are not dropped at this stage. That happens when the
       // start tag token is emitted so values can still be safely appended
       // to attributes, but we do want to report the parse error in time.
+      var attrName = _attributeName.toString();
       if (lowercaseAttrName) {
-        _attributeName = asciiUpper2Lower(_attributeName);
+        attrName = asciiUpper2Lower(attrName);
       }
+      _attributes.last.name = attrName;
       if (_attributeNames == null) _attributeNames = new Set();
-      if (_attributeNames.contains(_attributeName)) {
+      if (_attributeNames.contains(attrName)) {
         _addToken(new ParseErrorToken("duplicate-attribute"));
       }
-      _attributeNames.add(_attributeName);
+      _attributeNames.add(attrName);
 
       // XXX Fix for above XXX
       if (emitToken) {
         emitCurrentToken();
       }
     }
     return true;
   }
 
   bool afterAttributeNameState() {
@@ skipping 41 matching lines @@
     } else if (data == "'") {
       _markAttributeValueStart(0);
       state = attributeValueSingleQuotedState;
     } else if (data == ">") {
       _addToken(new ParseErrorToken(
           "expected-attribute-value-but-got-right-bracket"));
       emitCurrentToken();
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
       _markAttributeValueStart(-1);
-      _attributeValue = '${_attributeValue}\uFFFD';
+      _attributeValue.write('\uFFFD');
       state = attributeValueUnQuotedState;
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("expected-attribute-value-but-got-eof"));
       state = dataState;
     } else if ("=<`".contains(data)) {
       _addToken(new ParseErrorToken("equals-in-unquoted-attribute-value"));
       _markAttributeValueStart(-1);
-      _attributeValue = '$_attributeValue$data';
+      _attributeValue.write(data);
       state = attributeValueUnQuotedState;
     } else {
       _markAttributeValueStart(-1);
-      _attributeValue = '$_attributeValue$data';
+      _attributeValue.write(data);
       state = attributeValueUnQuotedState;
     }
     return true;
   }
 
   bool attributeValueDoubleQuotedState() {
     var data = stream.char();
     if (data == "\"") {
       _markAttributeValueEnd(-1);
       _markAttributeEnd(0);
       state = afterAttributeValueState;
     } else if (data == "&") {
       processEntityInAttribute('"');
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      _attributeValue = '${_attributeValue}\uFFFD';
+      _attributeValue.write('\uFFFD');
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-attribute-value-double-quote"));
       _markAttributeValueEnd(-1);
       state = dataState;
     } else {
-      _attributeValue = '$_attributeValue$data${stream.charsUntil("\"&")}';
+      _attributeValue.write(data);
+      _attributeValue.write(stream.charsUntil("\"&"));
     }
     return true;
   }
 
   bool attributeValueSingleQuotedState() {
     var data = stream.char();
     if (data == "'") {
       _markAttributeValueEnd(-1);
       _markAttributeEnd(0);
       state = afterAttributeValueState;
     } else if (data == "&") {
       processEntityInAttribute("'");
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      _attributeValue = '${_attributeValue}\uFFFD';
+      _attributeValue.write('\uFFFD');
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-attribute-value-single-quote"));
       _markAttributeValueEnd(-1);
       state = dataState;
     } else {
-      _attributeValue = '$_attributeValue$data${stream.charsUntil("\'&")}';
+      _attributeValue.write(data);
+      _attributeValue.write(stream.charsUntil("\'&"));
     }
     return true;
   }
 
   bool attributeValueUnQuotedState() {
     var data = stream.char();
     if (isWhitespace(data)) {
       _markAttributeValueEnd(-1);
       state = beforeAttributeNameState;
     } else if (data == "&") {
       processEntityInAttribute(">");
     } else if (data == ">") {
       _markAttributeValueEnd(-1);
       emitCurrentToken();
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-attribute-value-no-quotes"));
       _markAttributeValueEnd(-1);
       state = dataState;
     } else if ('"\'=<`'.contains(data)) {
       _addToken(new ParseErrorToken(
           "unexpected-character-in-unquoted-attribute-value"));
-      _attributeValue = '$_attributeValue$data';
+      _attributeValue.write(data);
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      _attributeValue = '${_attributeValue}\uFFFD';
+      _attributeValue.write('\uFFFD');
     } else {
-      _attributeValue = '$_attributeValue$data'
-          '${stream.charsUntil("&>\"\'=<`$spaceCharacters")}';
+      _attributeValue.write(data);
+      _attributeValue.write(stream.charsUntil("&>\"\'=<`$spaceCharacters"));
     }
     return true;
   }
 
   bool afterAttributeValueState() {
     var data = stream.char();
     if (isWhitespace(data)) {
       state = beforeAttributeNameState;
     } else if (data == ">") {
       emitCurrentToken();
@@ skipping 43 matching lines @@
     stream.char();
     state = dataState;
     return true;
   }
 
   bool markupDeclarationOpenState() {
     var charStack = [stream.char()];
     if (charStack.last == "-") {
       charStack.add(stream.char());
       if (charStack.last == "-") {
-        currentToken = new CommentToken("");
+        currentToken = new CommentToken();
         state = commentStartState;
         return true;
       }
     } else if (charStack.last == 'd' || charStack.last == 'D') {
       var matched = true;
       for (var expected in const ['oO', 'cC', 'tT', 'yY', 'pP', 'eE']) {
         var char = stream.char();
         charStack.add(char);
         if (char == EOF || !expected.contains(char)) {
           matched = false;
@@ skipping 32 matching lines @@
     state = bogusCommentState;
     return true;
   }
 
   bool commentStartState() {
     var data = stream.char();
     if (data == "-") {
       state = commentStartDashState;
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      currentStringToken.data = '${currentStringToken.data}\uFFFD';
+      currentStringToken.add('\uFFFD');
     } else if (data == ">") {
       _addToken(new ParseErrorToken("incorrect-comment"));
       _addToken(currentToken);
       state = dataState;
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-comment"));
       _addToken(currentToken);
       state = dataState;
     } else {
-      currentStringToken.data = '${currentStringToken.data}$data';
+      currentStringToken.add(data);
       state = commentState;
     }
     return true;
   }
 
   bool commentStartDashState() {
     var data = stream.char();
     if (data == "-") {
       state = commentEndState;
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      currentStringToken.data = '${currentStringToken.data}-\uFFFD';
+      currentStringToken.add('-\uFFFD');
     } else if (data == ">") {
       _addToken(new ParseErrorToken("incorrect-comment"));
       _addToken(currentToken);
       state = dataState;
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-comment"));
       _addToken(currentToken);
       state = dataState;
     } else {
-      currentStringToken.data = '${currentStringToken.data}-${data}';
+      currentStringToken.add('-').add(data);
       state = commentState;
     }
     return true;
   }
 
   bool commentState() {
     var data = stream.char();
     if (data == "-") {
       state = commentEndDashState;
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      currentStringToken.data = '${currentStringToken.data}\uFFFD';
+      currentStringToken.add('\uFFFD');
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-comment"));
       _addToken(currentToken);
       state = dataState;
     } else {
-      currentStringToken.data = '${currentStringToken.data}$data'
-          '${stream.charsUntil("-\u0000")}';
+      currentStringToken.add(data).add(stream.charsUntil("-\u0000"));

[Siggi Cherem (dart-lang), 2015/03/05 23:01:56]

maybe use cascades here and below? (instead of returning the stringtoken from
add?)

     }
     return true;
   }
 
   bool commentEndDashState() {
     var data = stream.char();
     if (data == "-") {
       state = commentEndState;
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      currentStringToken.data = "${currentStringToken.data}-\uFFFD";
+      currentStringToken.add('-\uFFFD');
       state = commentState;
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-comment-end-dash"));
       _addToken(currentToken);
       state = dataState;
     } else {
-      currentStringToken.data = "${currentStringToken.data}-${data}";
+      currentStringToken.add('-').add(data);
       state = commentState;
     }
     return true;
   }
 
   bool commentEndState() {
     var data = stream.char();
     if (data == ">") {
       _addToken(currentToken);
       state = dataState;
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      currentStringToken.data = '${currentStringToken.data}--\uFFFD';
+      currentStringToken.add('--\uFFFD');
       state = commentState;
     } else if (data == "!") {
       _addToken(
           new ParseErrorToken("unexpected-bang-after-double-dash-in-comment"));
       state = commentEndBangState;
     } else if (data == "-") {
       _addToken(
           new ParseErrorToken("unexpected-dash-after-double-dash-in-comment"));
-      currentStringToken.data = '${currentStringToken.data}$data';
+      currentStringToken.add(data);
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-comment-double-dash"));
       _addToken(currentToken);
       state = dataState;
     } else {
       // XXX
       _addToken(new ParseErrorToken("unexpected-char-in-comment"));
-      currentStringToken.data = "${currentStringToken.data}--${data}";
+      currentStringToken.add('--').add(data);
       state = commentState;
     }
     return true;
   }
 
   bool commentEndBangState() {
     var data = stream.char();
     if (data == ">") {
       _addToken(currentToken);
       state = dataState;
     } else if (data == "-") {
-      currentStringToken.data = '${currentStringToken.data}--!';
+      currentStringToken.add('--!');
       state = commentEndDashState;
     } else if (data == "\u0000") {
       _addToken(new ParseErrorToken("invalid-codepoint"));
-      currentStringToken.data = '${currentStringToken.data}--!\uFFFD';
+      currentStringToken.add('--!\uFFFD');
       state = commentState;
     } else if (data == EOF) {
       _addToken(new ParseErrorToken("eof-in-comment-end-bang-state"));
       _addToken(currentToken);
       state = dataState;
     } else {
-      currentStringToken.data = "${currentStringToken.data}--!${data}";
+      currentStringToken.add('--!').add(data);
       state = commentState;
     }
     return true;
   }
 
   bool doctypeState() {
     var data = stream.char();
     if (isWhitespace(data)) {
       state = beforeDoctypeNameState;
     } else if (data == EOF) {
@@ skipping 421 matching lines @@
       }
     }
 
     if (data.length > 0) {
       _addToken(new CharactersToken(data.join()));
     }
     state = dataState;
     return true;
   }
 }