Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 interface Scanner {
   Token tokenize();
 }
 
 /**
  * Common base class for a Dart scanner.
  */
 class AbstractScanner<T extends SourceString> implements Scanner {
   abstract int advance();
   abstract int nextByte();
   abstract int peek();
   abstract int select(int choice, PrecedenceInfo yes, PrecedenceInfo no);
+  /**
+   * Append a fixed token whose kind and content is determined by [info].
+   */
   abstract void appendPrecedenceToken(PrecedenceInfo info);
+  /**
+   * Append a token whose kind is determined by [info] and content is [value].
+   */
   abstract void appendStringToken(PrecedenceInfo info, String value);
+  /**
+   * Append a token whose kind is determined by [info] and content is defined by
+   * the SourceString [value].
+   */
   abstract void appendByteStringToken(PrecedenceInfo info, T value);
+  /**
+   * Append a keyword token whose kind is determined by [keyword].
+   */
   abstract void appendKeywordToken(Keyword keyword);
   abstract void appendWhiteSpace(int next);
   abstract void appendEofToken();
+  /**
+   * Creates an ascii SourceString whose content begins at the source byte
+   * offset [start] and ends at [offset] bytes from the current byte offset of
+   * the scanner. I.e. if the current byte offset is 10, [:asciiString(0,-1):]
+   * creates an ascii SourceString whose content is found at the [0,9[ byte
+   * interval of the source text.
+   */
   abstract T asciiString(int start, int offset);
   abstract T utf8String(int start, int offset);
   abstract Token firstToken();
   abstract void beginToken();
   abstract void addToCharOffset(int offset);
   abstract int get charOffset();
   abstract int get byteOffset();
   abstract void appendBeginGroup(PrecedenceInfo info, String value);
   abstract int appendEndGroup(PrecedenceInfo info, String value, int openKind);
   abstract void appendGt(PrecedenceInfo info, String value);
   abstract void appendGtGt(PrecedenceInfo info, String value);
   abstract void appendGtGtGt(PrecedenceInfo info, String value);
+  abstract void appendComment();
   abstract void discardOpenLt();
 
   // TODO(ahe): Move this class to implementation.
 
   Token tokenize() {
     int next = advance();
     while (next !== $EOF) {
       next = bigSwitch(next);
     }
     appendEofToken();
@@ skipping 469 matching lines @@
     } else {
       appendPrecedenceToken(SLASH_INFO);
       return next;
     }
   }
 
   int tokenizeSingleLineComment(int next) {
     while (true) {
       next = advance();
       if ($LF === next || $CR === next || $EOF === next) {
+        appendComment();
         return next;
       }
     }
   }
 
   int tokenizeMultiLineComment(int next) {
     int nesting = 1;
     next = advance();
     while (true) {
       if ($EOF === next) {
         // TODO(ahe): Report error.
         return next;
       } else if ($STAR === next) {
         next = advance();
         if ($SLASH === next) {
           --nesting;
+          next = advance();
           if (0 === nesting) {
-            return advance();
+            appendComment();
+            return next;
           } else {
             next = advance();
           }
         }
       } else if ($SLASH === next) {
         next = advance();
         if ($STAR === next) {
           next = advance();
           ++nesting;
         }
@@ skipping 105 matching lines @@
         throw new MalformedInputException("unterminated string literal",
                                           charOffset);
       }
       next = advance();
     }
     appendByteStringToken(STRING_INFO, utf8String(start, 0));
     return advance();
   }
 
   int tokenizeStringInterpolation(int start) {
-    beginToken();
+    appendByteStringToken(STRING_INFO, utf8String(start, -1));
+    beginToken(); // $ starts here
     int next = advance();
     if (next === $OPEN_CURLY_BRACKET) {
       return tokenizeInterpolatedExpression(next, start);
     } else {
       return tokenizeInterpolatedIdentifier(next, start);
     }
   }
 
   int tokenizeInterpolatedExpression(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
     appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    beginToken(); // expression starts here
     next = advance();
     while (next !== $EOF && next !== $STX) {
       next = bigSwitch(next);
     }
     if (next === $EOF) return next;
-    return advance();
+    next = advance();
+    beginToken(); // string interpolation suffix starts here
+    return next;
   }
 
   int tokenizeInterpolatedIdentifier(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
-    appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    appendBeginGroup(STRING_INTERPOLATION_INFO, "\$");

[Johnni Winther, 2012/06/07 08:22:17]

There seems to be a problem with using "\$" and not "\${" as the value for the
generated BeginGroupToken since Token.stringValue depends on it.

+    beginToken(); // identifier starts here
     next = tokenizeKeywordOrIdentifier(next, false);
-    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "}", OPEN_CURLY_BRACKET_TOKEN);
+    beginToken(); // string interpolation suffix starts here
+    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "", OPEN_CURLY_BRACKET_TOKEN);

[Johnni Winther, 2012/06/07 08:22:17]

There seems to be a problem with using "" and not "}" as the value for the
generated StringToken since Token.stringValue depends on it.

     return next;
   }
 
   int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
     next = advance();
     while (next != $EOF) {
       if (next === quoteChar) {
         appendByteStringToken(STRING_INFO, utf8String(start, 0));
         return advance();
       } else if (next === $LF || next === $CR) {
@@ skipping 56 matching lines @@
                                       charOffset);
   }
 }
 
 class MalformedInputException {
   final String message;
   final position;
   MalformedInputException(this.message, this.position);
   toString() => message;
 }