Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 interface Scanner {
   Token tokenize();
 }
 
 /**
  * Common base class for a Dart scanner.
  */
 class AbstractScanner<T extends SourceString> implements Scanner {
   abstract int advance();
   abstract int nextByte();
+
+  /**
+   * Returns the current character or byte depending on the underlying input
+   * kind. For example, [StringScanner] operates on [String] and thus returns
+   * characters whereas [ByteArrayScanner] operators on byte arrays and thus

[ahe, 2012/06/25 08:12:37]

... characters (Unicode codepoints represented as int) ...

[Johnni Winther, 2012/06/25 10:40:00]

Done.

+   * returns bytes.
+   */
   abstract int peek();
+
+  /**
+   * Appends a fixed token based on whether the current char is [choice] or not.
+   * If the current char is [choice] a fixed token whose kind and content
+   * is determined by [yes] is appended, otherwise a fixed token whose kind
+   * and content is determined by [no] is appended. */

[ahe, 2012/06/25 08:12:37]

*/ on its own line.

[Johnni Winther, 2012/06/25 10:40:00]

Done.

   abstract int select(int choice, PrecedenceInfo yes, PrecedenceInfo no);
+
+  /**
+   * Appends a fixed token whose kind and content is determined by [info].
+   */
   abstract void appendPrecedenceToken(PrecedenceInfo info);
+
+  /**
+   * Appends a token whose kind is determined by [info] and content is [value].
+   */
   abstract void appendStringToken(PrecedenceInfo info, String value);
+
+  /**
+   * Appends a token whose kind is determined by [info] and content is defined
+   * by the SourceString [value].
+   */
   abstract void appendByteStringToken(PrecedenceInfo info, T value);
+
+  /**
+   * Appends a keyword token whose kind is determined by [keyword].
+   */
   abstract void appendKeywordToken(Keyword keyword);
   abstract void appendWhiteSpace(int next);
   abstract void appendEofToken();
+
+  /**
+   * Creates an ascii SourceString whose content begins at the source byte

[ahe, 2012/06/25 08:12:37]

ascii -> ASCII

[Johnni Winther, 2012/06/25 10:40:00]

Done.

+   * offset [start] and ends at [offset] bytes from the current byte offset of
+   * the scanner. For example, if the current byte offset is 10,
+   * [:asciiString(0,-1):] creates an ascii SourceString whose content is found

[ahe, 2012/06/25 08:12:37]

Ditto.

[Johnni Winther, 2012/06/25 10:40:00]

Done.

+   * at the [0,9[ byte interval of the source text.
+   */
   abstract T asciiString(int start, int offset);
   abstract T utf8String(int start, int offset);
   abstract Token firstToken();
   abstract Token previousToken();
   abstract void beginToken();
   abstract void addToCharOffset(int offset);
   abstract int get charOffset();
   abstract int get byteOffset();
   abstract void appendBeginGroup(PrecedenceInfo info, String value);
   abstract int appendEndGroup(PrecedenceInfo info, String value, int openKind);
   abstract void appendGt(PrecedenceInfo info, String value);
   abstract void appendGtGt(PrecedenceInfo info, String value);
   abstract void appendGtGtGt(PrecedenceInfo info, String value);
+  abstract void appendComment();
 
   /**
    * We call this method to discard '<' from the "grouping" stack
    * (maintained by subclasses).
    *
    * [PartialParser.skipExpression] relies on the fact that we do not
    * create groups for stuff like:
    * [:a = b < c, d = e > f:].
    *
    * In other words, this method is called when the scanner recognizes
@@ skipping 490 matching lines @@
     } else {
       appendPrecedenceToken(SLASH_INFO);
       return next;
     }
   }
 
   int tokenizeSingleLineComment(int next) {
     while (true) {
       next = advance();
       if ($LF === next || $CR === next || $EOF === next) {
+        appendComment();
         return next;
       }
     }
   }
 
   int tokenizeMultiLineComment(int next) {
     int nesting = 1;
     next = advance();
     while (true) {
       if ($EOF === next) {
         // TODO(ahe): Report error.
         return next;
       } else if ($STAR === next) {
         next = advance();
         if ($SLASH === next) {
           --nesting;
           if (0 === nesting) {
-            return advance();
+            next = advance();
+            appendComment();
+            return next;
           } else {
             next = advance();
           }
         }
       } else if ($SLASH === next) {
         next = advance();
         if ($STAR === next) {
           next = advance();
           ++nesting;
         }
@@ skipping 105 matching lines @@
         throw new MalformedInputException("unterminated string literal",
                                           charOffset);
       }
       next = advance();
     }
     appendByteStringToken(STRING_INFO, utf8String(start, 0));
     return advance();
   }
 
   int tokenizeStringInterpolation(int start) {
-    beginToken();
+    appendByteStringToken(STRING_INFO, utf8String(start, -1));
+    beginToken(); // $ starts here.
     int next = advance();
     if (next === $OPEN_CURLY_BRACKET) {
       return tokenizeInterpolatedExpression(next, start);
     } else {
       return tokenizeInterpolatedIdentifier(next, start);
     }
   }
 
   int tokenizeInterpolatedExpression(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
     appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    beginToken(); // The expression starts here.
     next = advance();
     while (next !== $EOF && next !== $STX) {
       next = bigSwitch(next);
     }
     if (next === $EOF) return next;
-    return advance();
+    next = advance();
+    beginToken(); // The string interpolation suffix starts here.
+    return next;
   }
 
   int tokenizeInterpolatedIdentifier(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
-    appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    appendPrecedenceToken(STRING_INTERPOLATION_IDENTIFIER_INFO);
+    beginToken(); // The identifier starts here.
     next = tokenizeKeywordOrIdentifier(next, false);
-    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "}", OPEN_CURLY_BRACKET_TOKEN);
+    beginToken(); // The string interpolation suffix starts here.
     return next;
   }
 
   int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
     next = advance();
     while (next != $EOF) {
       if (next === quoteChar) {
         appendByteStringToken(STRING_INFO, utf8String(start, 0));
         return advance();
       } else if (next === $LF || next === $CR) {
@@ skipping 56 matching lines @@
                                       charOffset);
   }
 }
 
 class MalformedInputException {
   final String message;
   final position;
   MalformedInputException(this.message, this.position);
   toString() => message;
 }