Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 interface Scanner {
   Token tokenize();
 }
 
 /**
  * Common base class for a Dart scanner.
  */
 class AbstractScanner<T extends SourceString> implements Scanner {
   abstract int advance();
   abstract int nextByte();
+  /**

[ahe, 2012/06/22 08:42:49]

Please add a newline before all the documentation comments.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+   * Returns the current char.

[ahe, 2012/06/22 08:42:49]

No it doesn't :-)

I think the word you're looking for is "character", but what it really returns
is the next byte.

And that's not entirely true either. If this is StringScanner, it returns the
next character. If this is ByteArrayScanner, it returns the next byte.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+   */
   abstract int peek();
+  /**
+   * Appends a fixed token based on whether the current char is [choice] or not.
+   * If the current char is [choice] a fixed token whose kind and content
+   * is determined by [yes] is appended, otherwise a fixed token whose kind
+   * and content is determined by [no] is appended. */
   abstract int select(int choice, PrecedenceInfo yes, PrecedenceInfo no);
+  /**
+   * Appends a fixed token whose kind and content is determined by [info].
+   */
   abstract void appendPrecedenceToken(PrecedenceInfo info);
+  /**
+   * Appends a token whose kind is determined by [info] and content is [value].
+   */
   abstract void appendStringToken(PrecedenceInfo info, String value);
+  /**
+   * Appends a token whose kind is determined by [info] and content is defined by

[ahe, 2012/06/22 08:42:49]

Long line.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+   * the SourceString [value].
+   */
   abstract void appendByteStringToken(PrecedenceInfo info, T value);
+  /**
+   * Appends a keyword token whose kind is determined by [keyword].
+   */
   abstract void appendKeywordToken(Keyword keyword);
   abstract void appendWhiteSpace(int next);
   abstract void appendEofToken();
+  /**
+   * Creates an ascii SourceString whose content begins at the source byte
+   * offset [start] and ends at [offset] bytes from the current byte offset of
+   * the scanner. I.e. if the current byte offset is 10, [:asciiString(0,-1):]

[ahe, 2012/06/22 08:42:49]

A tech-writer once recommended that I avoided the use of "i.e." and "e.g.".
Apparently, those terms are confusing to many people. Instead, use "that is" and
"for example", respectively.

Also, I don't think you start a sentence with "i.e" or "that is". But I could be
wrong. However, I believe that all of these phrases should be followed by a
comma or colon.

In this case, I think you're giving an example, so it should be "for example".

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+   * creates an ascii SourceString whose content is found at the [0,9[ byte
+   * interval of the source text.
+   */
   abstract T asciiString(int start, int offset);
   abstract T utf8String(int start, int offset);
   abstract Token firstToken();
   abstract Token previousToken();
   abstract void beginToken();
   abstract void addToCharOffset(int offset);
   abstract int get charOffset();
   abstract int get byteOffset();
   abstract void appendBeginGroup(PrecedenceInfo info, String value);
   abstract int appendEndGroup(PrecedenceInfo info, String value, int openKind);
   abstract void appendGt(PrecedenceInfo info, String value);
   abstract void appendGtGt(PrecedenceInfo info, String value);
   abstract void appendGtGtGt(PrecedenceInfo info, String value);
+  abstract void appendComment();
 
   /**
    * We call this method to discard '<' from the "grouping" stack
    * (maintained by subclasses).
    *
    * [PartialParser.skipExpression] relies on the fact that we do not
    * create groups for stuff like:
    * [:a = b < c, d = e > f:].
    *
    * In other words, this method is called when the scanner recognizes
@@ skipping 489 matching lines @@
       return advance();
     } else {
       appendPrecedenceToken(SLASH_INFO);
       return next;
     }
   }
 
   int tokenizeSingleLineComment(int next) {
     while (true) {
       next = advance();
       if ($LF === next || $CR === next || $EOF === next) {

[ahe, 2012/06/22 08:42:49]

Should we include DOS line endings?

[Johnni Winther, 2012/06/22 09:55:52]

The terminating character is not included in the single line comment but instead
tokenized as whitespace (by bigSwitch), which therefore handles DOS endings as
well.

+        appendComment();
         return next;
       }
     }
   }
 
   int tokenizeMultiLineComment(int next) {
     int nesting = 1;
     next = advance();
     while (true) {
       if ($EOF === next) {
         // TODO(ahe): Report error.
         return next;
       } else if ($STAR === next) {
         next = advance();
         if ($SLASH === next) {
           --nesting;
           if (0 === nesting) {
-            return advance();
+            next = advance();
+            appendComment();
+            return next;
           } else {
             next = advance();
           }
         }
       } else if ($SLASH === next) {
         next = advance();
         if ($STAR === next) {
           next = advance();
           ++nesting;
         }
@@ skipping 105 matching lines @@
         throw new MalformedInputException("unterminated string literal",
                                           charOffset);
       }
       next = advance();
     }
     appendByteStringToken(STRING_INFO, utf8String(start, 0));
     return advance();
   }
 
   int tokenizeStringInterpolation(int start) {
-    beginToken();
+    appendByteStringToken(STRING_INFO, utf8String(start, -1));
+    beginToken(); // $ starts here

[ahe, 2012/06/22 08:42:49]

Not a proper sentence. A proper sentence starts with an uppercase letter and is
terminated by period.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

     int next = advance();
     if (next === $OPEN_CURLY_BRACKET) {
       return tokenizeInterpolatedExpression(next, start);
     } else {
       return tokenizeInterpolatedIdentifier(next, start);
     }
   }
 
   int tokenizeInterpolatedExpression(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
     appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    beginToken(); // expression starts here

[ahe, 2012/06/22 08:42:49]

Not proper sentence.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

     next = advance();
     while (next !== $EOF && next !== $STX) {
       next = bigSwitch(next);
     }
     if (next === $EOF) return next;
-    return advance();
+    next = advance();
+    beginToken(); // string interpolation suffix starts here

[ahe, 2012/06/22 08:42:49]

Ditto.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+    return next;
   }
 
   int tokenizeInterpolatedIdentifier(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
-    appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    appendBeginGroup(STRING_INTERPOLATION_INFO, "\$");
+    beginToken(); // identifier starts here

[ahe, 2012/06/22 08:42:49]

Ditto.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

     next = tokenizeKeywordOrIdentifier(next, false);
-    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "}", OPEN_CURLY_BRACKET_TOKEN);
+    beginToken(); // string interpolation suffix starts here

[ahe, 2012/06/22 08:42:49]

Ditto.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "", OPEN_CURLY_BRACKET_TOKEN);

[ahe, 2012/06/22 08:42:49]

This is not a CLOSE_CURLE_BRACKET.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

     return next;
   }
 
   int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
     next = advance();
     while (next != $EOF) {
       if (next === quoteChar) {
         appendByteStringToken(STRING_INFO, utf8String(start, 0));
         return advance();
       } else if (next === $LF || next === $CR) {
@@ skipping 56 matching lines @@
                                       charOffset);
   }
 }
 
 class MalformedInputException {
   final String message;
   final position;
   MalformedInputException(this.message, this.position);
   toString() => message;
 }