Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/scanner.dart

Index: lib/compiler/implementation/scanner/scanner.dart
diff --git a/lib/compiler/implementation/scanner/scanner.dart b/lib/compiler/implementation/scanner/scanner.dart
index 24f439dedf9f5cf42a2d34522f5c29c3614a0834..685bab4fb76502b7a84931d37903b4c490f03253 100644
--- a/lib/compiler/implementation/scanner/scanner.dart
+++ b/lib/compiler/implementation/scanner/scanner.dart
@@ -12,14 +12,42 @@ interface Scanner {
 class AbstractScanner<T extends SourceString> implements Scanner {
   abstract int advance();
   abstract int nextByte();
+  /**

[ahe, 2012/06/22 08:42:49]

Please add a newline before all the documentation comments.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+   * Returns the current char.

[ahe, 2012/06/22 08:42:49]

No it doesn't :-)

I think the word you're looking for is "character", but what it really returns
is the next byte.

And that's not entirely true either. If this is StringScanner, it returns the
next character. If this is ByteArrayScanner, it returns the next byte.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+   */
   abstract int peek();
+  /**
+   * Appends a fixed token based on whether the current char is [choice] or not.
+   * If the current char is [choice] a fixed token whose kind and content
+   * is determined by [yes] is appended, otherwise a fixed token whose kind
+   * and content is determined by [no] is appended. */
   abstract int select(int choice, PrecedenceInfo yes, PrecedenceInfo no);
+  /**
+   * Appends a fixed token whose kind and content is determined by [info].
+   */
   abstract void appendPrecedenceToken(PrecedenceInfo info);
+  /**
+   * Appends a token whose kind is determined by [info] and content is [value].
+   */
   abstract void appendStringToken(PrecedenceInfo info, String value);
+  /**
+   * Appends a token whose kind is determined by [info] and content is defined by

[ahe, 2012/06/22 08:42:49]

Long line.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+   * the SourceString [value].
+   */
   abstract void appendByteStringToken(PrecedenceInfo info, T value);
+  /**
+   * Appends a keyword token whose kind is determined by [keyword].
+   */
   abstract void appendKeywordToken(Keyword keyword);
   abstract void appendWhiteSpace(int next);
   abstract void appendEofToken();
+  /**
+   * Creates an ascii SourceString whose content begins at the source byte
+   * offset [start] and ends at [offset] bytes from the current byte offset of
+   * the scanner. I.e. if the current byte offset is 10, [:asciiString(0,-1):]

[ahe, 2012/06/22 08:42:49]

A tech-writer once recommended that I avoided the use of "i.e." and "e.g.".
Apparently, those terms are confusing to many people. Instead, use "that is" and
"for example", respectively.

Also, I don't think you start a sentence with "i.e" or "that is". But I could be
wrong. However, I believe that all of these phrases should be followed by a
comma or colon.

In this case, I think you're giving an example, so it should be "for example".

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+   * creates an ascii SourceString whose content is found at the [0,9[ byte
+   * interval of the source text.
+   */
   abstract T asciiString(int start, int offset);
   abstract T utf8String(int start, int offset);
   abstract Token firstToken();
@@ -33,6 +61,7 @@ class AbstractScanner<T extends SourceString> implements Scanner {
   abstract void appendGt(PrecedenceInfo info, String value);
   abstract void appendGtGt(PrecedenceInfo info, String value);
   abstract void appendGtGtGt(PrecedenceInfo info, String value);
+  abstract void appendComment();
 
   /**
    * We call this method to discard '<' from the "grouping" stack
@@ -543,6 +572,7 @@ class AbstractScanner<T extends SourceString> implements Scanner {
     while (true) {
       next = advance();
       if ($LF === next || $CR === next || $EOF === next) {

[ahe, 2012/06/22 08:42:49]

Should we include DOS line endings?

[Johnni Winther, 2012/06/22 09:55:52]

The terminating character is not included in the single line comment but instead
tokenized as whitespace (by bigSwitch), which therefore handles DOS endings as
well.

+        appendComment();
         return next;
       }
     }
@@ -560,7 +590,9 @@ class AbstractScanner<T extends SourceString> implements Scanner {
         if ($SLASH === next) {
           --nesting;
           if (0 === nesting) {
-            return advance();
+            next = advance();
+            appendComment();
+            return next;
           } else {
             next = advance();
           }
@@ -686,7 +718,8 @@ class AbstractScanner<T extends SourceString> implements Scanner {
   }
 
   int tokenizeStringInterpolation(int start) {
-    beginToken();
+    appendByteStringToken(STRING_INFO, utf8String(start, -1));
+    beginToken(); // $ starts here

[ahe, 2012/06/22 08:42:49]

Not a proper sentence. A proper sentence starts with an uppercase letter and is
terminated by period.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

     int next = advance();
     if (next === $OPEN_CURLY_BRACKET) {
       return tokenizeInterpolatedExpression(next, start);
@@ -696,21 +729,24 @@ class AbstractScanner<T extends SourceString> implements Scanner {
   }
 
   int tokenizeInterpolatedExpression(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
     appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    beginToken(); // expression starts here

[ahe, 2012/06/22 08:42:49]

Not proper sentence.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

     next = advance();
     while (next !== $EOF && next !== $STX) {
       next = bigSwitch(next);
     }
     if (next === $EOF) return next;
-    return advance();
+    next = advance();
+    beginToken(); // string interpolation suffix starts here

[ahe, 2012/06/22 08:42:49]

Ditto.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+    return next;
   }
 
   int tokenizeInterpolatedIdentifier(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
-    appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    appendBeginGroup(STRING_INTERPOLATION_INFO, "\$");
+    beginToken(); // identifier starts here

[ahe, 2012/06/22 08:42:49]

Ditto.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

     next = tokenizeKeywordOrIdentifier(next, false);
-    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "}", OPEN_CURLY_BRACKET_TOKEN);
+    beginToken(); // string interpolation suffix starts here

[ahe, 2012/06/22 08:42:49]

Ditto.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

+    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "", OPEN_CURLY_BRACKET_TOKEN);

[ahe, 2012/06/22 08:42:49]

This is not a CLOSE_CURLE_BRACKET.

[Johnni Winther, 2012/06/22 10:23:10]

Done.

     return next;
   }