Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/scanner.dart

Index: lib/compiler/implementation/scanner/scanner.dart
diff --git a/lib/compiler/implementation/scanner/scanner.dart b/lib/compiler/implementation/scanner/scanner.dart
index ff2a93e88d1ff34ca556d1d9ddd14a2b2b98f4c0..b6338b65c1966ac5f7ba8d578010bc7c1e42bca1 100644
--- a/lib/compiler/implementation/scanner/scanner.dart
+++ b/lib/compiler/implementation/scanner/scanner.dart
@@ -14,12 +14,32 @@ class AbstractScanner<T extends SourceString> implements Scanner {
   abstract int nextByte();
   abstract int peek();
   abstract int select(int choice, PrecedenceInfo yes, PrecedenceInfo no);
+  /**
+   * Append a fixed token whose kind and content is determined by [info].
+   */
   abstract void appendPrecedenceToken(PrecedenceInfo info);
+  /**
+   * Append a token whose kind is determined by [info] and content is [value].
+   */
   abstract void appendStringToken(PrecedenceInfo info, String value);
+  /**
+   * Append a token whose kind is determined by [info] and content is defined by
+   * the SourceString [value].
+   */
   abstract void appendByteStringToken(PrecedenceInfo info, T value);
+  /**
+   * Append a keyword token whose kind is determined by [keyword].
+   */
   abstract void appendKeywordToken(Keyword keyword);
   abstract void appendWhiteSpace(int next);
   abstract void appendEofToken();
+  /**
+   * Creates an ascii SourceString whose content begins at the source byte
+   * offset [start] and ends at [offset] bytes from the current byte offset of
+   * the scanner. I.e. if the current byte offset is 10, [:asciiString(0,-1):]
+   * creates an ascii SourceString whose content is found at the [0,9[ byte
+   * interval of the source text.
+   */
   abstract T asciiString(int start, int offset);
   abstract T utf8String(int start, int offset);
   abstract Token firstToken();
@@ -32,6 +52,7 @@ class AbstractScanner<T extends SourceString> implements Scanner {
   abstract void appendGt(PrecedenceInfo info, String value);
   abstract void appendGtGt(PrecedenceInfo info, String value);
   abstract void appendGtGtGt(PrecedenceInfo info, String value);
+  abstract void appendComment();
   abstract void discardOpenLt();
 
   // TODO(ahe): Move this class to implementation.
@@ -521,6 +542,7 @@ class AbstractScanner<T extends SourceString> implements Scanner {
     while (true) {
       next = advance();
       if ($LF === next || $CR === next || $EOF === next) {
+        appendComment();
         return next;
       }
     }
@@ -537,8 +559,10 @@ class AbstractScanner<T extends SourceString> implements Scanner {
         next = advance();
         if ($SLASH === next) {
           --nesting;
+          next = advance();
           if (0 === nesting) {
-            return advance();
+            appendComment();
+            return next;
           } else {
             next = advance();
           }
@@ -664,7 +688,8 @@ class AbstractScanner<T extends SourceString> implements Scanner {
   }
 
   int tokenizeStringInterpolation(int start) {
-    beginToken();
+    appendByteStringToken(STRING_INFO, utf8String(start, -1));
+    beginToken(); // $ starts here
     int next = advance();
     if (next === $OPEN_CURLY_BRACKET) {
       return tokenizeInterpolatedExpression(next, start);
@@ -674,21 +699,24 @@ class AbstractScanner<T extends SourceString> implements Scanner {
   }
 
   int tokenizeInterpolatedExpression(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
     appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    beginToken(); // expression starts here
     next = advance();
     while (next !== $EOF && next !== $STX) {
       next = bigSwitch(next);
     }
     if (next === $EOF) return next;
-    return advance();
+    next = advance();
+    beginToken(); // string interpolation suffix starts here
+    return next;
   }
 
   int tokenizeInterpolatedIdentifier(int next, int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -2));
-    appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+    appendBeginGroup(STRING_INTERPOLATION_INFO, "\$");

[Johnni Winther, 2012/06/07 08:22:17]

There seems to be a problem with using "\$" and not "\${" as the value for the
generated BeginGroupToken since Token.stringValue depends on it.

+    beginToken(); // identifier starts here
     next = tokenizeKeywordOrIdentifier(next, false);
-    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "}", OPEN_CURLY_BRACKET_TOKEN);
+    beginToken(); // string interpolation suffix starts here
+    appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "", OPEN_CURLY_BRACKET_TOKEN);

[Johnni Winther, 2012/06/07 08:22:17]

There seems to be a problem with using "" and not "}" as the value for the
generated StringToken since Token.stringValue depends on it.

     return next;
   }