Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/token.dart

Index: lib/compiler/implementation/scanner/token.dart
diff --git a/lib/compiler/implementation/scanner/token.dart b/lib/compiler/implementation/scanner/token.dart
index ddd921b074745f311777c71ec04380a0e8b2f4a4..b0855ca4802aadd5f46561559f5a33348a71269d 100644
--- a/lib/compiler/implementation/scanner/token.dart
+++ b/lib/compiler/implementation/scanner/token.dart
@@ -71,6 +71,9 @@ final int PERCENT_EQ_TOKEN = TILDE_SLASH_TOKEN + 1;
 final int GT_GT_TOKEN = PERCENT_EQ_TOKEN + 1;
 final int CARET_EQ_TOKEN = GT_GT_TOKEN + 1;
 final int IS_TOKEN = CARET_EQ_TOKEN + 1;
+final int COMMENT_TOKEN = IS_TOKEN + 1;
+final int GT_GT_GT_TOKEN = COMMENT_TOKEN + 1;
+final int STRING_INTERPOLATION_IDENTIFIER_TOKEN = COMMENT_TOKEN + 1;
 
 // TODO(ahe): Get rid of this.
 final int UNKNOWN_TOKEN = 1024;
@@ -79,20 +82,54 @@ final int UNKNOWN_TOKEN = 1024;
  * A token that doubles as a linked list.
  */
 class Token {
+  /**
+   * The precedence info for this token. [info] determines the kind and the
+   * precedence level of this token.
+   */
   final PrecedenceInfo info;
+
+  /**
+   * The character offset of the start of this token within the source text.
+   */
   final int charOffset;
+
+  /**
+   * The next token in the token stream.
+   */
   Token next;
 
   Token(PrecedenceInfo this.info, int this.charOffset);
 
   get value() => info.value;
+
+  /**
+   *  [stringValue] should not be used outside the scanner/parser.

[ahe, 2012/06/25 08:12:37]

That is not entirely correct. stringValue should only be used for testing
keywords and symbols. An identifier, string, number, etc will return null. But
as a rule of thumb, use slowToString() instead.

[Johnni Winther, 2012/06/25 10:40:00]

Done.

+   */
   String get stringValue() => info.value.stringValue;
+
+  /**
+   * The kind enum of this token as determined by its [info].
+   */
   int get kind() => info.kind;
+
+  /**
+   * The precedence level for this token.
+   */
   int get precedence() => info.precedence;
 
   String toString() => info.value.toString();
 
   String slowToString() => toString();
+
+  /**
+   * The text parsed by this token.
+   */
+  String get text() => slowToString();

[ahe, 2012/06/25 08:12:37]

I'd prefer to use "slowToString()". As it has "slow" in its name.

[Johnni Winther, 2012/06/25 10:40:00]

Done.

+
+  /**
+   * The number of characters parsed by this token.
+   */
+  int get charLength() => text.length;

[ahe, 2012/06/25 08:12:37]

This is slow.

[Johnni Winther, 2012/06/25 10:40:00]

Done.

 }
 
 /**
@@ -367,7 +404,7 @@ final PrecedenceInfo LT_INFO =
 
 // Shift operators.
 final PrecedenceInfo GT_GT_GT_INFO =
-  const PrecedenceInfo(const SourceString('>>>'), 11, GT_GT_TOKEN);
+  const PrecedenceInfo(const SourceString('>>>'), 11, GT_GT_GT_TOKEN);
 final PrecedenceInfo GT_GT_INFO =
   const PrecedenceInfo(const SourceString('>>'), 11, GT_GT_TOKEN);
 final PrecedenceInfo LT_LT_INFO =
@@ -436,9 +473,16 @@ final PrecedenceInfo STRING_INTERPOLATION_INFO =
   const PrecedenceInfo(const SourceString('\${'), 0,
                        STRING_INTERPOLATION_TOKEN);
 
+final PrecedenceInfo STRING_INTERPOLATION_IDENTIFIER_INFO =
+  const PrecedenceInfo(const SourceString('\$'), 0,
+                       STRING_INTERPOLATION_IDENTIFIER_TOKEN);
+
 final PrecedenceInfo HEXADECIMAL_INFO =
   const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN);
 
+final PrecedenceInfo COMMENT_INFO =
+  const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN);
+
 // For reporting lexical errors.
 final PrecedenceInfo ERROR_INFO =
   const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN);