Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/token.dart

Index: lib/compiler/implementation/scanner/token.dart
diff --git a/lib/compiler/implementation/scanner/token.dart b/lib/compiler/implementation/scanner/token.dart
index ddd921b074745f311777c71ec04380a0e8b2f4a4..0b6c2bdd92527fa8bfa04aa182764a1752225a95 100644
--- a/lib/compiler/implementation/scanner/token.dart
+++ b/lib/compiler/implementation/scanner/token.dart
@@ -71,6 +71,7 @@ final int PERCENT_EQ_TOKEN = TILDE_SLASH_TOKEN + 1;
 final int GT_GT_TOKEN = PERCENT_EQ_TOKEN + 1;
 final int CARET_EQ_TOKEN = GT_GT_TOKEN + 1;
 final int IS_TOKEN = CARET_EQ_TOKEN + 1;
+final int COMMENT_TOKEN = IS_TOKEN + 1;
 
 // TODO(ahe): Get rid of this.
 final int UNKNOWN_TOKEN = 1024;
@@ -80,7 +81,13 @@ final int UNKNOWN_TOKEN = 1024;
  */
 class Token {
   final PrecedenceInfo info;
+  /**
+   * The character offset of the start of this token within the source text.  
+   */ 
   final int charOffset;
+  /**
+   * The next token in the token stream.
+   */
   Token next;
 
   Token(PrecedenceInfo this.info, int this.charOffset);
@@ -93,6 +100,16 @@ class Token {
   String toString() => info.value.toString();
 
   String slowToString() => toString();
+  
+  /**
+   * The text parsed by this token.
+   */
+  String get text() => slowToString();
+  
+  /**
+   * The number of characters parsed by this token.
+   */
+  int get charLength() => text.length;
 }
 
 /**
@@ -439,6 +456,9 @@ final PrecedenceInfo STRING_INTERPOLATION_INFO =
 final PrecedenceInfo HEXADECIMAL_INFO =
   const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN);
 
+final PrecedenceInfo COMMENT_INFO =
+  const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN);
+
 // For reporting lexical errors.
 final PrecedenceInfo ERROR_INFO =
   const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN);