Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 final int EOF_TOKEN = 0;
 
 final int KEYWORD_TOKEN = $k;
 final int IDENTIFIER_TOKEN = $a;
 final int DOUBLE_TOKEN = $d;
 final int INT_TOKEN = $i;
@@ skipping 53 matching lines @@
 final int PLUS_PLUS_TOKEN = STAR_EQ_TOKEN + 1;
 final int PLUS_EQ_TOKEN = PLUS_PLUS_TOKEN + 1;
 final int MINUS_MINUS_TOKEN = PLUS_EQ_TOKEN + 1;
 final int MINUS_EQ_TOKEN = MINUS_MINUS_TOKEN + 1;
 final int TILDE_SLASH_EQ_TOKEN = MINUS_EQ_TOKEN + 1;
 final int TILDE_SLASH_TOKEN = TILDE_SLASH_EQ_TOKEN + 1;
 final int PERCENT_EQ_TOKEN = TILDE_SLASH_TOKEN + 1;
 final int GT_GT_TOKEN = PERCENT_EQ_TOKEN + 1;
 final int CARET_EQ_TOKEN = GT_GT_TOKEN + 1;
 final int IS_TOKEN = CARET_EQ_TOKEN + 1;
+final int COMMENT_TOKEN = IS_TOKEN + 1;
 
 // TODO(ahe): Get rid of this.
 final int UNKNOWN_TOKEN = 1024;
 
 /**
  * A token that doubles as a linked list.
  */
 class Token {
   final PrecedenceInfo info;
+  /**
+   * The character offset of the start of this token within the source text.  
+   */ 
   final int charOffset;
+  /**
+   * The next token in the token stream.
+   */
   Token next;
 
   Token(PrecedenceInfo this.info, int this.charOffset);
 
   get value() => info.value;
   String get stringValue() => info.value.stringValue;
   int get kind() => info.kind;
   int get precedence() => info.precedence;
 
   String toString() => info.value.toString();
 
   String slowToString() => toString();
+  
+  /**
+   * The text parsed by this token.
+   */
+  String get text() => slowToString();
+  
+  /**
+   * The number of characters parsed by this token.
+   */
+  int get charLength() => text.length;
 }
 
 /**
  * A keyword token.
  */
 class KeywordToken extends Token {
   final Keyword value;
   String get stringValue() => value.syntax;
 
   KeywordToken(Keyword value, int charOffset)
@@ skipping 326 matching lines @@
 final PrecedenceInfo DOUBLE_INFO =
   const PrecedenceInfo(const SourceString('double'), 0, DOUBLE_TOKEN);
 
 final PrecedenceInfo STRING_INTERPOLATION_INFO =
   const PrecedenceInfo(const SourceString('\${'), 0,
                        STRING_INTERPOLATION_TOKEN);
 
 final PrecedenceInfo HEXADECIMAL_INFO =
   const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN);
 
+final PrecedenceInfo COMMENT_INFO =
+  const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN);
+
 // For reporting lexical errors.
 final PrecedenceInfo ERROR_INFO =
   const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN);