Move scanner out of generated and clean up imports

pkg/analyzer/lib/dart/ast/token.dart

Index: pkg/analyzer/lib/dart/ast/token.dart
diff --git a/pkg/analyzer/lib/dart/ast/token.dart b/pkg/analyzer/lib/dart/ast/token.dart
index cbc8ae241d3872021e1d243f6f8d30bdd169b28c..6560a8e547e7c94d597789b8c52dfb59c68d6c44 100644
--- a/pkg/analyzer/lib/dart/ast/token.dart
+++ b/pkg/analyzer/lib/dart/ast/token.dart
@@ -367,10 +367,12 @@ abstract class Token {
 
 /**
  * The types of tokens that can be returned by the scanner.
+ *
+ * Clients may not extend, implement or mix-in this class.
  */
 class TokenType {
   /**
-   * The type of the token that marks the end of the input.
+   * The type of the token that marks the start or end of the input.
    */
   static const TokenType EOF = const _EndOfFileTokenType();
 
@@ -598,6 +600,10 @@ class TokenType {
    */
   final String lexeme;
 
+  /**
+   * Initialize a newly created token type to have the given [name],
+   * [_tokenClass] and [lexeme].
+   */
   const TokenType._(this.name,
       [this._tokenClass = TokenClass.NO_CLASS, this.lexeme = null]);