Move scanner out of generated and clean up imports

pkg/analyzer/lib/dart/ast/token.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Defines the tokens that are produced by the scanner, used by the parser, and
  * referenced from the [AST structure](ast.dart).
  */
 library analyzer.dart.ast.token;
 
@@ skipping 349 matching lines @@
         first = token;
         offset = token.offset;
       }
     }
     return first;
   }
 }
 
 /**
  * The types of tokens that can be returned by the scanner.
+ *
+ * Clients may not extend, implement or mix-in this class.
  */
 class TokenType {
   /**
-   * The type of the token that marks the end of the input.
+   * The type of the token that marks the start or end of the input.
    */
   static const TokenType EOF = const _EndOfFileTokenType();
 
   static const TokenType DOUBLE = const TokenType._('DOUBLE');
 
   static const TokenType HEXADECIMAL = const TokenType._('HEXADECIMAL');
 
   static const TokenType IDENTIFIER = const TokenType._('IDENTIFIER');
 
   static const TokenType INT = const TokenType._('INT');
@@ skipping 207 matching lines @@
    * The name of the token type.
    */
   final String name;
 
   /**
    * The lexeme that defines this type of token, or `null` if there is more than
    * one possible lexeme for this type of token.
    */
   final String lexeme;
 
+  /**
+   * Initialize a newly created token type to have the given [name],
+   * [_tokenClass] and [lexeme].
+   */
   const TokenType._(this.name,
       [this._tokenClass = TokenClass.NO_CLASS, this.lexeme = null]);
 
   /**
    * Return `true` if this type of token represents an additive operator.
    */
   bool get isAdditiveOperator => _tokenClass == TokenClass.ADDITIVE_OPERATOR;
 
   /**
    * Return `true` if this type of token represents an assignment operator.
@@ skipping 112 matching lines @@
  */
 class _EndOfFileTokenType extends TokenType {
   /**
    * Initialize a newly created token.
    */
   const _EndOfFileTokenType() : super._('EOF', TokenClass.NO_CLASS, '');
 
   @override
   String toString() => '-eof-';
 }