Extract DocumentationCommentToken with field.

pkg/analyzer/lib/src/generated/scanner.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library engine.scanner;
 
 import 'dart:collection';
 
 import 'error.dart';
 import 'instrumentation.dart';
@@ skipping 221 matching lines @@
 /**
  * A `CommentToken` is a token representing a comment.
  */
 class CommentToken extends StringToken {
   /**
    * The [Token] that contains this comment.
    */
   Token parent;
 
   /**
+   * Initialize a newly created token to represent a token of the given [type]
+   * with the given [value] at the given [offset].
+   */
+  CommentToken(TokenType type, String value, int offset)
+      : super(type, value, offset);
+
+  @override
+  CommentToken copy() => new CommentToken(type, _value, offset);
+}
+
+/**
+ * A documentation comment token.
+ */
+class DocumentationCommentToken extends CommentToken {
+  /**
    * The references embedded within the documentation comment.
    * This list will be empty unless this is a documentation comment that has
    * references embedded within it.
    */
   final List<Token> references = <Token>[];
 
   /**
    * Initialize a newly created token to represent a token of the given [type]
    * with the given [value] at the given [offset].
    */
-  CommentToken(TokenType type, String value, int offset)
+  DocumentationCommentToken(TokenType type, String value, int offset)
       : super(type, value, offset);
 
   @override
-  CommentToken copy() => new CommentToken(type, _value, offset);
+  CommentToken copy() => new DocumentationCommentToken(type, _value, offset);
 }
 
 /**
  * The enumeration `Keyword` defines the keywords in the Dart programming
  * language.
  */
 class Keyword {
   static const Keyword ASSERT = const Keyword('ASSERT', "assert");
 
   static const Keyword BREAK = const Keyword('BREAK', "break");
@@ skipping 722 matching lines @@
     _groupingStack.add(token);
     _stackEnd++;
   }
 
   void _appendCommentToken(TokenType type, String value) {
     // Ignore comment tokens if client specified that it doesn't need them.
     if (!_preserveComments) {
       return;
     }
     // OK, remember comment tokens.
+    CommentToken token;
+    if (_isDocumentationComment(value)) {
+      token = new DocumentationCommentToken(type, value, _tokenStart);
+    } else {
+      token = new CommentToken(type, value, _tokenStart);
+    }
     if (_firstComment == null) {
-      _firstComment = new CommentToken(type, value, _tokenStart);
+      _firstComment = token;
       _lastComment = _firstComment;
     } else {
-      _lastComment =
-          _lastComment.setNext(new CommentToken(type, value, _tokenStart));
+      _lastComment = _lastComment.setNext(token);
     }
   }
 
   void _appendEndToken(TokenType type, TokenType beginType) {
     Token token;
     if (_firstComment == null) {
       token = new Token(type, _tokenStart);
     } else {
       token = new TokenWithComment(type, _tokenStart, _firstComment);
       _firstComment = null;
@@ skipping 758 matching lines @@
   int _tokenizeTilde(int next) {
     // ~ ~/ ~/=
     next = _reader.advance();
     if (next == 0x2F) {
       return _select(0x3D, TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH);
     } else {
       _appendTokenOfType(TokenType.TILDE);
       return next;
     }
   }
+
+  /**
+   * Checks if [value] is a single-line or multi-line comment.
+   */
+  static bool _isDocumentationComment(String value) {
+    return StringUtilities.startsWith3(value, 0, 0x2F, 0x2F, 0x2F) ||
+        StringUtilities.startsWith3(value, 0, 0x2F, 0x2A, 0x2A);
+  }
 }
 
 /**
  * The enumeration `ScannerErrorCode` defines the error codes used for errors
  * detected by the scanner.
  */
 class ScannerErrorCode extends ErrorCode {
   static const ScannerErrorCode ILLEGAL_CHARACTER =
       const ScannerErrorCode('ILLEGAL_CHARACTER', "Illegal character {0}");
 
@@ skipping 827 matching lines @@
   CommentToken get precedingComments => _precedingComment;
 
   void set precedingComments(CommentToken comment) {
     _precedingComment = comment;
     _setCommentParent(_precedingComment);
   }
 
   @override
   Token copy() => new TokenWithComment(type, offset, precedingComments);
 }