fasta.CommentToken implement analyzer.CommentToken

pkg/front_end/lib/src/fasta/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.token;
 
 import '../../scanner/token.dart' as analyzer;
 
 import 'keyword.dart' show Keyword;
 
 import 'precedence.dart' show BAD_INPUT_INFO, EOF_INFO, PrecedenceInfo;
 
 import 'token_constants.dart' show IDENTIFIER_TOKEN;
 
 import 'string_canonicalizer.dart';
 
 /**
  * A token that doubles as a linked list.
  */
-abstract class Token implements analyzer.Token {
+abstract class Token implements analyzer.TokenWithComment {
   /**
    * The character offset of the start of this token within the source text.
    */
   int charOffset;
 
   Token(this.charOffset);
 
   /**
    * The next token in the token stream.
    */
   Token next;
 
   /**
    * The previous token in the token stream.
    *
    * Deprecated :: This exists for compatibility with the Analyzer token stream
    * and will be removed at some future date.
    */
   @deprecated
   Token previousToken;
 
   /**
    * Return the first comment in the list of comments that precede this token,
    * or `null` if there are no comments preceding this token. Additional
    * comments can be reached by following the token stream using [next] until
    * `null` is returned.
    */
-  Token precedingComments;
+  CommentToken precedingCommentTokens;
+
+  @override
+  analyzer.CommentToken get precedingComments => precedingCommentTokens;
+
+  @override
+  void set precedingComments(analyzer.CommentToken token) {
+    precedingCommentTokens = token;
+  }
 
   /**
    * The precedence info for this token. [info] determines the kind and the
    * precedence level of this token.
    *
    * Defined as getter to save a field in the [KeywordToken] subclass.
    */
   PrecedenceInfo get info;
 
   /**
@@ skipping 98 matching lines @@
   analyzer.Token get previous => previousToken;
 
   @override
   set previous(analyzer.Token newToken) {
     previousToken = newToken as Token;
   }
 
   @override
   void applyDelta(int delta) {
     charOffset += delta;
-    Token token = precedingComments;
+    CommentToken token = precedingComments;
     while (token != null) {
       token.applyDelta(delta);
       token = token.next;
     }
   }
 
   @override
   analyzer.Token copy() {
     return copyWithoutComments()
-      ..precedingComments = copyComments(precedingComments) as Token;
+      ..precedingComments = copyComments(precedingComments);
   }
 
   @override
   analyzer.Token copyComments(analyzer.Token token) {
     if (token == null) {
       return null;
     }
     Token head = token.copy();
     Token tail = head;
     token = token.next;
@@ skipping 107 matching lines @@
 
   @override
   Object value() => keyword;
 }
 
 /**
  * A String-valued token. Represents identifiers, string literals,
  * number literals, comments, and error tokens, using the corresponding
  * precedence info.
  */
-class StringToken extends Token {
+class StringToken extends Token implements analyzer.StringToken {
   /**
    * The length threshold above which substring tokens are computed lazily.
    *
    * For string tokens that are substrings of the program source, the actual
    * substring extraction is performed lazily. This is beneficial because
    * not all scanned code is actually used. For unused parts, the substrings
    * are never computed and allocated.
    */
   static const int LAZY_THRESHOLD = 4;
 
@@ skipping 81 matching lines @@
     return canonicalizer.canonicalize(s, start, end, false);
   }
 
   static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) {
     return canonicalizer.canonicalize(data, start, end, asciiOnly);
   }
 
   @override
   Token copyWithoutComments() =>
       new StringToken._(info, valueOrLazySubstring, charOffset);
+
+  @override
+  String value() => lexeme;
+}
+
+class CommentToken extends StringToken implements analyzer.CommentToken {
+  /**
+   * Creates a lazy comment token. If [canonicalize] is true, the string
+   * is canonicalized before the token is created.
+   */
+  CommentToken.fromSubstring(
+      PrecedenceInfo info, String data, int start, int end, int charOffset,
+      {bool canonicalize: false})
+      : super.fromSubstring(info, data, start, end, charOffset,
+            canonicalize: canonicalize);
+
+  /**
+   * Creates a lazy string token. If [asciiOnly] is false, the byte array
+   * is passed through a UTF-8 decoder.
+   */
+  CommentToken.fromUtf8Bytes(PrecedenceInfo info, List<int> data, int start,
+      int end, bool asciiOnly, int charOffset)
+      : super.fromUtf8Bytes(info, data, start, end, asciiOnly, charOffset);
+
+  CommentToken._(PrecedenceInfo info, valueOrLazySubstring, int charOffset)
+      : super._(info, valueOrLazySubstring, charOffset);
+
+  @override
+  CommentToken copy() =>
+      new CommentToken._(info, valueOrLazySubstring, charOffset);
+
+  @override
+  analyzer.TokenWithComment get parent {
+    Token token = next;
+    while (token is CommentToken) {
+      token = token.next;
+    }
+    return token;
+  }
+
+  @override
+  void set parent(analyzer.TokenWithComment ignored) {
+    throw 'unsupported operation';

[Paul Berry, 2017/03/21 14:23:07]

Change this to `throw new StateError('unsupported operation');`

Rationale: Error objects contain a stackTrace member which is automatically set
by the VM at the time the error is first thrown.  If it is later caught and
re-thrown, it is not changed.  This makes it easier to track down the source of
errors when they occur.

[ahe, 2017/03/21 14:36:50]

I generally use strings for internal errors. We have API to preserve stack
traces, so it shouldn't be necessary to use subclasses of Error. This works both
for futures and rethrow.

However, in this case, I'd suggest using "internalError" from ../errors.dart, as
I'd like to keep the code free of "throw" in preference of using API like what's
in error.dart. internalError adds context to errors, and this has been helpful
for collecting crash reports.

[danrubel, 2017/03/22 17:10:48]

Fixed. https://codereview.chromium.org/2767083002/

+  }
+
+  @override
+  void remove() {
+    // TODO: implement remove

[Paul Berry, 2017/03/21 14:23:07]

Nit: change this to TODO(danrubel)

[danrubel, 2017/03/22 17:10:48]

Fixed. https://codereview.chromium.org/2767083002/

+    throw 'not implemented yet';

[Paul Berry, 2017/03/21 14:23:07]

Similarly, change to `throw new UnimplementedError();`

[danrubel, 2017/03/22 17:10:48]

Fixed. https://codereview.chromium.org/2767083002/

+  }
+}
+
+class DartDocToken extends CommentToken
+    implements analyzer.DocumentationCommentToken {
+  /**
+   * The references embedded within the documentation comment.
+   * This list will be empty unless this is a documentation comment that has
+   * references embedded within it.
+   */
+  final List<Token> references = <Token>[];
+
+  /**
+   * Creates a lazy comment token. If [canonicalize] is true, the string
+   * is canonicalized before the token is created.
+   */
+  DartDocToken.fromSubstring(
+      PrecedenceInfo info, String data, int start, int end, int charOffset,
+      {bool canonicalize: false})
+      : super.fromSubstring(info, data, start, end, charOffset,
+            canonicalize: canonicalize);
+
+  /**
+   * Creates a lazy string token. If [asciiOnly] is false, the byte array
+   * is passed through a UTF-8 decoder.
+   */
+  DartDocToken.fromUtf8Bytes(PrecedenceInfo info, List<int> data, int start,
+      int end, bool asciiOnly, int charOffset)
+      : super.fromUtf8Bytes(info, data, start, end, asciiOnly, charOffset);
+
+  DartDocToken._(PrecedenceInfo info, valueOrLazySubstring, int charOffset)
+      : super._(info, valueOrLazySubstring, charOffset);
+
+  @override
+  DartDocToken copy() {
+    DartDocToken copy =
+        new DartDocToken._(info, valueOrLazySubstring, charOffset);
+    references.forEach((ref) => copy.references.add(ref.copy()));
+    return copy;
+  }
 }
 
 /**
  * This class represents the necessary information to compute a substring
  * lazily. The substring can either originate from a string or from
  * a [:List<int>:] of UTF-8 bytes.
  */
 abstract class LazySubstring {
   /** The original data, either a string or a List<int> */
   get data;
@@ skipping 78 matching lines @@
       identical(value, "<=") ||
       identical(value, "<") ||
       identical(value, "&") ||
       identical(value, "^") ||
       identical(value, "|");
 }
 
 bool isTernaryOperator(String value) => identical(value, "[]=");
 
 bool isMinusOperator(String value) => identical(value, "-");