flatten token hierarchy

pkg/front_end/lib/src/scanner/scanner.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:charcode/ascii.dart';
 import 'package:front_end/src/scanner/errors.dart';
 import 'package:front_end/src/scanner/reader.dart';
 import 'package:front_end/src/scanner/string_utilities.dart';
 import 'package:front_end/src/scanner/token.dart';
 
@@ skipping 472 matching lines @@
     }
     _appendEofToken();
     return firstToken;
   }
 
   void _appendBeginToken(TokenType type) {
     BeginToken token;
     if (_firstComment == null) {
       token = new BeginToken(type, _tokenStart);
     } else {
-      token = new BeginTokenWithComment(type, _tokenStart, _firstComment);
+      token = new BeginToken(type, _tokenStart, _firstComment);
       _firstComment = null;
       _lastComment = null;
     }
     _tail = _tail.setNext(token);
     _groupingStack.add(token);
     _stackEnd++;
   }
 
   void _appendCommentToken(TokenType type, String value) {
     CommentToken token = null;
@@ skipping 17 matching lines @@
     } else {
       _lastComment = _lastComment.setNext(token);
     }
   }
 
   void _appendEndToken(TokenType type, TokenType beginType) {
     Token token;
     if (_firstComment == null) {
       token = new Token(type, _tokenStart);
     } else {
-      token = new TokenWithComment(type, _tokenStart, _firstComment);
+      token = new Token(type, _tokenStart, _firstComment);
       _firstComment = null;
       _lastComment = null;
     }
     _tail = _tail.setNext(token);
     if (_stackEnd >= 0) {
       BeginToken begin = _groupingStack[_stackEnd];
       if (begin.type == beginType) {
         begin.endToken = token;
         _groupingStack.removeAt(_stackEnd--);
       }
@@ skipping 16 matching lines @@
     if (_stackEnd >= 0) {
       _hasUnmatchedGroups = true;
       // TODO(brianwilkerson) Fix the ungrouped tokens?
     }
   }
 
   void _appendKeywordToken(Keyword keyword) {
     if (_firstComment == null) {
       _tail = _tail.setNext(new KeywordToken(keyword, _tokenStart));
     } else {
-      _tail = _tail.setNext(
-          new KeywordTokenWithComment(keyword, _tokenStart, _firstComment));
+      _tail =
+          _tail.setNext(new KeywordToken(keyword, _tokenStart, _firstComment));
       _firstComment = null;
       _lastComment = null;
     }
   }
 
   void _appendStringToken(TokenType type, String value) {
     if (_firstComment == null) {
       _tail = _tail.setNext(new StringToken(type, value, _tokenStart));
     } else {
-      _tail = _tail.setNext(
-          new StringTokenWithComment(type, value, _tokenStart, _firstComment));
+      _tail = _tail
+          .setNext(new StringToken(type, value, _tokenStart, _firstComment));
       _firstComment = null;
       _lastComment = null;
     }
   }
 
   void _appendStringTokenWithOffset(TokenType type, String value, int offset) {
     if (_firstComment == null) {
       _tail = _tail.setNext(new StringToken(type, value, _tokenStart + offset));
     } else {
-      _tail = _tail.setNext(new StringTokenWithComment(
-          type, value, _tokenStart + offset, _firstComment));
+      _tail = _tail.setNext(
+          new StringToken(type, value, _tokenStart + offset, _firstComment));
       _firstComment = null;
       _lastComment = null;
     }
   }
 
   void _appendTokenOfType(TokenType type) {
     if (_firstComment == null) {
       _tail = _tail.setNext(new Token(type, _tokenStart));
     } else {
-      _tail =
-          _tail.setNext(new TokenWithComment(type, _tokenStart, _firstComment));
+      _tail = _tail.setNext(new Token(type, _tokenStart, _firstComment));
       _firstComment = null;
       _lastComment = null;
     }
   }
 
   void _appendTokenOfTypeWithOffset(TokenType type, int offset) {
     if (_firstComment == null) {
       _tail = _tail.setNext(new Token(type, offset));
     } else {
-      _tail = _tail.setNext(new TokenWithComment(type, offset, _firstComment));
+      _tail = _tail.setNext(new Token(type, offset, _firstComment));
       _firstComment = null;
       _lastComment = null;
     }
   }
 
   void _beginToken() {
     _tokenStart = _reader.offset;
   }
 
   /**
@@ skipping 712 matching lines @@
   }
 
   /**
    * Checks if [value] is a single-line or multi-line comment.
    */
   static bool _isDocumentationComment(String value) {
     return StringUtilities.startsWith3(value, 0, $slash, $slash, $slash) ||
         StringUtilities.startsWith3(value, 0, $slash, $asterisk, $asterisk);
   }
 }