Implement shell-style comments and record shebangs as comments.

pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.abstract_scanner;
 
 import 'dart:collection' show
     ListMixin;
 
 import 'dart:typed_data' show
@@ skipping 19 matching lines @@
     BeginGroupToken,
     KeywordToken,
     SymbolToken,
     Token;
 
 import 'token_constants.dart';
 
 import 'characters.dart';
 
 abstract class AbstractScanner implements Scanner {
+  /// If true, include comments in the token stream.
   final bool includeComments;
 
   /**
    * The string offset for the next token that will be created.
    *
    * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values
    * are different. One string character can be encoded using multiple UTF-8
    * bytes.
    */
   int tokenStart = -1;
@@ skipping 351 matching lines @@
     if (next < 0x1f) {
       return unexpected(next);
     }
 
     next = currentAsUnicode(next);
 
     return unexpected(next);
   }
 
   int tokenizeTag(int next) {
-    // # or #!.*[\n\r]
-    if (scanOffset == 0) {
-      if (identical(peek(), $BANG)) {
-        int start = scanOffset + 1;
-        bool asciiOnly = true;
-        do {
-          next = advance();
-          if (next > 127) asciiOnly = false;
-        } while (!identical(next, $LF) &&
-            !identical(next, $CR) &&
-            !identical(next, $EOF));
-        if (!asciiOnly) handleUnicode(start);
-        return next;
-      }
+    // #!.*[\n\r]
+    if (scanOffset == 0 && identical(peek(), $BANG)) {
+      tokenizeSingleLineComment(next, scanOffset);
     }
     appendPrecedenceToken(HASH_INFO);
     return advance();
   }
 
   int tokenizeTilde(int next) {
     // ~ ~/ ~/=
     next = advance();
     if (identical(next, $SLASH)) {
       return select($EQ, TILDE_SLASH_EQ_INFO, TILDE_SLASH_INFO);
@@ skipping 817 matching lines @@
       switchToUint32(newLength);
     }
   }
 
   void switchToUint32(int newLength) {
     final newArray = new Uint32List(newLength);
     newArray.setRange(0, arrayLength, array);
     array = newArray;
   }
 }