Implement shell-style comments and record shebangs as comments.

pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.abstract_scanner;
 
 import '../scanner.dart' show
     ErrorToken,
     Scanner,
     buildUnexpectedCharacterToken;
@@ skipping 12 matching lines @@
     BeginGroupToken,
     KeywordToken,
     SymbolToken,
     Token;
 
 import 'token_constants.dart';
 
 import 'characters.dart';
 
 abstract class AbstractScanner implements Scanner {
+  /// If true, include comments in the token stream.
   final bool includeComments;
 
+  /// If true, `#` starts a single line comment. This is the comment style
+  /// known from, for example, Unix shell and Python.
+  final bool enableShellStyleComments;

[ahe, 2017/02/21 16:57:10]

I should probably remove this again. After thinking a bit more about it, I don't
think it makes sense to attempt parsing .packages files or YAML files with this
scanner.

[Paul Berry, 2017/02/21 16:59:34]

SGTM.  I admit I was surprised you were heading down that path in the first
place :)

[ahe, 2017/02/21 17:37:34]

Yeah, it feels like a stupid idea now :-)

Sometimes I need things to be very concrete before I really understand it.

+
   /**
    * The string offset for the next token that will be created.
    *
    * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values
    * are different. One string character can be encoded using multiple UTF-8
    * bytes.
    */
   int tokenStart = -1;
 
   /**
    * A pointer to the token stream created by this scanner. The first token
    * is a special token and not part of the source file. This is an
    * implementation detail to avoids special cases in the scanner. This token
    * is not exposed to clients of the scanner, which are expected to invoke
    * [firstToken] to access the token stream.
    */
   final Token tokens = new SymbolToken(EOF_INFO, -1);
 
   /**
    * A pointer to the last scanned token.
    */
   Token tail;
 
   final List<int> lineStarts = <int>[0];
 
-  AbstractScanner(this.includeComments) {
+  AbstractScanner(this.includeComments, this.enableShellStyleComments) {
     this.tail = this.tokens;
   }
 
   /**
    * Advances and returns the next character.
    *
    * If the next character is non-ASCII, then the returned value depends on the
    * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while
    * the [StringScanner] returns a UTF-16 code unit.
    *
@@ skipping 323 matching lines @@
       return unexpected(next);
     }
 
     next = currentAsUnicode(next);
 
     return unexpected(next);
   }
 
   int tokenizeTag(int next) {
     // # or #!.*[\n\r]
-    if (scanOffset == 0) {
-      if (identical(peek(), $BANG)) {
-        int start = scanOffset + 1;
-        bool asciiOnly = true;
-        do {
-          next = advance();
-          if (next > 127) asciiOnly = false;
-        } while (!identical(next, $LF) &&
-            !identical(next, $CR) &&
-            !identical(next, $EOF));
-        if (!asciiOnly) handleUnicode(start);
-        return next;
-      }
+    if (scanOffset == 0 && identical(peek(), $BANG)) {
+      tokenizeSingleLineComment(next, scanOffset + 1);
+    } else if (enableShellStyleComments) {
+      tokenizeSingleLineComment(next, scanOffset);
     }
     appendPrecedenceToken(HASH_INFO);
     return advance();
   }
 
   int tokenizeTilde(int next) {
     // ~ ~/ ~/=
     next = advance();
     if (identical(next, $SLASH)) {
       return select($EQ, TILDE_SLASH_EQ_INFO, TILDE_SLASH_INFO);
@@ skipping 731 matching lines @@
 
 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) {
   return const {
     '(': CLOSE_PAREN_INFO,
     '[': CLOSE_SQUARE_BRACKET_INFO,
     '{': CLOSE_CURLY_BRACKET_INFO,
     '<': GT_INFO,
     r'${': CLOSE_CURLY_BRACKET_INFO,
   }[begin.value];
 }