Implement shell-style comments and record shebangs as comments.

pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart

Index: pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
diff --git a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
index f4de827c7fa5399a8d90e8207653e0b2503b8d77..3532a30d6ecc9ede2f4ba0c85d21049416994cf8 100644
--- a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
@@ -30,8 +30,13 @@ import 'token_constants.dart';
 import 'characters.dart';
 
 abstract class AbstractScanner implements Scanner {
+  /// If true, include comments in the token stream.
   final bool includeComments;
 
+  /// If true, `#` starts a single line comment. This is the comment style
+  /// known from, for example, Unix shell and Python.
+  final bool enableShellStyleComments;

[ahe, 2017/02/21 16:57:10]

I should probably remove this again. After thinking a bit more about it, I don't
think it makes sense to attempt parsing .packages files or YAML files with this
scanner.

[Paul Berry, 2017/02/21 16:59:34]

SGTM.  I admit I was surprised you were heading down that path in the first
place :)

[ahe, 2017/02/21 17:37:34]

Yeah, it feels like a stupid idea now :-)

Sometimes I need things to be very concrete before I really understand it.

+
   /**
    * The string offset for the next token that will be created.
    *
@@ -57,7 +62,7 @@ abstract class AbstractScanner implements Scanner {
 
   final List<int> lineStarts = <int>[0];
 
-  AbstractScanner(this.includeComments) {
+  AbstractScanner(this.includeComments, this.enableShellStyleComments) {
     this.tail = this.tokens;
   }
 
@@ -401,19 +406,10 @@ abstract class AbstractScanner implements Scanner {
 
   int tokenizeTag(int next) {
     // # or #!.*[\n\r]
-    if (scanOffset == 0) {
-      if (identical(peek(), $BANG)) {
-        int start = scanOffset + 1;
-        bool asciiOnly = true;
-        do {
-          next = advance();
-          if (next > 127) asciiOnly = false;
-        } while (!identical(next, $LF) &&
-            !identical(next, $CR) &&
-            !identical(next, $EOF));
-        if (!asciiOnly) handleUnicode(start);
-        return next;
-      }
+    if (scanOffset == 0 && identical(peek(), $BANG)) {
+      tokenizeSingleLineComment(next, scanOffset + 1);
+    } else if (enableShellStyleComments) {
+      tokenizeSingleLineComment(next, scanOffset);
     }
     appendPrecedenceToken(HASH_INFO);
     return advance();