Add option for printing tokens produced by the scanner.

pkg/front_end/lib/src/fasta/scanner/scanner_main.dart

Index: pkg/front_end/lib/src/fasta/scanner/scanner_main.dart
diff --git a/pkg/front_end/lib/src/fasta/scanner/scanner_main.dart b/pkg/front_end/lib/src/fasta/scanner/scanner_main.dart
index f56aa98d06d63ba47a2719aedaa9f8090694596b..dfa2723be3c70d02695e357b652b54364af30f4c 100644
--- a/pkg/front_end/lib/src/fasta/scanner/scanner_main.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/scanner_main.dart
@@ -12,7 +12,13 @@ scanAll(Map<Uri, List<int>> files) {
   Stopwatch sw = new Stopwatch()..start();
   int byteCount = 0;
   files.forEach((Uri uri, List<int> bytes) {
-    scan(bytes);
+    var token = scan(bytes).tokens;
+    if (const bool.fromEnvironment("printTokens")) {
+      while (!token.isEof) {
+        print("${token.charOffset}: ${token}");
+        token = token.next;
+      }
+    }
     byteCount += bytes.length - 1;
   });
   sw.stop();