Begin refactoring compiler out of diet parser and scanner

pkg/compiler/lib/src/scanner/scanner_task.dart

Index: pkg/compiler/lib/src/scanner/scanner_task.dart
diff --git a/pkg/compiler/lib/src/scanner/scanner_task.dart b/pkg/compiler/lib/src/scanner/scanner_task.dart
index b40dbf7acb94c443d2fde6529d0be6df3b492765..c6342b3cd2bbad5b637c97b6a91a835b3c0c5b44 100644
--- a/pkg/compiler/lib/src/scanner/scanner_task.dart
+++ b/pkg/compiler/lib/src/scanner/scanner_task.dart
@@ -8,13 +8,24 @@ import '../common/tasks.dart' show CompilerTask;
 import '../compiler.dart' show Compiler;
 import '../elements/elements.dart' show CompilationUnitElement, LibraryElement;
 import '../script.dart' show Script;
+import '../parser/diet_parser_task.dart' show DietParserTask;
 import '../tokens/token.dart' show Token;
+import '../tokens/token_constants.dart' as Tokens show COMMENT_TOKEN, EOF_TOKEN;
+import '../tokens/token_map.dart' show TokenMap;
 
 import 'scanner.dart' show Scanner;
 import 'string_scanner.dart' show StringScanner;
 
 class ScannerTask extends CompilerTask {
-  ScannerTask(Compiler compiler) : super(compiler);
+  final DietParserTask _dietParser;
+  final bool _preserveComments;
+  final TokenMap _commentMap;
+
+  ScannerTask(Compiler compiler, this._dietParser, this._commentMap,

[Siggi Cherem (dart-lang), 2016/04/08 19:37:56]

consider making _commentMap an optional argument too? (it's only needed when you
preserveComments)

[Harry Terkelsen, 2016/04/08 20:52:42]

Done.

+      {bool preserveComments: false})
+      : _preserveComments = preserveComments,

[Siggi Cherem (dart-lang), 2016/04/08 19:37:56]

... I wish the language just did this public/private name translation
automatically from this._preserveComments.

[Harry Terkelsen, 2016/04/08 20:52:42]

+1! This an annoying and common problem

+        super(compiler);
+
   String get name => 'Scanner';
 
   void scanLibrary(LibraryElement library) {
@@ -37,13 +48,12 @@ class ScannerTask extends CompilerTask {
 
   void scanElements(CompilationUnitElement compilationUnit) {
     Script script = compilationUnit.script;
-    Token tokens = new Scanner(script.file,
-            includeComments: compiler.options.preserveComments)
-        .tokenize();
-    if (compiler.options.preserveComments) {
-      tokens = compiler.processAndStripComments(tokens);
+    Token tokens =
+        new Scanner(script.file, includeComments: _preserveComments).tokenize();
+    if (_preserveComments) {
+      tokens = processAndStripComments(tokens);
     }
-    compiler.dietParser.dietParse(compilationUnit, tokens);
+    _dietParser.dietParse(compilationUnit, tokens);
   }
 
   /**
@@ -59,4 +69,26 @@ class ScannerTask extends CompilerTask {
           .tokenize();
     });
   }
+
+  Token processAndStripComments(Token currentToken) {

[Siggi Cherem (dart-lang), 2016/04/08 19:37:56]

I really like what you did here to contain comments within the scanner.
FYI - today this is only used for source_mirrors, but we should definitely keep
it around for the future. 

[Harry Terkelsen, 2016/04/08 20:52:41]

Acknowledged.

+    Token firstToken = currentToken;
+    Token prevToken;
+    while (currentToken.kind != Tokens.EOF_TOKEN) {
+      if (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) {
+        Token firstCommentToken = currentToken;
+        while (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) {
+          currentToken = currentToken.next;
+        }
+        _commentMap[currentToken] = firstCommentToken;
+        if (prevToken == null) {
+          firstToken = currentToken;
+        } else {
+          prevToken.next = currentToken;
+        }
+      }
+      prevToken = currentToken;
+      currentToken = currentToken.next;
+    }
+    return firstToken;
+  }
 }