Begin refactoring compiler out of diet parser and scanner

pkg/compiler/lib/src/scanner/scanner_task.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library dart2js.scanner.task;
 
 import '../common/tasks.dart' show CompilerTask;
 import '../compiler.dart' show Compiler;
 import '../elements/elements.dart' show CompilationUnitElement, LibraryElement;
 import '../script.dart' show Script;
+import '../parser/diet_parser_task.dart' show DietParserTask;
 import '../tokens/token.dart' show Token;
+import '../tokens/token_constants.dart' as Tokens show COMMENT_TOKEN, EOF_TOKEN;
+import '../tokens/token_map.dart' show TokenMap;
 
 import 'scanner.dart' show Scanner;
 import 'string_scanner.dart' show StringScanner;
 
 class ScannerTask extends CompilerTask {
-  ScannerTask(Compiler compiler) : super(compiler);
+  final DietParserTask _dietParser;
+  final bool _preserveComments;
+  final TokenMap _commentMap;
+
+  ScannerTask(Compiler compiler, this._dietParser, this._commentMap,

[Siggi Cherem (dart-lang), 2016/04/08 19:37:56]

consider making _commentMap an optional argument too? (it's only needed when you
preserveComments)

[Harry Terkelsen, 2016/04/08 20:52:42]

Done.

+      {bool preserveComments: false})
+      : _preserveComments = preserveComments,

[Siggi Cherem (dart-lang), 2016/04/08 19:37:56]

... I wish the language just did this public/private name translation
automatically from this._preserveComments.

[Harry Terkelsen, 2016/04/08 20:52:42]

+1! This an annoying and common problem

+        super(compiler);
+
   String get name => 'Scanner';
 
   void scanLibrary(LibraryElement library) {
     CompilationUnitElement compilationUnit = library.entryCompilationUnit;
     String canonicalUri = library.canonicalUri.toString();
     String resolvedUri = compilationUnit.script.resourceUri.toString();
     if (canonicalUri == resolvedUri) {
       reporter.log("Scanning library $canonicalUri");
     } else {
       reporter.log("Scanning library $canonicalUri ($resolvedUri)");
     }
     scan(compilationUnit);
   }
 
   void scan(CompilationUnitElement compilationUnit) {
     measure(() {
       scanElements(compilationUnit);
     });
   }
 
   void scanElements(CompilationUnitElement compilationUnit) {
     Script script = compilationUnit.script;
-    Token tokens = new Scanner(script.file,
-            includeComments: compiler.options.preserveComments)
-        .tokenize();
-    if (compiler.options.preserveComments) {
-      tokens = compiler.processAndStripComments(tokens);
+    Token tokens =
+        new Scanner(script.file, includeComments: _preserveComments).tokenize();
+    if (_preserveComments) {
+      tokens = processAndStripComments(tokens);
     }
-    compiler.dietParser.dietParse(compilationUnit, tokens);
+    _dietParser.dietParse(compilationUnit, tokens);
   }
 
   /**
    * Returns the tokens for the [source].
    *
    * The [StringScanner] implementation works on strings that end with a '0'
    * value ('\x00'). If [source] does not end with '0', the string is copied
    * before scanning.
    */
   Token tokenize(String source) {
     return measure(() {
       return new StringScanner.fromString(source, includeComments: false)
           .tokenize();
     });
   }
+
+  Token processAndStripComments(Token currentToken) {

[Siggi Cherem (dart-lang), 2016/04/08 19:37:56]

I really like what you did here to contain comments within the scanner.
FYI - today this is only used for source_mirrors, but we should definitely keep
it around for the future. 

[Harry Terkelsen, 2016/04/08 20:52:41]

Acknowledged.

+    Token firstToken = currentToken;
+    Token prevToken;
+    while (currentToken.kind != Tokens.EOF_TOKEN) {
+      if (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) {
+        Token firstCommentToken = currentToken;
+        while (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) {
+          currentToken = currentToken.next;
+        }
+        _commentMap[currentToken] = firstCommentToken;
+        if (prevToken == null) {
+          firstToken = currentToken;
+        } else {
+          prevToken.next = currentToken;
+        }
+      }
+      prevToken = currentToken;
+      currentToken = currentToken.next;
+    }
+    return firstToken;
+  }
 }