Begin refactoring compiler out of diet parser and scanner

pkg/compiler/lib/src/compiler.dart

Index: pkg/compiler/lib/src/compiler.dart
diff --git a/pkg/compiler/lib/src/compiler.dart b/pkg/compiler/lib/src/compiler.dart
index 3b9632c26b969e8b4e982b1c54db9a81c23887aa..f2d9e8d2bec53c7325063f0128df4905269ff1bc 100644
--- a/pkg/compiler/lib/src/compiler.dart
+++ b/pkg/compiler/lib/src/compiler.dart
@@ -47,6 +47,7 @@ import 'enqueue.dart'
         ResolutionEnqueuer,
         QueueFilter;
 import 'environment.dart';
+import 'id_generator.dart';
 import 'io/source_information.dart' show SourceInformation;
 import 'js_backend/backend_helpers.dart' as js_backend show BackendHelpers;
 import 'js_backend/js_backend.dart' as js_backend show JavaScriptBackend;
@@ -76,7 +77,6 @@ import 'script.dart' show Script;
 import 'ssa/nodes.dart' show HInstruction;
 import 'tracer.dart' show Tracer;
 import 'tokens/token.dart' show StringToken, Token, TokenPair;
-import 'tokens/token_constants.dart' as Tokens show COMMENT_TOKEN, EOF_TOKEN;
 import 'tokens/token_map.dart' show TokenMap;
 import 'tree/tree.dart' show Node, TypeAnnotation;
 import 'typechecker.dart' show TypeCheckerTask;
@@ -89,9 +89,9 @@ import 'universe/world_impact.dart' show ImpactStrategy, WorldImpact;
 import 'util/util.dart' show Link, Setlet;
 import 'world.dart' show World;
 
-abstract class Compiler implements LibraryLoaderListener {
+abstract class Compiler implements LibraryLoaderListener, IdGenerator {

[Siggi Cherem (dart-lang), 2016/04/08 19:37:56]

what are your thoughts on splitting it up in two objects:

class Compiler {
  IdGenerator idGenerator = new IdGenerator();

  ...
}

?

Technically the real goal is to split it up where it is used (e.g.
ClosureClassElement should take an idGenerator eventually), so it's not
necessary to split it here too. But I wonder if it will make it easier in the
future... 

[Harry Terkelsen, 2016/04/08 20:52:41]

Done. But I had to keep Compiler as an implementor of IdGenerator because some
places (like ClosureClassElement) call getNextId directly from compiler

   final Stopwatch totalCompileTime = new Stopwatch();
-  int nextFreeClassId = 0;
+  int _nextFreeId = 0;
   World world;
   Types types;
   _CompilerCoreTypes _coreTypes;
@@ -340,7 +340,7 @@ abstract class Compiler implements LibraryLoaderListener {
     }
 
     tasks = [
-      dietParser = new DietParserTask(this, parsing.parserOptions),
+      dietParser = new DietParserTask(this, parsing.parserOptions, this),
       scanner = createScannerTask(),
       serialization = new SerializationTask(this),
       libraryLoader = new LibraryLoaderTask(
@@ -371,7 +371,9 @@ abstract class Compiler implements LibraryLoaderListener {
   /// Creates the scanner task.
   ///
   /// Override this to mock the scanner for testing.
-  ScannerTask createScannerTask() => new ScannerTask(this);
+  ScannerTask createScannerTask() =>
+      new ScannerTask(this, dietParser, commentMap,
+          preserveComments: options.preserveComments);
 
   /// Creates the resolver task.
   ///
@@ -390,7 +392,7 @@ abstract class Compiler implements LibraryLoaderListener {
   bool get disableTypeInference =>
       options.disableTypeInference || compilationFailed;
 
-  int getNextFreeClassId() => nextFreeClassId++;
+  int getNextFreeId() => _nextFreeId++;
 
   void unimplemented(Spannable spannable, String methodName) {
     reporter.internalError(spannable, "$methodName not implemented.");
@@ -1149,28 +1151,6 @@ abstract class Compiler implements LibraryLoaderListener {
 
   bool get isMockCompilation => false;
 
-  Token processAndStripComments(Token currentToken) {
-    Token firstToken = currentToken;
-    Token prevToken;
-    while (currentToken.kind != Tokens.EOF_TOKEN) {
-      if (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) {
-        Token firstCommentToken = currentToken;
-        while (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) {
-          currentToken = currentToken.next;
-        }
-        commentMap[currentToken] = firstCommentToken;
-        if (prevToken == null) {
-          firstToken = currentToken;
-        } else {
-          prevToken.next = currentToken;
-        }
-      }
-      prevToken = currentToken;
-      currentToken = currentToken.next;
-    }
-    return firstToken;
-  }
-
   void reportUnusedCode() {
     void checkLive(member) {
       if (member.isMalformed) return;