| Index: pkg/compiler/lib/src/scanner/scanner_task.dart
|
| diff --git a/pkg/compiler/lib/src/scanner/scanner_task.dart b/pkg/compiler/lib/src/scanner/scanner_task.dart
|
| index 3d694a3d7110d4762f94b5764be74e85461665d2..b3c158bc6681ba62b08c87d396dc21830cf31899 100644
|
| --- a/pkg/compiler/lib/src/scanner/scanner_task.dart
|
| +++ b/pkg/compiler/lib/src/scanner/scanner_task.dart
|
| @@ -76,6 +76,17 @@ class ScannerTask extends CompilerTask {
|
| });
|
| }
|
|
|
| + Token tokenizeUnit(CompilationUnitElement unit) {
|
| + return measureElement(unit, () {
|
| + Token tokens = new Scanner(
|
| + unit.script.file, includeComments: _preserveComments).tokenize();
|
| + if (_preserveComments) {
|
| + tokens = processAndStripComments(tokens);
|
| + }
|
| + return tokens;
|
| + });
|
| + }
|
| +
|
| Token processAndStripComments(Token currentToken) {
|
| Token firstToken = currentToken;
|
| Token prevToken;
|
|
|