| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library dart2js.scanner.task; | 5 library dart2js.scanner.task; |
| 6 | 6 |
| 7 import '../common/tasks.dart' show CompilerTask; | 7 import '../common/tasks.dart' show CompilerTask; |
| 8 import '../compiler.dart' show Compiler; | 8 import '../compiler.dart' show Compiler; |
| 9 import '../elements/elements.dart' show CompilationUnitElement, LibraryElement; | 9 import '../elements/elements.dart' show CompilationUnitElement, LibraryElement; |
| 10 import '../script.dart' show Script; | 10 import '../script.dart' show Script; |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 69 * value ('\x00'). If [source] does not end with '0', the string is copied | 69 * value ('\x00'). If [source] does not end with '0', the string is copied |
| 70 * before scanning. | 70 * before scanning. |
| 71 */ | 71 */ |
| 72 Token tokenize(String source) { | 72 Token tokenize(String source) { |
| 73 return measure(() { | 73 return measure(() { |
| 74 return new StringScanner.fromString(source, includeComments: false) | 74 return new StringScanner.fromString(source, includeComments: false) |
| 75 .tokenize(); | 75 .tokenize(); |
| 76 }); | 76 }); |
| 77 } | 77 } |
| 78 | 78 |
| 79 Token tokenizeUnit(CompilationUnitElement unit) { |
| 80 return measureElement(unit, () { |
| 81 Token tokens = new Scanner( |
| 82 unit.script.file, includeComments: _preserveComments).tokenize(); |
| 83 if (_preserveComments) { |
| 84 tokens = processAndStripComments(tokens); |
| 85 } |
| 86 return tokens; |
| 87 }); |
| 88 } |
| 89 |
| 79 Token processAndStripComments(Token currentToken) { | 90 Token processAndStripComments(Token currentToken) { |
| 80 Token firstToken = currentToken; | 91 Token firstToken = currentToken; |
| 81 Token prevToken; | 92 Token prevToken; |
| 82 while (currentToken.kind != Tokens.EOF_TOKEN) { | 93 while (currentToken.kind != Tokens.EOF_TOKEN) { |
| 83 if (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) { | 94 if (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) { |
| 84 Token firstCommentToken = currentToken; | 95 Token firstCommentToken = currentToken; |
| 85 while (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) { | 96 while (identical(currentToken.kind, Tokens.COMMENT_TOKEN)) { |
| 86 currentToken = currentToken.next; | 97 currentToken = currentToken.next; |
| 87 } | 98 } |
| 88 _commentMap[currentToken] = firstCommentToken; | 99 _commentMap[currentToken] = firstCommentToken; |
| 89 if (prevToken == null) { | 100 if (prevToken == null) { |
| 90 firstToken = currentToken; | 101 firstToken = currentToken; |
| 91 } else { | 102 } else { |
| 92 prevToken.next = currentToken; | 103 prevToken.next = currentToken; |
| 93 } | 104 } |
| 94 } | 105 } |
| 95 prevToken = currentToken; | 106 prevToken = currentToken; |
| 96 currentToken = currentToken.next; | 107 currentToken = currentToken.next; |
| 97 } | 108 } |
| 98 return firstToken; | 109 return firstToken; |
| 99 } | 110 } |
| 100 } | 111 } |
| OLD | NEW |