| OLD | NEW |
| 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE.md file. | 3 // BSD-style license that can be found in the LICENSE.md file. |
| 4 | 4 |
| 5 library fasta.scanner; | 5 library fasta.scanner; |
| 6 | 6 |
| 7 import 'dart:convert' show UNICODE_REPLACEMENT_CHARACTER_RUNE, UTF8; | 7 import 'dart:convert' show UNICODE_REPLACEMENT_CHARACTER_RUNE, UTF8; |
| 8 | 8 |
| 9 import 'scanner/token.dart' show Token; | 9 import 'scanner/token.dart' show Token; |
| 10 | 10 |
| (...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 55 final Token tokens; | 55 final Token tokens; |
| 56 final List<int> lineStarts; | 56 final List<int> lineStarts; |
| 57 final bool hasErrors; | 57 final bool hasErrors; |
| 58 | 58 |
| 59 ScannerResult(this.tokens, this.lineStarts, this.hasErrors); | 59 ScannerResult(this.tokens, this.lineStarts, this.hasErrors); |
| 60 } | 60 } |
| 61 | 61 |
| 62 /// Scan/tokenize the given UTF8 [bytes]. | 62 /// Scan/tokenize the given UTF8 [bytes]. |
| 63 /// If [recover] is null, then the [defaultRecoveryStrategy] is used. | 63 /// If [recover] is null, then the [defaultRecoveryStrategy] is used. |
| 64 ScannerResult scan(List<int> bytes, | 64 ScannerResult scan(List<int> bytes, |
| 65 {bool includeComments: false, Recover recover}) { | 65 {bool includeComments: false, |
| 66 bool scanGenericMethodComments: false, |
| 67 Recover recover}) { |
| 66 if (bytes.last != 0) { | 68 if (bytes.last != 0) { |
| 67 throw new ArgumentError("[bytes]: the last byte must be null."); | 69 throw new ArgumentError("[bytes]: the last byte must be null."); |
| 68 } | 70 } |
| 69 Scanner scanner = | 71 Scanner scanner = new Utf8BytesScanner(bytes, |
| 70 new Utf8BytesScanner(bytes, includeComments: includeComments); | 72 includeComments: includeComments, |
| 73 scanGenericMethodComments: scanGenericMethodComments); |
| 71 return _tokenizeAndRecover(scanner, recover, bytes: bytes); | 74 return _tokenizeAndRecover(scanner, recover, bytes: bytes); |
| 72 } | 75 } |
| 73 | 76 |
| 74 /// Scan/tokenize the given [source]. | 77 /// Scan/tokenize the given [source]. |
| 75 /// If [recover] is null, then the [defaultRecoveryStrategy] is used. | 78 /// If [recover] is null, then the [defaultRecoveryStrategy] is used. |
| 76 ScannerResult scanString(String source, | 79 ScannerResult scanString(String source, |
| 77 {bool includeComments: false, Recover recover}) { | 80 {bool includeComments: false, |
| 81 bool scanGenericMethodComments: false, |
| 82 Recover recover}) { |
| 78 assert(source != null, 'source must not be null'); | 83 assert(source != null, 'source must not be null'); |
| 79 StringScanner scanner = | 84 StringScanner scanner = new StringScanner(source, |
| 80 new StringScanner(source, includeComments: includeComments); | 85 includeComments: includeComments, |
| 86 scanGenericMethodComments: scanGenericMethodComments); |
| 81 return _tokenizeAndRecover(scanner, recover, source: source); | 87 return _tokenizeAndRecover(scanner, recover, source: source); |
| 82 } | 88 } |
| 83 | 89 |
| 84 ScannerResult _tokenizeAndRecover(Scanner scanner, Recover recover, | 90 ScannerResult _tokenizeAndRecover(Scanner scanner, Recover recover, |
| 85 {List<int> bytes, String source}) { | 91 {List<int> bytes, String source}) { |
| 86 Token tokens = scanner.tokenize(); | 92 Token tokens = scanner.tokenize(); |
| 87 if (scanner.hasErrors) { | 93 if (scanner.hasErrors) { |
| 88 if (bytes == null) bytes = UTF8.encode(source); | 94 if (bytes == null) bytes = UTF8.encode(source); |
| 89 recover ??= defaultRecoveryStrategy; | 95 recover ??= defaultRecoveryStrategy; |
| 90 tokens = recover(bytes, tokens, scanner.lineStarts); | 96 tokens = recover(bytes, tokens, scanner.lineStarts); |
| 91 } | 97 } |
| 92 return new ScannerResult(tokens, scanner.lineStarts, scanner.hasErrors); | 98 return new ScannerResult(tokens, scanner.lineStarts, scanner.hasErrors); |
| 93 } | 99 } |
| OLD | NEW |