| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 import "package:expect/expect.dart"; | 5 import "package:expect/expect.dart"; |
| 6 import 'package:front_end/src/fasta/scanner.dart'; | 6 import 'package:front_end/src/fasta/scanner.dart'; |
| 7 import 'package:front_end/src/fasta/scanner/characters.dart'; | 7 import 'package:front_end/src/fasta/scanner/characters.dart'; |
| 8 import 'package:front_end/src/fasta/scanner/precedence.dart'; | 8 import 'package:front_end/src/scanner/token.dart' show TokenType; |
| 9 import 'dart:typed_data'; | 9 import 'dart:typed_data'; |
| 10 | 10 |
| 11 Token scan(List<int> bytes) { | 11 Token scan(List<int> bytes) { |
| 12 List<int> zeroTerminated = new Uint8List(bytes.length + 1); | 12 List<int> zeroTerminated = new Uint8List(bytes.length + 1); |
| 13 zeroTerminated.setRange(0, bytes.length, bytes); | 13 zeroTerminated.setRange(0, bytes.length, bytes); |
| 14 zeroTerminated[bytes.length] = 0; | 14 zeroTerminated[bytes.length] = 0; |
| 15 return new Utf8BytesScanner(zeroTerminated).tokenize(); | 15 return new Utf8BytesScanner(zeroTerminated).tokenize(); |
| 16 } | 16 } |
| 17 | 17 |
| 18 Token scanUTF8(List<int> bytes) { | 18 Token scanUTF8(List<int> bytes) { |
| (...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 177 // UTF-8: F0 90 90 92 | 177 // UTF-8: F0 90 90 92 |
| 178 token = scanUTF8([0xf0, 0x90, 0x90, 0x92]); | 178 token = scanUTF8([0xf0, 0x90, 0x90, 0x92]); |
| 179 Expect.stringEquals("'𐐒'", token.lexeme); | 179 Expect.stringEquals("'𐐒'", token.lexeme); |
| 180 } else { | 180 } else { |
| 181 print('Skipping non-BMP character test'); | 181 print('Skipping non-BMP character test'); |
| 182 } | 182 } |
| 183 | 183 |
| 184 // Regression test for issue 1761. | 184 // Regression test for issue 1761. |
| 185 // "#!" | 185 // "#!" |
| 186 token = scan([0x23, 0x21]); | 186 token = scan([0x23, 0x21]); |
| 187 Expect.equals(token.info, SCRIPT_INFO); // Treated as a comment. | 187 Expect.equals(token.info, TokenType.SCRIPT_TAG); // Treated as a comment. |
| 188 | 188 |
| 189 // Regression test for issue 1761. | 189 // Regression test for issue 1761. |
| 190 // "#! Hello, World!" | 190 // "#! Hello, World!" |
| 191 token = scan([ | 191 token = scan([ |
| 192 0x23, | 192 0x23, |
| 193 0x21, | 193 0x21, |
| 194 0x20, | 194 0x20, |
| 195 0x48, | 195 0x48, |
| 196 0x65, | 196 0x65, |
| 197 0x6c, | 197 0x6c, |
| 198 0x6c, | 198 0x6c, |
| 199 0x6f, | 199 0x6f, |
| 200 0x2c, | 200 0x2c, |
| 201 0x20, | 201 0x20, |
| 202 0x57, | 202 0x57, |
| 203 0x6f, | 203 0x6f, |
| 204 0x72, | 204 0x72, |
| 205 0x6c, | 205 0x6c, |
| 206 0x64, | 206 0x64, |
| 207 0x21 | 207 0x21 |
| 208 ]); | 208 ]); |
| 209 Expect.equals(token.info, SCRIPT_INFO); // Treated as a comment. | 209 Expect.equals(token.info, TokenType.SCRIPT_TAG); // Treated as a comment. |
| 210 } | 210 } |
| OLD | NEW |