OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 import "package:expect/expect.dart"; | 5 import "package:expect/expect.dart"; |
6 import 'package:compiler/src/scanner/utf8_bytes_scanner.dart'; | 6 import 'package:dart_scanner/dart_scanner.dart'; |
7 import 'package:compiler/src/tokens/precedence_constants.dart'; | 7 import 'package:dart_scanner/src/characters.dart'; |
8 import 'package:compiler/src/tokens/token.dart'; | 8 import 'package:dart_scanner/src/precedence.dart'; |
9 import 'package:compiler/src/util/characters.dart'; | |
10 import 'dart:typed_data'; | 9 import 'dart:typed_data'; |
11 | 10 |
| 11 |
12 Token scan(List<int> bytes) { | 12 Token scan(List<int> bytes) { |
13 List<int> zeroTerminated = new Uint8List(bytes.length + 1); | 13 List<int> zeroTerminated = new Uint8List(bytes.length + 1); |
14 zeroTerminated.setRange(0, bytes.length, bytes); | 14 zeroTerminated.setRange(0, bytes.length, bytes); |
15 zeroTerminated[bytes.length] = 0; | 15 zeroTerminated[bytes.length] = 0; |
16 return new Utf8BytesScanner.fromBytes(zeroTerminated).tokenize(); | 16 return new Utf8BytesScanner(zeroTerminated).tokenize(); |
17 } | 17 } |
18 | 18 |
19 Token scanUTF8(List<int> bytes) { | 19 Token scanUTF8(List<int> bytes) { |
20 int l = bytes.length; | 20 int l = bytes.length; |
21 List<int> stringLiteral = new Uint8List(l + 3); | 21 List<int> stringLiteral = new Uint8List(l + 3); |
22 stringLiteral[0] = 0x27; // single quote | 22 stringLiteral[0] = 0x27; // single quote |
23 stringLiteral[l + 1] = 0x27; // single quote | 23 stringLiteral[l + 1] = 0x27; // single quote |
24 // The bytes given to the scanner must be 0-terminated. | 24 // The bytes given to the scanner must be 0-terminated. |
25 stringLiteral[l + 2] = $EOF; | 25 stringLiteral[l + 2] = $EOF; |
26 for (int i = 0; i < l; i++) { | 26 for (int i = 0; i < l; i++) { |
27 stringLiteral[i + 1] = bytes[i]; | 27 stringLiteral[i + 1] = bytes[i]; |
28 } | 28 } |
29 return new Utf8BytesScanner.fromBytes(stringLiteral).tokenize(); | 29 return new Utf8BytesScanner(stringLiteral).tokenize(); |
30 } | 30 } |
31 | 31 |
32 bool isRunningOnJavaScript() => identical(1, 1.0); | 32 bool isRunningOnJavaScript() => identical(1, 1.0); |
33 | 33 |
34 main() { | 34 main() { |
35 // Google favorite: "Îñţérñåţîöñåļîžåţîờñ". | 35 // Google favorite: "Îñţérñåţîöñåļîžåţîờñ". |
36 Token token = scanUTF8([ | 36 Token token = scanUTF8([ |
37 0xc3, | 37 0xc3, |
38 0x8e, | 38 0x8e, |
39 0xc3, | 39 0xc3, |
(...skipping 162 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
202 0x20, | 202 0x20, |
203 0x57, | 203 0x57, |
204 0x6f, | 204 0x6f, |
205 0x72, | 205 0x72, |
206 0x6c, | 206 0x6c, |
207 0x64, | 207 0x64, |
208 0x21 | 208 0x21 |
209 ]); | 209 ]); |
210 Expect.equals(token.info, EOF_INFO); // Treated as a comment. | 210 Expect.equals(token.info, EOF_INFO); // Treated as a comment. |
211 } | 211 } |
OLD | NEW |