| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 import "package:expect/expect.dart"; | 5 import "package:expect/expect.dart"; |
| 6 import 'package:compiler/src/scanner/utf8_bytes_scanner.dart'; | 6 import 'package:dart_scanner/dart_scanner.dart'; |
| 7 import 'package:compiler/src/tokens/precedence_constants.dart'; | 7 import 'package:dart_scanner/src/characters.dart'; |
| 8 import 'package:compiler/src/tokens/token.dart'; | 8 import 'package:dart_scanner/src/precedence.dart'; |
| 9 import 'package:compiler/src/util/characters.dart'; | |
| 10 import 'dart:typed_data'; | 9 import 'dart:typed_data'; |
| 11 | 10 |
| 11 |
| 12 Token scan(List<int> bytes) { | 12 Token scan(List<int> bytes) { |
| 13 List<int> zeroTerminated = new Uint8List(bytes.length + 1); | 13 List<int> zeroTerminated = new Uint8List(bytes.length + 1); |
| 14 zeroTerminated.setRange(0, bytes.length, bytes); | 14 zeroTerminated.setRange(0, bytes.length, bytes); |
| 15 zeroTerminated[bytes.length] = 0; | 15 zeroTerminated[bytes.length] = 0; |
| 16 return new Utf8BytesScanner.fromBytes(zeroTerminated).tokenize(); | 16 return new Utf8BytesScanner(zeroTerminated).tokenize(); |
| 17 } | 17 } |
| 18 | 18 |
| 19 Token scanUTF8(List<int> bytes) { | 19 Token scanUTF8(List<int> bytes) { |
| 20 int l = bytes.length; | 20 int l = bytes.length; |
| 21 List<int> stringLiteral = new Uint8List(l + 3); | 21 List<int> stringLiteral = new Uint8List(l + 3); |
| 22 stringLiteral[0] = 0x27; // single quote | 22 stringLiteral[0] = 0x27; // single quote |
| 23 stringLiteral[l + 1] = 0x27; // single quote | 23 stringLiteral[l + 1] = 0x27; // single quote |
| 24 // The bytes given to the scanner must be 0-terminated. | 24 // The bytes given to the scanner must be 0-terminated. |
| 25 stringLiteral[l + 2] = $EOF; | 25 stringLiteral[l + 2] = $EOF; |
| 26 for (int i = 0; i < l; i++) { | 26 for (int i = 0; i < l; i++) { |
| 27 stringLiteral[i + 1] = bytes[i]; | 27 stringLiteral[i + 1] = bytes[i]; |
| 28 } | 28 } |
| 29 return new Utf8BytesScanner.fromBytes(stringLiteral).tokenize(); | 29 return new Utf8BytesScanner(stringLiteral).tokenize(); |
| 30 } | 30 } |
| 31 | 31 |
| 32 bool isRunningOnJavaScript() => identical(1, 1.0); | 32 bool isRunningOnJavaScript() => identical(1, 1.0); |
| 33 | 33 |
| 34 main() { | 34 main() { |
| 35 // Google favorite: "Îñţérñåţîöñåļîžåţîờñ". | 35 // Google favorite: "Îñţérñåţîöñåļîžåţîờñ". |
| 36 Token token = scanUTF8([ | 36 Token token = scanUTF8([ |
| 37 0xc3, | 37 0xc3, |
| 38 0x8e, | 38 0x8e, |
| 39 0xc3, | 39 0xc3, |
| (...skipping 162 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 202 0x20, | 202 0x20, |
| 203 0x57, | 203 0x57, |
| 204 0x6f, | 204 0x6f, |
| 205 0x72, | 205 0x72, |
| 206 0x6c, | 206 0x6c, |
| 207 0x64, | 207 0x64, |
| 208 0x21 | 208 0x21 |
| 209 ]); | 209 ]); |
| 210 Expect.equals(token.info, EOF_INFO); // Treated as a comment. | 210 Expect.equals(token.info, EOF_INFO); // Treated as a comment. |
| 211 } | 211 } |
| OLD | NEW |