Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(227)

Side by Side Diff: tests/compiler/dart2js/scanner_test.dart

Issue 2738313002: rename fasta.Token.value --> lexeme (Closed)
Patch Set: merge Created 3 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tests/compiler/dart2js/scanner_offset_length_test.dart ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 import "package:expect/expect.dart"; 5 import "package:expect/expect.dart";
6 import 'package:front_end/src/fasta/scanner.dart'; 6 import 'package:front_end/src/fasta/scanner.dart';
7 import 'package:front_end/src/fasta/scanner/characters.dart'; 7 import 'package:front_end/src/fasta/scanner/characters.dart';
8 import 'package:front_end/src/fasta/scanner/precedence.dart'; 8 import 'package:front_end/src/fasta/scanner/precedence.dart';
9 import 'dart:typed_data'; 9 import 'dart:typed_data';
10 10
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after
67 0xc5, 67 0xc5,
68 0xa3, 68 0xa3,
69 0xc3, 69 0xc3,
70 0xae, 70 0xae,
71 0xe1, 71 0xe1,
72 0xbb, 72 0xbb,
73 0x9d, 73 0x9d,
74 0xc3, 74 0xc3,
75 0xb1 75 0xb1
76 ]); 76 ]);
77 Expect.stringEquals("'Îñţérñåţîöñåļîžåţîờñ'", token.value); 77 Expect.stringEquals("'Îñţérñåţîöñåļîžåţîờñ'", token.lexeme);
78 78
79 // Blueberry porridge in Danish: "blåbærgrød". 79 // Blueberry porridge in Danish: "blåbærgrød".
80 token = scanUTF8([ 80 token = scanUTF8([
81 0x62, 81 0x62,
82 0x6c, 82 0x6c,
83 0xc3, 83 0xc3,
84 0xa5, 84 0xa5,
85 0x62, 85 0x62,
86 0xc3, 86 0xc3,
87 0xa6, 87 0xa6,
88 0x72, 88 0x72,
89 0x67, 89 0x67,
90 0x72, 90 0x72,
91 0xc3, 91 0xc3,
92 0xb8, 92 0xb8,
93 0x64 93 0x64
94 ]); 94 ]);
95 Expect.stringEquals("'blåbærgrød'", token.value); 95 Expect.stringEquals("'blåbærgrød'", token.lexeme);
96 96
97 // "சிவா அணாமாைல", that is "Siva Annamalai" in Tamil. 97 // "சிவா அணாமாைல", that is "Siva Annamalai" in Tamil.
98 token = scanUTF8([ 98 token = scanUTF8([
99 0xe0, 99 0xe0,
100 0xae, 100 0xae,
101 0x9a, 101 0x9a,
102 0xe0, 102 0xe0,
103 0xae, 103 0xae,
104 0xbf, 104 0xbf,
105 0xe0, 105 0xe0,
(...skipping 18 matching lines...) Expand all
124 0xe0, 124 0xe0,
125 0xae, 125 0xae,
126 0xbe, 126 0xbe,
127 0xe0, 127 0xe0,
128 0xaf, 128 0xaf,
129 0x88, 129 0x88,
130 0xe0, 130 0xe0,
131 0xae, 131 0xae,
132 0xb2 132 0xb2
133 ]); 133 ]);
134 Expect.stringEquals("'சிவா அணாமாைல'", token.value); 134 Expect.stringEquals("'சிவா அணாமாைல'", token.lexeme);
135 135
136 // "िसवा अणामालै", that is "Siva Annamalai" in Devanagari. 136 // "िसवा अणामालै", that is "Siva Annamalai" in Devanagari.
137 token = scanUTF8([ 137 token = scanUTF8([
138 0xe0, 138 0xe0,
139 0xa4, 139 0xa4,
140 0xbf, 140 0xbf,
141 0xe0, 141 0xe0,
142 0xa4, 142 0xa4,
143 0xb8, 143 0xb8,
144 0xe0, 144 0xe0,
(...skipping 18 matching lines...) Expand all
163 0xe0, 163 0xe0,
164 0xa4, 164 0xa4,
165 0xbe, 165 0xbe,
166 0xe0, 166 0xe0,
167 0xa4, 167 0xa4,
168 0xb2, 168 0xb2,
169 0xe0, 169 0xe0,
170 0xa5, 170 0xa5,
171 0x88 171 0x88
172 ]); 172 ]);
173 Expect.stringEquals("'िसवा अणामालै'", token.value); 173 Expect.stringEquals("'िसवा अणामालै'", token.lexeme);
174 174
175 if (!isRunningOnJavaScript()) { 175 if (!isRunningOnJavaScript()) {
176 // DESERET CAPITAL LETTER BEE, unicode 0x10412(0xD801+0xDC12) 176 // DESERET CAPITAL LETTER BEE, unicode 0x10412(0xD801+0xDC12)
177 // UTF-8: F0 90 90 92 177 // UTF-8: F0 90 90 92
178 token = scanUTF8([0xf0, 0x90, 0x90, 0x92]); 178 token = scanUTF8([0xf0, 0x90, 0x90, 0x92]);
179 Expect.stringEquals("'𐐒'", token.value); 179 Expect.stringEquals("'𐐒'", token.lexeme);
180 } else { 180 } else {
181 print('Skipping non-BMP character test'); 181 print('Skipping non-BMP character test');
182 } 182 }
183 183
184 // Regression test for issue 1761. 184 // Regression test for issue 1761.
185 // "#!" 185 // "#!"
186 token = scan([0x23, 0x21]); 186 token = scan([0x23, 0x21]);
187 Expect.equals(token.info, SCRIPT_INFO); // Treated as a comment. 187 Expect.equals(token.info, SCRIPT_INFO); // Treated as a comment.
188 188
189 // Regression test for issue 1761. 189 // Regression test for issue 1761.
(...skipping 11 matching lines...) Expand all
201 0x20, 201 0x20,
202 0x57, 202 0x57,
203 0x6f, 203 0x6f,
204 0x72, 204 0x72,
205 0x6c, 205 0x6c,
206 0x64, 206 0x64,
207 0x21 207 0x21
208 ]); 208 ]);
209 Expect.equals(token.info, SCRIPT_INFO); // Treated as a comment. 209 Expect.equals(token.info, SCRIPT_INFO); // Treated as a comment.
210 } 210 }
OLDNEW
« no previous file with comments | « tests/compiler/dart2js/scanner_offset_length_test.dart ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698