Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(232)

Side by Side Diff: tests/lib/convert/json_utf8_chunk_test.dart

Issue 649113005: Make JSON parsing work as a chunked conversion sink. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Also add an UTF-8 base JSON parser, without intermediate string representations. Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
(Empty)
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file.
4
5 library test;
6
7 import "package:expect/expect.dart";
8 import "dart:convert";
9
10 bool badFormat(e) => e is FormatException;
11
12 main() {
13 testNumbers();
14 testStrings();
15 testKeywords();
16 testAll();
17 testMalformed();
18 }
19
20 // Create an UTF-8 sink from a chunked JSON decoder, then let [action]
21 // put data into it, and check that what comes out is equal to [expect].
22 void jsonTest(name, expect, action(sink), [bool allowMalformed = false]) {
Søren Gjesse 2014/10/24 11:12:24 name -> testName or nameOfTest
Lasse Reichstein Nielsen 2014/10/27 12:42:34 Done.
23 jsonParse(name, (value) {
24 Expect.equals(expect, value, "$name$value");
Søren Gjesse 2014/10/24 11:12:25 Separator before $value.
Lasse Reichstein Nielsen 2014/10/27 12:42:34 Done.
25 }, action, allowMalformed);
26 }
27
28 void jsonParse(name, check, action, [bool allowMalformed = false]) {
29 var sink = new ChunkedConversionSink.withCallback((values) {
30 var value = values[0];
31 check(value);
32 });
33 var decoderSink = JSON.decoder.startChunkedConversion(sink)
34 .asUtf8Sink(allowMalformed);
35 try {
36 action(decoderSink);
37 } on FormatException catch (e, s) {
38 print("Source: ${e.source} @ ${e.offset}");
39 Expect.fail("Unexpected throw($name): $e\n$s");
40 }
41 }
42
43 void testStrings() {
44 // String literal containing characters, all escape types,
45 // and a number of UTF-8 encoded characters.
46 var s = r'"abc\f\ndef\r\t\b\"\/\\\u0001\u9999\uffff'
47 '\x7f\xc2\x80\xdf\xbf\xe0\xa0\x80\xef\xbf\xbf'
48 '\xf0\x90\x80\x80\xf4\x8f\xbf\xbf"'; // UTF-8.
49 var expected = "abc\f\ndef\r\t\b\"\/\\\u0001\u9999\uffff"
50 "\x7f\x80\u07ff\u0800\uffff"
51 "\u{10000}\u{10ffff}";
52 for (var i = 1; i < s.length - 1; i++) {
53 var s1 = s.substring(0, i);
54 var s2 = s.substring(i);
55 jsonTest("$s1|$s2-$i", expected, (sink) {
56 sink.add(s1.codeUnits);
57 sink.add(s2.codeUnits);
58 sink.close();
59 });
60 jsonTest("$s1|$s2-$i-slice", expected, (sink) {
61 sink.addSlice(s.codeUnits, 0, i, false);
62 sink.addSlice(s.codeUnits, i, s.length, true);
63 });
64 for (var j = i; j < s.length - 1; j++) {
65 var s2a = s.substring(i, j);
66 var s2b = s.substring(j);
67 jsonTest("$s1|$s2a|$s2b-$i-$j", expected, (sink) {
68 sink.add(s1.codeUnits);
69 sink.add(s2a.codeUnits);
70 sink.add(s2b.codeUnits);
71 sink.close();
72 });
73 }
74 }
75 }
76
77 void testNumbers() {
78 for (var number in ["-0.12e-12", "-34.12E+12", "0.0e0", "9.9E9", "0", "9"
79 "1234.56789123456701418035663664340972900390625",
80 "1.2345678912345671e-14",
81 "99999999999999999999"]) {
82 var expected = num.parse(number);
83 for (int i = 1; i < number.length - 1; i++) {
84 var p1 = number.substring(0, i);
85 var p2 = number.substring(i);
86 jsonTest("$p1|$p2", expected, (sink) {
87 sink.add(p1.codeUnits);
88 sink.add(p2.codeUnits);
89 sink.close();
90 });
91
92 jsonTest("$p1|$p2/slice", expected, (sink) {
93 sink.addSlice(number.codeUnits, 0, i, false);
94 sink.addSlice(number.codeUnits, i, number.length, true);
95 });
96
97 for (int j = i; j < number.length - 1; j++) {
98 var p2a = number.substring(i, j);
99 var p2b = number.substring(j);
100 jsonTest("$p1|$p2a|$p2b", expected, (sink) {
101 sink.add(p1.codeUnits);
102 sink.add(p2a.codeUnits);
103 sink.add(p2b.codeUnits);
104 sink.close();
105 });
106 }
107 }
108 }
109 }
110
111 // Test that `null`, `true`, and `false` keywords are decoded correctly.
112 void testKeywords() {
113 for (var expected in [null, true, false]) {
114 var s = "$expected";
115 for (int i = 1; i < s.length - 1; i++) {
116 var s1 = s.substring(0, i);
117 var s2 = s.substring(i);
118 jsonTest("$s1|$s2", expected, (sink) {
119 sink.add(s1.codeUnits);
120 sink.add(s2.codeUnits);
121 sink.close();
122 });
123 jsonTest("$s1|$s2", expected, (sink) {
124 sink.addSlice(s.codeUnits, 0, i, false);
125 sink.addSlice(s.codeUnits, i, s.length, true);
126 });
127 for (var j = i; j < s.length - 1; j++) {
128 var s2a = s.substring(i, j);
129 var s2b = s.substring(j);
130 jsonTest("$s1|$s2a|$s2b", expected, (sink) {
131 sink.add(s1.codeUnits);
132 sink.add(s2a.codeUnits);
133 sink.add(s2b.codeUnits);
134 sink.close();
135 });
136 }
137 }
138 }
139 }
140
141 void testAll() {
142 var s = r'{"":[true,false,42, -33e-3,null,"\u0080"], "z": 0}';
143 bool check(o) {
144 if (o is Map) {
145 Expect.equals(2, o.length);
146 Expect.equals(0, o["z"]);
147 var v = o[""];
148 if (v is List) {
149 Expect.listEquals([true, false, 42, -33e-3, null, "\u0080"], v);
150 } else {
151 Expect.fail("Expected list, found ${v.runtimeType}");
152 }
153 } else {
154 Expect.fail("Expected map, found ${o.runtimeType}");
155 }
156 }
157 for (var i = 1; i < s.length - 1; i++) {
158 var s1 = s.substring(0, i);
159 var s2 = s.substring(i);
160 jsonParse("$s1|$s2-$i", check, (sink) {
161 sink.add(s1.codeUnits);
162 sink.add(s2.codeUnits);
163 sink.close();
164 });
165 jsonParse("$s1|$s2-$i-slice", check, (sink) {
166 sink.addSlice(s.codeUnits, 0, i, false);
167 sink.addSlice(s.codeUnits, i, s.length, true);
168 });
169 for (var j = i; j < s.length - 1; j++) {
170 var s2a = s.substring(i, j);
171 var s2b = s.substring(j);
172 jsonParse("$s1|$s2a|$s2b-$i-$j", check, (sink) {
173 sink.add(s1.codeUnits);
174 sink.add(s2a.codeUnits);
175 sink.add(s2b.codeUnits);
176 sink.close();
177 });
178 }
179 }
180
181 }
182
183 // Check that [codes] decode to [expect] when allowing malformed UTF-8,
184 // and throws otherwise.
185 void jsonMalformedTest(name, expect, codes) {
186 // Helper method.
187 void test(name, expect, action(sink)) {
188 var tag = "Malform:$name-$expect";
189 { // Allowing malformed, expect [expect]
190 var sink = new ChunkedConversionSink.withCallback((values) {
191 var value = values[0];
192 Expect.equals(expect, value, tag);
193 });
194 var decoderSink = JSON.decoder.startChunkedConversion(sink)
195 .asUtf8Sink(true);
196 try {
197 action(decoderSink);
198 } catch (e, s) {
199 Expect.fail("Unexpected throw ($tag): $e\n$s");
200 }
201 }
202 { // Not allowing malformed, expect throw.
203 var sink = new ChunkedConversionSink.withCallback((values) {
204 Expect.unreachable(tag);
205 });
206 var decoderSink = JSON.decoder.startChunkedConversion(sink)
207 .asUtf8Sink(false);
208 Expect.throws(() { action(decoderSink); }, null, tag);
209 }
210 }
211
212 // Test all two and three part slices.
213 for (int i = 1; i < codes.length - 1; i++) {
214 test("$name:$i", expect, (sink) {
215 sink.add(codes.sublist(0, i));
216 sink.add(codes.sublist(i));
217 sink.close();
218 });
219 test("$name:$i-slice", expect, (sink) {
220 sink.addSlice(codes, 0, i, false);
221 sink.addSlice(codes, i, codes.length, true);
222 });
223 for (int j = i; j < codes.length - 1; j++) {
224 test("$name:$i|$j", expect, (sink) {
225 sink.add(codes.sublist(0, i));
226 sink.add(codes.sublist(i, j));
227 sink.add(codes.sublist(j));
228 sink.close();
229 });
230 }
231 }
232 }
233
234 // Test that `codeString.codeUnits` fails to parse as UTF-8 JSON,
235 // even with decoder not throwing on malformed encodings.
236 void jsonThrows(Strig name, String codeString) {
237 testJsonThrows(tag, action) {
238 // Not allowing malformed, expect throw.
239 var sink = new ChunkedConversionSink.withCallback((values) {
240 Expect.unreachable(tag);
241 });
242 var decoderSink = JSON.decoder.startChunkedConversion(sink)
243 .asUtf8Sink(true);
244 Expect.throws(() { action(decoderSink); }, null, tag);
245 }
246
247 var codes = codeString.codeUnits;
248 for (int i = 1; i < codes.length - 1; i++) {
249 testJsonThrows("$name:$i", (sink) {
250 sink.add(codes.sublist(0, i));
251 sink.add(codes.sublist(i));
252 sink.close();
253 });
254 testJsonThrows("$name:$i-slice", (sink) {
255 sink.addSlice(codes, 0, i, false);
256 sink.addSlice(codes, i, codes.length, true);
257 });
258 for (int j = i; j < codes.length - 1; j++) {
259 testJsonThrows("$name:$i|$j", (sink) {
260 sink.add(codes.sublist(0, i));
261 sink.add(codes.sublist(i, j));
262 sink.add(codes.sublist(j));
263 sink.close();
264 });
265 }
266 }
267 }
268
269 // Malformed UTF-8 encodings.
270 void testMalformed() {
271 // Overlong encodings.
272 jsonMalformedTest("overlong-0-2", "@\uFFFD@",
273 [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]);
274 jsonMalformedTest("overlong-0-3", "@\uFFFD@",
275 [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]);
276 jsonMalformedTest("overlong-0-4", "@\uFFFD@",
277 [0x22, 0x40, 0xf0, 0x80, 0x80, 0x80, 0x40, 0x22]);
278
279 jsonMalformedTest("overlong-7f-2", "@\uFFFD@",
280 [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]);
281 jsonMalformedTest("overlong-7f-3", "@\uFFFD@",
282 [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]);
283 jsonMalformedTest("overlong-7f-4", "@\uFFFD@",
284 [0x22, 0x40, 0xf0, 0x80, 0x81, 0xbf, 0x40, 0x22]);
285
286 jsonMalformedTest("overlong-80-3", "@\uFFFD@",
287 [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]);
288 jsonMalformedTest("overlong-80-4", "@\uFFFD@",
289 [0x22, 0x40, 0xf0, 0x80, 0x82, 0x80, 0x40, 0x22]);
290
291 jsonMalformedTest("overlong-7ff-3", "@\uFFFD@",
292 [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]);
293 jsonMalformedTest("overlong-7ff-4", "@\uFFFD@",
294 [0x22, 0x40, 0xf0, 0x80, 0x9f, 0xbf, 0x40, 0x22]);
295
296 jsonMalformedTest("overlong-800-4", "@\uFFFD@",
297 [0x22, 0x40, 0xf0, 0x80, 0xa0, 0x80, 0x40, 0x22]);
298 jsonMalformedTest("overlong-ffff-4", "@\uFFFD@",
299 [0x22, 0x40, 0xf0, 0x8f, 0xbf, 0xbf, 0x40, 0x22]);
300
301 // Unterminated multibyte sequences.
302 jsonMalformedTest("unterminated-2-normal", "@\uFFFD@",
303 [0x22, 0x40, 0xc0, 0x40, 0x22]);
304
305 jsonMalformedTest("unterminated-3-normal", "@\uFFFD@",
306 [0x22, 0x40, 0xe0, 0x80, 0x40, 0x22]);
307
308 jsonMalformedTest("unterminated-4-normal", "@\uFFFD@",
309 [0x22, 0x40, 0xf0, 0x80, 0x80, 0x40, 0x22]);
310
311 jsonMalformedTest("unterminated-2-multi", "@\uFFFD\x80@",
312 [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]);
313
314 jsonMalformedTest("unterminated-3-multi", "@\uFFFD\x80@",
315 [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
316
317 jsonMalformedTest("unterminated-4-multi", "@\uFFFD\x80@",
318 [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]);
319
320 jsonMalformedTest("unterminated-2-escape", "@\uFFFD\n@",
321 [0x22, 0x40, 0xc0, 0x5c, 0x6e, 0x40, 0x22]);
322
323 jsonMalformedTest("unterminated-3-escape", "@\uFFFD\n@",
324 [0x22, 0x40, 0xe0, 0x80, 0x5c, 0x6e, 0x40, 0x22]);
325
326 jsonMalformedTest("unterminated-4-escape", "@\uFFFD\n@",
327 [0x22, 0x40, 0xf0, 0x80, 0x80, 0x5c, 0x6e, 0x40, 0x22]);
328
329 jsonMalformedTest("unterminated-2-end", "@\uFFFD",
330 [0x22, 0x40, 0xc0, 0x22]);
331
332 jsonMalformedTest("unterminated-3-end", "@\uFFFD",
333 [0x22, 0x40, 0xe0, 0x80, 0x22]);
334
335 jsonMalformedTest("unterminated-4-end", "@\uFFFD",
336 [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]);
337
338 // Unexpected continuation byte
339 // - after a normal character.
340 jsonMalformedTest("continuation-normal", "@\uFFFD@",
341 [0x22, 0x40, 0x80, 0x40, 0x22]);
342
343 // - after a valid continuation byte.
344 jsonMalformedTest("continuation-continuation-2", "@\x80\uFFFD@",
345 [0x22, 0x40, 0xc2, 0x80, 0x80, 0x40, 0x22]);
346 jsonMalformedTest("continuation-continuation-3", "@\u0800\uFFFD@",
347 [0x22, 0x40, 0xe0, 0xa0, 0x80, 0x80, 0x40, 0x22]);
348 jsonMalformedTest("continuation-continuation-4", "@\u{10000}\uFFFD@",
349 [0x22, 0x40, 0xf0, 0x90, 0x80, 0x80, 0x80, 0x40, 0x22]);
350
351 // - after another invalid continuation byte
352 jsonMalformedTest("continuation-twice", "@\uFFFD\uFFFD\uFFFD@",
353 [0x22, 0x40, 0x80, 0x80, 0x80, 0x40, 0x22]);
354 // - at start.
355 jsonMalformedTest("continuation-start", "\uFFFD@",
356 [0x22, 0x80, 0x40, 0x22]);
357
358 // Unexpected leading byte where continuation byte expected.
359 jsonMalformedTest("leading-2", "@\uFFFD\x80@",
360 [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]);
361 jsonMalformedTest("leading-3-1", "@\uFFFD\x80@",
362 [0x22, 0x40, 0xe0, 0xc2, 0x80, 0x40, 0x22]);
363 jsonMalformedTest("leading-3-2", "@\uFFFD\x80@",
364 [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
365 jsonMalformedTest("leading-4-1", "@\uFFFD\x80@",
366 [0x22, 0x40, 0xf0, 0xc2, 0x80, 0x40, 0x22]);
367 jsonMalformedTest("leading-4-2", "@\uFFFD\x80@",
368 [0x22, 0x40, 0xf0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
369 jsonMalformedTest("leading-4-3", "@\uFFFD\x80@",
370 [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]);
371
372 // Overlong encodings of ASCII outside of strings always fail.
373 // Use Latin-1 strings as argument since most chars are correct,
374 // pass string.codeUnits to decoder as UTF-8.
375 jsonThrows("number-1", "\xc0\xab0.0e-0"); // '-' is 0x2b => \xc0\xab
376 jsonThrows("number-2", "-\xc0\xb0.0e-0"); // '0' is 0x30 => \xc0\xb0
377 jsonThrows("number-3", "-0\xc0\xae0e-0"); // '.' is 0x2e => \xc0\xae
378 jsonThrows("number-4", "-0.\xc0\xb0e-0");
379 jsonThrows("number-5", "-0.0\xc1\xa5-0"); // 'e' is 0x65 => \xc1\xa5
380 jsonThrows("number-6", "-0.0e\xc0\xab0");
381 jsonThrows("number-7", "-0.0e-\xc0\xb0");
382
383 jsonThrows("true-1", "\xc1\xb4rue"); // 't' is 0x74
384 jsonThrows("true-2", "t\xc1\xb2ue"); // 'r' is 0x72
385 jsonThrows("true-3", "tr\xc1\xb5e"); // 'u' is 0x75
386 jsonThrows("true-4", "tru\xc1\xa5"); // 'e' is 0x65
387
388 jsonThrows("false-1", "\xc1\xa6alse"); // 'f' is 0x66
389 jsonThrows("false-2", "f\xc1\xa1lse"); // 'a' is 0x61
390 jsonThrows("false-3", "fa\xc1\xacse"); // 'l' is 0x6c
391 jsonThrows("false-4", "fal\xc1\xb3e"); // 's' is 0x73
392 jsonThrows("false-5", "fals\xc1\xa5"); // 'e' is 0x65
393
394 jsonThrows("null-1", "\xc1\xaeull"); // 'n' is 0x6e
395 jsonThrows("null-2", "n\xc1\xb5ll"); // 'u' is 0x75
396 jsonThrows("null-3", "nu\xc1\xacl"); // 'l' is 0x6c
397 jsonThrows("null-4", "nul\xc1\xac"); // 'l' is 0x6c
398
399 jsonThrows("array-1", "\xc1\x9b0,0]"); // '[' is 0x5b
400 jsonThrows("array-2", "[0,0\xc1\x9d"); // ']' is 0x5d
401 jsonThrows("array-2", "[0\xc0\xac0]"); // ',' is 0x2c
402
403 jsonThrows("object-1", '\xc1\xbb"x":0}'); // '{' is 0x7b
404 jsonThrows("object-2", '{"x":0\xc1\xbd'); // '}' is 0x7d
405 jsonThrows("object-2", '{"x\xc0\xba0}'); // ':' is 0x3a
406
407 jsonThrows("string-1", '\xc0\xa2x"'); // '"' is 0x22
408 jsonThrows("string-1", '"x\xc0\xa2'); // Unterminated string.
409
410 jsonThrows("whitespace-1", "\xc0\xa01"); // ' ' is 0x20
411 }
OLDNEW
« tests/lib/convert/json_chunk_test.dart ('K') | « tests/lib/convert/json_test.dart ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698