Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(336)

Side by Side Diff: test/runner/json_reporter_test.dart

Issue 1641353002: Expand the JSON reporter to enable a progress bar. (Closed) Base URL: git@github.com:dart-lang/test@master
Patch Set: Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« json_reporter.md ('K') | « pubspec.yaml ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 @TestOn("vm") 5 @TestOn("vm")
6 6
7 import 'dart:convert'; 7 import 'dart:convert';
8 8
9 import 'package:scheduled_test/descriptor.dart' as d; 9 import 'package:scheduled_test/descriptor.dart' as d;
10 import 'package:scheduled_test/scheduled_stream.dart'; 10 import 'package:scheduled_test/scheduled_stream.dart';
(...skipping 13 matching lines...) Expand all
24 void main() { 24 void main() {
25 useSandbox(); 25 useSandbox();
26 26
27 test("runs several successful tests and reports when each completes", () { 27 test("runs several successful tests and reports when each completes", () {
28 _expectReport(""" 28 _expectReport("""
29 test('success 1', () {}); 29 test('success 1', () {});
30 test('success 2', () {}); 30 test('success 2', () {});
31 test('success 3', () {}); 31 test('success 3', () {});
32 """, [ 32 """, [
33 _start, 33 _start,
34 _allSuites(),
34 _suite(0), 35 _suite(0),
35 _testStart(1, "loading test.dart", groupIDs: []), 36 _testStart(1, "loading test.dart", groupIDs: []),
36 _testDone(1, hidden: true), 37 _testDone(1, hidden: true),
37 _group(2), 38 _group(2, testCount: 3),
38 _testStart(3, "success 1"), 39 _testStart(3, "success 1"),
39 _testDone(3), 40 _testDone(3),
40 _testStart(4, "success 2"), 41 _testStart(4, "success 2"),
41 _testDone(4), 42 _testDone(4),
42 _testStart(5, "success 3"), 43 _testStart(5, "success 3"),
43 _testDone(5), 44 _testDone(5),
44 _done() 45 _done()
45 ]); 46 ]);
46 }); 47 });
47 48
48 test("runs several failing tests and reports when each fails", () { 49 test("runs several failing tests and reports when each fails", () {
49 _expectReport(""" 50 _expectReport("""
50 test('failure 1', () => throw new TestFailure('oh no')); 51 test('failure 1', () => throw new TestFailure('oh no'));
51 test('failure 2', () => throw new TestFailure('oh no')); 52 test('failure 2', () => throw new TestFailure('oh no'));
52 test('failure 3', () => throw new TestFailure('oh no')); 53 test('failure 3', () => throw new TestFailure('oh no'));
53 """, [ 54 """, [
54 _start, 55 _start,
56 _allSuites(),
55 _suite(0), 57 _suite(0),
56 _testStart(1, "loading test.dart", groupIDs: []), 58 _testStart(1, "loading test.dart", groupIDs: []),
57 _testDone(1, hidden: true), 59 _testDone(1, hidden: true),
58 _group(2), 60 _group(2, testCount: 3),
59 _testStart(3, "failure 1"), 61 _testStart(3, "failure 1"),
60 _error(3, "oh no", isFailure: true), 62 _error(3, "oh no", isFailure: true),
61 _testDone(3, result: "failure"), 63 _testDone(3, result: "failure"),
62 _testStart(4, "failure 2"), 64 _testStart(4, "failure 2"),
63 _error(4, "oh no", isFailure: true), 65 _error(4, "oh no", isFailure: true),
64 _testDone(4, result: "failure"), 66 _testDone(4, result: "failure"),
65 _testStart(5, "failure 3"), 67 _testStart(5, "failure 3"),
66 _error(5, "oh no", isFailure: true), 68 _error(5, "oh no", isFailure: true),
67 _testDone(5, result: "failure"), 69 _testDone(5, result: "failure"),
68 _done(success: false) 70 _done(success: false)
(...skipping 17 matching lines...) Expand all
86 }); 88 });
87 89
88 test("runs failing tests along with successful tests", () { 90 test("runs failing tests along with successful tests", () {
89 _expectReport(""" 91 _expectReport("""
90 test('failure 1', () => throw new TestFailure('oh no')); 92 test('failure 1', () => throw new TestFailure('oh no'));
91 test('success 1', () {}); 93 test('success 1', () {});
92 test('failure 2', () => throw new TestFailure('oh no')); 94 test('failure 2', () => throw new TestFailure('oh no'));
93 test('success 2', () {}); 95 test('success 2', () {});
94 """, [ 96 """, [
95 _start, 97 _start,
98 _allSuites(),
96 _suite(0), 99 _suite(0),
97 _testStart(1, "loading test.dart", groupIDs: []), 100 _testStart(1, "loading test.dart", groupIDs: []),
98 _testDone(1, hidden: true), 101 _testDone(1, hidden: true),
99 _group(2), 102 _group(2, testCount: 4),
100 _testStart(3, "failure 1"), 103 _testStart(3, "failure 1"),
101 _error(3, "oh no", isFailure: true), 104 _error(3, "oh no", isFailure: true),
102 _testDone(3, result: "failure"), 105 _testDone(3, result: "failure"),
103 _testStart(4, "success 1"), 106 _testStart(4, "success 1"),
104 _testDone(4), 107 _testDone(4),
105 _testStart(5, "failure 2"), 108 _testStart(5, "failure 2"),
106 _error(5, "oh no", isFailure: true), 109 _error(5, "oh no", isFailure: true),
107 _testDone(5, result: "failure"), 110 _testDone(5, result: "failure"),
108 _testStart(6, "success 2"), 111 _testStart(6, "success 2"),
109 _testDone(6), 112 _testDone(6),
110 _done(success: false) 113 _done(success: false)
111 ]); 114 ]);
112 }); 115 });
113 116
114 test("gracefully handles multiple test failures in a row", () { 117 test("gracefully handles multiple test failures in a row", () {
115 _expectReport(""" 118 _expectReport("""
116 // This completer ensures that the test isolate isn't killed until all 119 // This completer ensures that the test isolate isn't killed until all
117 // errors have been thrown. 120 // errors have been thrown.
118 var completer = new Completer(); 121 var completer = new Completer();
119 test('failures', () { 122 test('failures', () {
120 new Future.microtask(() => throw 'first error'); 123 new Future.microtask(() => throw 'first error');
121 new Future.microtask(() => throw 'second error'); 124 new Future.microtask(() => throw 'second error');
122 new Future.microtask(() => throw 'third error'); 125 new Future.microtask(() => throw 'third error');
123 new Future.microtask(completer.complete); 126 new Future.microtask(completer.complete);
124 }); 127 });
125 test('wait', () => completer.future); 128 test('wait', () => completer.future);
126 """, [ 129 """, [
127 _start, 130 _start,
131 _allSuites(),
128 _suite(0), 132 _suite(0),
129 _testStart(1, "loading test.dart", groupIDs: []), 133 _testStart(1, "loading test.dart", groupIDs: []),
130 _testDone(1, hidden: true), 134 _testDone(1, hidden: true),
131 _group(2), 135 _group(2, testCount: 2),
132 _testStart(3, "failures"), 136 _testStart(3, "failures"),
133 _error(3, "first error"), 137 _error(3, "first error"),
134 _error(3, "second error"), 138 _error(3, "second error"),
135 _error(3, "third error"), 139 _error(3, "third error"),
136 _testDone(3, result: "error"), 140 _testDone(3, result: "error"),
137 _testStart(4, "wait"), 141 _testStart(4, "wait"),
138 _testDone(4), 142 _testDone(4),
139 _done(success: false) 143 _done(success: false)
140 ]); 144 ]);
141 }); 145 });
(...skipping 10 matching lines...) Expand all
152 new Future.microtask(testDone.complete); 156 new Future.microtask(testDone.complete);
153 throw 'oh no'; 157 throw 'oh no';
154 }); 158 });
155 }); 159 });
156 test('wait', () { 160 test('wait', () {
157 waitStarted.complete(); 161 waitStarted.complete();
158 return testDone.future; 162 return testDone.future;
159 }); 163 });
160 """, [ 164 """, [
161 _start, 165 _start,
166 _allSuites(),
162 _suite(0), 167 _suite(0),
163 _testStart(1, "loading test.dart", groupIDs: []), 168 _testStart(1, "loading test.dart", groupIDs: []),
164 _testDone(1, hidden: true), 169 _testDone(1, hidden: true),
165 _group(2), 170 _group(2, testCount: 2),
166 _testStart(3, "failure"), 171 _testStart(3, "failure"),
167 _testDone(3), 172 _testDone(3),
168 _testStart(4, "wait"), 173 _testStart(4, "wait"),
169 _error(3, "oh no"), 174 _error(3, "oh no"),
170 _error(3, 175 _error(3,
171 "This test failed after it had already completed. Make sure to " 176 "This test failed after it had already completed. Make sure to "
172 "use [expectAsync]\n" 177 "use [expectAsync]\n"
173 "or the [completes] matcher when testing async code."), 178 "or the [completes] matcher when testing async code."),
174 _testDone(4), 179 _testDone(4),
175 _done(success: false) 180 _done(success: false)
176 ]); 181 ]);
177 }); 182 });
178 183
179 test("reports each test in its proper groups", () { 184 test("reports each test in its proper groups", () {
180 _expectReport(""" 185 _expectReport("""
181 group('group 1', () { 186 group('group 1', () {
182 group('.2', () { 187 group('.2', () {
183 group('.3', () { 188 group('.3', () {
184 test('success', () {}); 189 test('success', () {});
185 }); 190 });
186 }); 191 });
187 192
188 test('success', () {}); 193 test('success', () {});
189 test('success', () {}); 194 test('success', () {});
190 }); 195 });
191 """, [ 196 """, [
192 _start, 197 _start,
198 _allSuites(),
193 _suite(0), 199 _suite(0),
194 _testStart(1, "loading test.dart", groupIDs: []), 200 _testStart(1, "loading test.dart", groupIDs: []),
195 _testDone(1, hidden: true), 201 _testDone(1, hidden: true),
196 _group(2), 202 _group(2, testCount: 3),
197 _group(3, name: "group 1", parentID: 2), 203 _group(3, name: "group 1", parentID: 2, testCount: 3),
198 _group(4, name: "group 1 .2", parentID: 3), 204 _group(4, name: "group 1 .2", parentID: 3),
199 _group(5, name: "group 1 .2 .3", parentID: 4), 205 _group(5, name: "group 1 .2 .3", parentID: 4),
200 _testStart(6, 'group 1 .2 .3 success', groupIDs: [2, 3, 4, 5]), 206 _testStart(6, 'group 1 .2 .3 success', groupIDs: [2, 3, 4, 5]),
201 _testDone(6), 207 _testDone(6),
202 _testStart(7, 'group 1 success', groupIDs: [2, 3]), 208 _testStart(7, 'group 1 success', groupIDs: [2, 3]),
203 _testDone(7), 209 _testDone(7),
204 _testStart(8, 'group 1 success', groupIDs: [2, 3]), 210 _testStart(8, 'group 1 success', groupIDs: [2, 3]),
205 _testDone(8), 211 _testDone(8),
206 _done() 212 _done()
207 ]); 213 ]);
208 }); 214 });
209 215
210 group("print:", () { 216 group("print:", () {
211 test("handles multiple prints", () { 217 test("handles multiple prints", () {
212 _expectReport(""" 218 _expectReport("""
213 test('test', () { 219 test('test', () {
214 print("one"); 220 print("one");
215 print("two"); 221 print("two");
216 print("three"); 222 print("three");
217 print("four"); 223 print("four");
218 }); 224 });
219 """, [ 225 """, [
220 _start, 226 _start,
227 _allSuites(),
221 _suite(0), 228 _suite(0),
222 _testStart(1, "loading test.dart", groupIDs: []), 229 _testStart(1, "loading test.dart", groupIDs: []),
223 _testDone(1, hidden: true), 230 _testDone(1, hidden: true),
224 _group(2), 231 _group(2),
225 _testStart(3, 'test'), 232 _testStart(3, 'test'),
226 _print(3, "one"), 233 _print(3, "one"),
227 _print(3, "two"), 234 _print(3, "two"),
228 _print(3, "three"), 235 _print(3, "three"),
229 _print(3, "four"), 236 _print(3, "four"),
230 _testDone(3), 237 _testDone(3),
(...skipping 16 matching lines...) Expand all
247 new Future(testDone.complete); 254 new Future(testDone.complete);
248 }); 255 });
249 }); 256 });
250 257
251 test('wait', () { 258 test('wait', () {
252 waitStarted.complete(); 259 waitStarted.complete();
253 return testDone.future; 260 return testDone.future;
254 }); 261 });
255 """, [ 262 """, [
256 _start, 263 _start,
264 _allSuites(),
257 _suite(0), 265 _suite(0),
258 _testStart(1, "loading test.dart", groupIDs: []), 266 _testStart(1, "loading test.dart", groupIDs: []),
259 _testDone(1, hidden: true), 267 _testDone(1, hidden: true),
260 _group(2), 268 _group(2, testCount: 2),
261 _testStart(3, 'test'), 269 _testStart(3, 'test'),
262 _testDone(3), 270 _testDone(3),
263 _testStart(4, 'wait'), 271 _testStart(4, 'wait'),
264 _print(3, "one"), 272 _print(3, "one"),
265 _print(3, "two"), 273 _print(3, "two"),
266 _print(3, "three"), 274 _print(3, "three"),
267 _print(3, "four"), 275 _print(3, "four"),
268 _testDone(4), 276 _testDone(4),
269 _done() 277 _done()
270 ]); 278 ]);
(...skipping 18 matching lines...) Expand all
289 }); 297 });
290 298
291 print("one"); 299 print("one");
292 print("two"); 300 print("two");
293 throw "first error"; 301 throw "first error";
294 }); 302 });
295 303
296 test('wait', () => completer.future); 304 test('wait', () => completer.future);
297 """, [ 305 """, [
298 _start, 306 _start,
307 _allSuites(),
299 _suite(0), 308 _suite(0),
300 _testStart(1, "loading test.dart", groupIDs: []), 309 _testStart(1, "loading test.dart", groupIDs: []),
301 _testDone(1, hidden: true), 310 _testDone(1, hidden: true),
302 _group(2), 311 _group(2, testCount: 2),
303 _testStart(3, 'test'), 312 _testStart(3, 'test'),
304 _print(3, "one"), 313 _print(3, "one"),
305 _print(3, "two"), 314 _print(3, "two"),
306 _error(3, "first error"), 315 _error(3, "first error"),
307 _print(3, "three"), 316 _print(3, "three"),
308 _print(3, "four"), 317 _print(3, "four"),
309 _error(3, "second error"), 318 _error(3, "second error"),
310 _print(3, "five"), 319 _print(3, "five"),
311 _print(3, "six"), 320 _print(3, "six"),
312 _testDone(3, result: "error"), 321 _testDone(3, result: "error"),
313 _testStart(4, 'wait'), 322 _testStart(4, 'wait'),
314 _testDone(4), 323 _testDone(4),
315 _done(success: false) 324 _done(success: false)
316 ]); 325 ]);
317 }); 326 });
318 }); 327 });
319 328
320 group("skip:", () { 329 group("skip:", () {
321 test("reports skipped tests", () { 330 test("reports skipped tests", () {
322 _expectReport(""" 331 _expectReport("""
323 test('skip 1', () {}, skip: true); 332 test('skip 1', () {}, skip: true);
324 test('skip 2', () {}, skip: true); 333 test('skip 2', () {}, skip: true);
325 test('skip 3', () {}, skip: true); 334 test('skip 3', () {}, skip: true);
326 """, [ 335 """, [
327 _start, 336 _start,
337 _allSuites(),
328 _suite(0), 338 _suite(0),
329 _testStart(1, "loading test.dart", groupIDs: []), 339 _testStart(1, "loading test.dart", groupIDs: []),
330 _testDone(1, hidden: true), 340 _testDone(1, hidden: true),
331 _group(2), 341 _group(2, testCount: 3),
332 _testStart(3, "skip 1", skip: true), 342 _testStart(3, "skip 1", skip: true),
333 _testDone(3), 343 _testDone(3),
334 _testStart(4, "skip 2", skip: true), 344 _testStart(4, "skip 2", skip: true),
335 _testDone(4), 345 _testDone(4),
336 _testStart(5, "skip 3", skip: true), 346 _testStart(5, "skip 3", skip: true),
337 _testDone(5), 347 _testDone(5),
338 _done() 348 _done()
339 ]); 349 ]);
340 }); 350 });
341 351
342 test("reports skipped groups", () { 352 test("reports skipped groups", () {
343 _expectReport(""" 353 _expectReport("""
344 group('skip', () { 354 group('skip', () {
345 test('success 1', () {}); 355 test('success 1', () {});
346 test('success 2', () {}); 356 test('success 2', () {});
347 test('success 3', () {}); 357 test('success 3', () {});
348 }, skip: true); 358 }, skip: true);
349 """, [ 359 """, [
350 _start, 360 _start,
361 _allSuites(),
351 _suite(0), 362 _suite(0),
352 _testStart(1, "loading test.dart", groupIDs: []), 363 _testStart(1, "loading test.dart", groupIDs: []),
353 _testDone(1, hidden: true), 364 _testDone(1, hidden: true),
354 _group(2), 365 _group(2, testCount: 0),
355 _group(3, name: "skip", parentID: 2, skip: true), 366 _group(3, name: "skip", parentID: 2, skip: true, testCount: 0),
356 _testStart(4, "skip", groupIDs: [2, 3], skip: true), 367 _testStart(4, "skip", groupIDs: [2, 3], skip: true),
357 _testDone(4), 368 _testDone(4),
358 _done() 369 _done()
359 ]); 370 ]);
360 }); 371 });
361 372
362 test("reports the skip reason if available", () { 373 test("reports the skip reason if available", () {
363 _expectReport(""" 374 _expectReport("""
364 test('skip 1', () {}, skip: 'some reason'); 375 test('skip 1', () {}, skip: 'some reason');
365 test('skip 2', () {}, skip: 'or another'); 376 test('skip 2', () {}, skip: 'or another');
366 """, [ 377 """, [
367 _start, 378 _start,
379 _allSuites(),
368 _suite(0), 380 _suite(0),
369 _testStart(1, "loading test.dart", groupIDs: []), 381 _testStart(1, "loading test.dart", groupIDs: []),
370 _testDone(1, hidden: true), 382 _testDone(1, hidden: true),
371 _group(2), 383 _group(2, testCount: 2),
372 _testStart(3, "skip 1", skip: "some reason"), 384 _testStart(3, "skip 1", skip: "some reason"),
373 _testDone(3), 385 _testDone(3),
374 _testStart(4, "skip 2", skip: "or another"), 386 _testStart(4, "skip 2", skip: "or another"),
375 _testDone(4), 387 _testDone(4),
376 _done() 388 _done()
377 ]); 389 ]);
378 }); 390 });
379 }); 391 });
380 } 392 }
381 393
(...skipping 27 matching lines...) Expand all
409 // Remove excess trailing whitespace. 421 // Remove excess trailing whitespace.
410 for (var i = 0; i < stdoutLines.length; i++) { 422 for (var i = 0; i < stdoutLines.length; i++) {
411 var event = JSON.decode(stdoutLines[i]); 423 var event = JSON.decode(stdoutLines[i]);
412 expect(event.remove("time"), new isInstanceOf<int>()); 424 expect(event.remove("time"), new isInstanceOf<int>());
413 event.remove("stackTrace"); 425 event.remove("stackTrace");
414 expect(event, equals(expected[i])); 426 expect(event, equals(expected[i]));
415 } 427 }
416 }); 428 });
417 } 429 }
418 430
431 /// Returns the event emitted by the JSON reporter providing information about
432 /// all suites.
433 ///
434 /// The [count] defaults to 1.
435 Map _allSuites({int count}) {
436 return {
437 "type": "allSuites",
438 "count": count ?? 1
439 };
440 }
441
419 /// Returns the event emitted by the JSON reporter indicating that a suite has 442 /// Returns the event emitted by the JSON reporter indicating that a suite has
420 /// begun running. 443 /// begun running.
421 /// 444 ///
422 /// The [platform] defaults to `"vm"`, the [path] defaults to `"test.dart"`. 445 /// The [platform] defaults to `"vm"`, the [path] defaults to `"test.dart"`.
423 Map _suite(int id, {String platform, String path}) { 446 Map _suite(int id, {String platform, String path}) {
424 return { 447 return {
425 "type": "suite", 448 "type": "suite",
426 "suite": { 449 "suite": {
427 "id": id, 450 "id": id,
428 "platform": platform ?? "vm", 451 "platform": platform ?? "vm",
429 "path": path ?? "test.dart" 452 "path": path ?? "test.dart"
430 } 453 }
431 }; 454 };
432 } 455 }
433 456
434 /// Returns the event emitted by the JSON reporter indicating that a group has 457 /// Returns the event emitted by the JSON reporter indicating that a group has
435 /// begun running. 458 /// begun running.
436 /// 459 ///
437 /// If [skip] is `true`, the group is expected to be marked as skipped without a 460 /// If [skip] is `true`, the group is expected to be marked as skipped without a
438 /// reason. If it's a [String], the group is expected to be marked as skipped 461 /// reason. If it's a [String], the group is expected to be marked as skipped
439 /// with that reason. 462 /// with that reason.
440 Map _group(int id, {String name, int suiteID, int parentID, skip}) { 463 ///
464 /// The [testCount] parameter indicates the number of tests in the group. It
465 /// defaults to 1.
466 Map _group(int id, {String name, int suiteID, int parentID, skip,
467 int testCount}) {
441 return { 468 return {
442 "type": "group", 469 "type": "group",
443 "group": { 470 "group": {
444 "id": id, 471 "id": id,
445 "name": name, 472 "name": name,
446 "suiteID": suiteID ?? 0, 473 "suiteID": suiteID ?? 0,
447 "parentID": parentID, 474 "parentID": parentID,
448 "metadata": _metadata(skip: skip) 475 "metadata": _metadata(skip: skip),
476 "testCount": testCount ?? 1
449 } 477 }
450 }; 478 };
451 } 479 }
452 480
453 /// Returns the event emitted by the JSON reporter indicating that a test has 481 /// Returns the event emitted by the JSON reporter indicating that a test has
454 /// begun running. 482 /// begun running.
455 /// 483 ///
456 /// If [parentIDs] is passed, it's the IDs of groups containing this test. If 484 /// If [parentIDs] is passed, it's the IDs of groups containing this test. If
457 /// [skip] is `true`, the test is expected to be marked as skipped without a 485 /// [skip] is `true`, the test is expected to be marked as skipped without a
458 /// reason. If it's a [String], the test is expected to be marked as skipped 486 /// reason. If it's a [String], the test is expected to be marked as skipped
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after
515 /// Returns the serialized metadata corresponding to [skip]. 543 /// Returns the serialized metadata corresponding to [skip].
516 Map _metadata({skip}) { 544 Map _metadata({skip}) {
517 if (skip == true) { 545 if (skip == true) {
518 return {"skip": true, "skipReason": null}; 546 return {"skip": true, "skipReason": null};
519 } else if (skip is String) { 547 } else if (skip is String) {
520 return {"skip": true, "skipReason": skip}; 548 return {"skip": true, "skipReason": skip};
521 } else { 549 } else {
522 return {"skip": false, "skipReason": null}; 550 return {"skip": false, "skipReason": null};
523 } 551 }
524 } 552 }
OLDNEW
« json_reporter.md ('K') | « pubspec.yaml ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698