Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(995)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: test/runner/json_reporter_test.dart

Issue 1461293005: Add a JSON reporter. (Closed) Base URL: git@github.com:dart-lang/test@master
Patch Set: Created 5 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« json_reporter.schema.json ('K') | « test/runner/expanded_reporter_test.dart ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: test/runner/json_reporter_test.dart
diff --git a/test/runner/json_reporter_test.dart b/test/runner/json_reporter_test.dart
new file mode 100644
index 0000000000000000000000000000000000000000..c7dd6a6027284ad093e2bdc38e9d0ceb00b1cd53
--- /dev/null
+++ b/test/runner/json_reporter_test.dart
@@ -0,0 +1,404 @@
+// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
kevmoo 2015/11/21 03:37:17 several type errors, failures etc...you're on this
nweiz 2015/11/23 22:24:08 Yes.
+
+import 'dart:convert';
+
+import 'package:path/path.dart' as p;
+import 'package:scheduled_test/descriptor.dart' as d;
+import 'package:scheduled_test/scheduled_process.dart';
+import 'package:scheduled_test/scheduled_stream.dart';
+import 'package:scheduled_test/scheduled_test.dart';
+
+import 'package:test/src/runner/version.dart';
+
+import '../io.dart';
+
+final _start = {
+ "type": "start",
+ "protocolVersion": "0.1.0",
+ "runnerVersion": testVersion
+};
+
+void main() {
+ useSandbox();
+
+ test("runs several successful tests and reports when each completes", () {
+ _expectReport(
+ """
+ test('success 1', () {});
+ test('success 2', () {});
+ test('success 3', () {});""",
+ [
+ _start,
+ _testStart(0, "loading test.dart"),
+ _testDone(0, hidden: true),
+ _testStart(1, "success 1"),
+ _testDone(1),
+ _testStart(2, "success 2"),
+ _testDone(2),
+ _testStart(3, "success 3"),
+ _testDone(3),
+ _done()
+ ]);
+ });
+
+ test("runs several failing tests and reports when each fails", () {
+ _expectReport(
+ """
+ test('failure 1', () => throw new TestFailure('oh no'));
+ test('failure 2', () => throw new TestFailure('oh no'));
+ test('failure 3', () => throw new TestFailure('oh no'));""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"failure 1","time":\\d+}
+ {"type":"fail","name":"failure 1","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 6:33 main.<fn>\\\\n"}
+ {"type":"start","name":"failure 2","time":\\d+}
+ {"type":"fail","name":"failure 2","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 7:33 main.<fn>\\\\n"}
+ {"type":"start","name":"failure 3","time":\\d+}
+ {"type":"fail","name":"failure 3","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 8:33 main.<fn>\\\\n"}
+ {"type":"exit","name":"exit","time":\\d+}""");
+ });
+
+ test("includes the full stack trace with --verbose-trace", () {
+ d
+ .file(
+ "test.dart",
+ """
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+ test("failure", () => throw "oh no");
+}
+""")
+ .create();
+
+ var test = runTest(["--verbose-trace", "test.dart"], compact: true);
+ test.stdout.expect(consumeThrough(contains("dart:isolate-patch")));
+ test.shouldExit(1);
+ });
+
+ test("runs failing tests along with successful tests", () {
+ _expectReport(
+ """
+ test('failure 1', () => throw new TestFailure('oh no'));
+ test('success 1', () {});
+ test('failure 2', () => throw new TestFailure('oh no'));
+ test('success 2', () {});""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"failure 1","time":\\d+}
+ {"type":"fail","name":"failure 1","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 6:33 main.<fn>\\\\n"}
+ {"type":"start","name":"success 1","time":\\d+}
+ {"type":"pass","name":"success 1","time":\\d+}
+ {"type":"start","name":"failure 2","time":\\d+}
+ {"type":"fail","name":"failure 2","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 8:33 main.<fn>\\\\n"}
+ {"type":"start","name":"success 2","time":\\d+}
+ {"type":"pass","name":"success 2","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+
+ test("always prints the full test name", () {
+ _expectReport(
+ """
+ test(
+ 'really gosh dang long test name. Even longer than that. No, yet '
+ 'longer. A little more... okay, that should do it.',
+ () {});""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"really gosh dang long test name. Even longer tha
+n that. No, yet longer. A little more... okay, that should do it.","time":\\d+}
+ {"type":"pass","name":"really gosh dang long test name. Even longer than
+ that. No, yet longer. A little more... okay, that should do it.","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+
+ test("gracefully handles multiple test failures in a row", () {
+ _expectReport(
+ """
+ // This completer ensures that the test isolate isn't killed until all
+ // errors have been thrown.
+ var completer = new Completer();
+ test('failures', () {
+ new Future.microtask(() => throw 'first error');
+ new Future.microtask(() => throw 'second error');
+ new Future.microtask(() => throw 'third error');
+ new Future.microtask(completer.complete);
+ });
+ test('wait', () => completer.future);""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"failures","time":\\d+}
+ {"type":"error","name":"failures","time":\\d+,"errorMessage":"first erro
+r","stackTrace":"test.dart 10:38 main.<fn>.<fn>\\\\n===== asynchronous gap ====
+=======================\\\\ndart:async Future.Future.microtask\\\\ntest.da
+rt 10:15 main.<fn>\\\\n"}
+ {"type":"error","name":"failures","time":\\d+,"errorMessage":"second err
+or","stackTrace":"test.dart 11:38 main.<fn>.<fn>\\\\n===== asynchronous gap ===
+========================\\\\ndart:async Future.Future.microtask\\\\ntest.d
+art 11:15 main.<fn>\\\\n"}
+ {"type":"error","name":"failures","time":\\d+,"errorMessage":"third erro
+r","stackTrace":"test.dart 12:38 main.<fn>.<fn>\\\\n===== asynchronous gap ====
+=======================\\\\ndart:async Future.Future.microtask\\\\ntest.da
+rt 12:15 main.<fn>\\\\n"}
+ {"type":"start","name":"wait","time":\\d+}
+ {"type":"pass","name":"wait","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+
+ group("print:", () {
+ test("handles multiple prints", () {
+ _expectReport(
+ """
+ test('test', () {
+ print("one");
+ print("two");
+ print("three");
+ print("four");
+ });""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"test","time":\\d+}
+ {"type":"print","name":"test","time":\\d+,"message":"one"}
+ {"type":"print","name":"test","time":\\d+,"message":"two"}
+ {"type":"print","name":"test","time":\\d+,"message":"three"}
+ {"type":"print","name":"test","time":\\d+,"message":"four"}
+ {"type":"pass","name":"test","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+
+ test("handles a print after the test completes", () {
+ _expectReport(
+ """
+ // This completer ensures that the test isolate isn't killed until all
+ // prints have happened.
+ var testDone = new Completer();
+ var waitStarted = new Completer();
+ test('test', () async {
+ waitStarted.future.then((_) {
+ new Future(() => print("one"));
+ new Future(() => print("two"));
+ new Future(() => print("three"));
+ new Future(() => print("four"));
+ new Future(testDone.complete);
+ });
+ });
+
+ test('wait', () {
+ waitStarted.complete();
+ return testDone.future;
+ });""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"test","time":\\d+}
+ {"type":"pass","name":"test","time":\\d+}
+ {"type":"start","name":"wait","time":\\d+}
+ {"type":"print","name":"test","time":\\d+,"message":"one"}
+ {"type":"print","name":"test","time":\\d+,"message":"two"}
+ {"type":"print","name":"test","time":\\d+,"message":"three"}
+ {"type":"print","name":"test","time":\\d+,"message":"four"}
+ {"type":"pass","name":"wait","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+
+ test("interleaves prints and errors", () {
+ _expectReport(
+ """
+ // This completer ensures that the test isolate isn't killed until all
+ // prints have happened.
+ var completer = new Completer();
+ test('test', () {
+ scheduleMicrotask(() {
+ print("three");
+ print("four");
+ throw "second error";
+ });
+
+ scheduleMicrotask(() {
+ print("five");
+ print("six");
+ completer.complete();
+ });
+
+ print("one");
+ print("two");
+ throw "first error";
+ });
+
+ test('wait', () => completer.future);""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"test","time":\\d+}
+ {"type":"print","name":"test","time":\\d+,"message":"one"}
+ {"type":"print","name":"test","time":\\d+,"message":"two"}
+ {"type":"error","name":"test","time":\\d+,"errorMessage":"first error","
+stackTrace":"test.dart 24:11 main.<fn>\\\\n"}
+ {"type":"print","name":"test","time":\\d+,"message":"three"}
+ {"type":"print","name":"test","time":\\d+,"message":"four"}
+ {"type":"error","name":"test","time":\\d+,"errorMessage":"second error",
+"stackTrace":"test.dart 13:13 main.<fn>.<fn>\\\\n===== asynchronous gap =======
+====================\\\\ndart:async scheduleMicrotask\\\\ntest.dart 10:11
+ main.<fn>\\\\n"}
+ {"type":"print","name":"test","time":\\d+,"message":"five"}
+ {"type":"print","name":"test","time":\\d+,"message":"six"}
+ {"type":"start","name":"wait","time":\\d+}
+ {"type":"pass","name":"wait","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+ });
+
+ group("skip:", () {
+ test("displays skipped tests separately", () {
+ _expectReport(
+ """
+ test('skip 1', () {}, skip: true);
+ test('skip 2', () {}, skip: true);
+ test('skip 3', () {}, skip: true);""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"skip 1","time":\\d+}
+ {"type":"skip","name":"skip 1","time":\\d+}
+ {"type":"start","name":"skip 2","time":\\d+}
+ {"type":"skip","name":"skip 2","time":\\d+}
+ {"type":"start","name":"skip 3","time":\\d+}
+ {"type":"skip","name":"skip 3","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+
+ test("runs skipped tests along with successful tests", () {
+ _expectReport(
+ """
+ test('skip 1', () {}, skip: true);
+ test('success 1', () {});
+ test('skip 2', () {}, skip: true);
+ test('success 2', () {});""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"skip 1","time":\\d+}
+ {"type":"skip","name":"skip 1","time":\\d+}
+ {"type":"start","name":"success 1","time":\\d+}
+ {"type":"pass","name":"success 1","time":\\d+}
+ {"type":"start","name":"skip 2","time":\\d+}
+ {"type":"skip","name":"skip 2","time":\\d+}
+ {"type":"start","name":"success 2","time":\\d+}
+ {"type":"pass","name":"success 2","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+
+ test("runs skipped tests along with successful and failing tests", () {
+ _expectReport(
+ """
+ test('failure 1', () => throw new TestFailure('oh no'));
+ test('skip 1', () {}, skip: true);
+ test('success 1', () {});
+ test('failure 2', () => throw new TestFailure('oh no'));
+ test('skip 2', () {}, skip: true);
+ test('success 2', () {});""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"failure 1","time":\\d+}
+ {"type":"fail","name":"failure 1","time":\\d+,"failMessage":"oh no","s
+tackTrace":"test.dart 6:35 main.<fn>\\\\n"}
+ {"type":"start","name":"skip 1","time":\\d+}
+ {"type":"skip","name":"skip 1","time":1\\d+}
+ {"type":"start","name":"success 1","time":\\d+}
+ {"type":"pass","name":"success 1","time":\\d+}
+ {"type":"start","name":"failure 2","time":\\d+}
+ {"type":"fail","name":"failure 2","time":\\d+,"failMessage":"oh no","s
+tackTrace":"test.dart 9:35 main.<fn>\\\\n"}
+ {"type":"start","name":"skip 2","time":\\d+}
+ {"type":"skip","name":"skip 2","time":\\d+}
+ {"type":"start","name":"success 2","time":\\d+}
+ {"type":"pass","name":"success 2","time":\\d+}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+
+ test("displays the skip reason if available", () {
+ _expectReport(
+ """
+ test('skip 1', () {}, skip: 'some reason');
+ test('skip 2', () {}, skip: 'or another');""",
+ """
+ {"type":"enter","name":"enter","time":\\d+}
+ {"type":"start","name":"skip 1","time":\\d+}
+ {"type":"skip","name":"skip 1","time":\\d+,"reason":"some reason"}
+ {"type":"start","name":"skip 2","time":\\d+}
+ {"type":"skip","name":"skip 2","time":\\d+,"reason":"or another"}
+ {"type":"exit","name":"exit","time":\\d+}
+ """);
+ });
+ });
+}
+
+void _expectReport(String tests, List<Map> expected) {
+ var dart = """
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+$tests
+}
+""";
+
+ d.file("test.dart", dart).create();
+
+ var test = runTest(["test.dart"], reporter: "json");
+ test.shouldExit();
+
+ schedule(() async {
+ var stdoutLines = await test.stdoutStream().toList();
+
+ expect(stdoutLines.length, equals(expected.length),
+ reason: "Expected $stdoutLines to match $expected.");
+
+ // Remove excess trailing whitespace.
+ for (var i = 0; i < stdoutLines.length; i++) {
+ var event = JSON.decode(stdoutLines[i]);
+ expect(event.remove("time"), new isInstanceOf<int>());
+ expect(event, equals(expected[i]));
+ }
+ });
+}
+
+Map _testStart(int id, String name, {skip}) {
+ var metadata;
+ if (skip == true) {
+ metadata = {"skip": true, "skipReason": null};
+ } else if (skip is String) {
+ metadata = {"skip": true, "skipReason": skip};
+ } else {
+ metadata = {"skip": false, "skipReason": null};
+ }
+
+ return {
+ "type": "testStart",
+ "test": {"id": id, "name": name, "metadata": metadata}
+ };
+}
+
+Map _testDone(int id, {String result, bool hidden: false}) {
+ result ??= "success";
+ return {"type": "testDone", "testID": id, "result": result, "hidden": hidden};
+}
+
+Map _done({bool success: true}) => {"type": "done", "success": success};
« json_reporter.schema.json ('K') | « test/runner/expanded_reporter_test.dart ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698