Add a JSON reporter.

test/runner/json_reporter_test.dart

+// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")

[kevmoo, 2015/11/21 03:37:17]

several type errors, failures etc...you're on this, right?

[nweiz, 2015/11/23 22:24:08]

Yes.

+
+import 'dart:convert';
+
+import 'package:path/path.dart' as p;
+import 'package:scheduled_test/descriptor.dart' as d;
+import 'package:scheduled_test/scheduled_process.dart';
+import 'package:scheduled_test/scheduled_stream.dart';
+import 'package:scheduled_test/scheduled_test.dart';
+
+import 'package:test/src/runner/version.dart';
+
+import '../io.dart';
+
+final _start = {
+  "type": "start",
+  "protocolVersion": "0.1.0",
+  "runnerVersion": testVersion
+};
+
+void main() {
+  useSandbox();
+
+  test("runs several successful tests and reports when each completes", () {
+    _expectReport(
+        """
+        test('success 1', () {});
+        test('success 2', () {});
+        test('success 3', () {});""",
+        [
+          _start,
+          _testStart(0, "loading test.dart"),
+          _testDone(0, hidden: true),
+          _testStart(1, "success 1"),
+          _testDone(1),
+          _testStart(2, "success 2"),
+          _testDone(2),
+          _testStart(3, "success 3"),
+          _testDone(3),
+          _done()
+        ]);
+  });
+
+  test("runs several failing tests and reports when each fails", () {
+    _expectReport(
+        """
+        test('failure 1', () => throw new TestFailure('oh no'));
+        test('failure 2', () => throw new TestFailure('oh no'));
+        test('failure 3', () => throw new TestFailure('oh no'));""",
+        """
+        {"type":"enter","name":"enter","time":\\d+}
+        {"type":"start","name":"failure 1","time":\\d+}
+        {"type":"fail","name":"failure 1","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 6:33  main.<fn>\\\\n"}
+        {"type":"start","name":"failure 2","time":\\d+}
+        {"type":"fail","name":"failure 2","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 7:33  main.<fn>\\\\n"}
+        {"type":"start","name":"failure 3","time":\\d+}
+        {"type":"fail","name":"failure 3","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 8:33  main.<fn>\\\\n"}
+        {"type":"exit","name":"exit","time":\\d+}""");
+  });
+
+  test("includes the full stack trace with --verbose-trace", () {
+    d
+        .file(
+            "test.dart",
+            """
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+  test("failure", () => throw "oh no");
+}
+""")
+        .create();
+
+    var test = runTest(["--verbose-trace", "test.dart"], compact: true);
+    test.stdout.expect(consumeThrough(contains("dart:isolate-patch")));
+    test.shouldExit(1);
+  });
+
+  test("runs failing tests along with successful tests", () {
+    _expectReport(
+        """
+        test('failure 1', () => throw new TestFailure('oh no'));
+        test('success 1', () {});
+        test('failure 2', () => throw new TestFailure('oh no'));
+        test('success 2', () {});""",
+        """
+        {"type":"enter","name":"enter","time":\\d+}
+        {"type":"start","name":"failure 1","time":\\d+}
+        {"type":"fail","name":"failure 1","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 6:33  main.<fn>\\\\n"}
+        {"type":"start","name":"success 1","time":\\d+}
+        {"type":"pass","name":"success 1","time":\\d+}
+        {"type":"start","name":"failure 2","time":\\d+}
+        {"type":"fail","name":"failure 2","time":\\d+,"failMessage":"oh no","sta
+ckTrace":"test.dart 8:33  main.<fn>\\\\n"}
+        {"type":"start","name":"success 2","time":\\d+}
+        {"type":"pass","name":"success 2","time":\\d+}
+        {"type":"exit","name":"exit","time":\\d+}
+        """);
+  });
+
+  test("always prints the full test name", () {
+    _expectReport(
+        """
+        test(
+           'really gosh dang long test name. Even longer than that. No, yet '
+               'longer. A little more... okay, that should do it.',
+           () {});""",
+        """
+        {"type":"enter","name":"enter","time":\\d+}
+        {"type":"start","name":"really gosh dang long test name. Even longer tha
+n that. No, yet longer. A little more... okay, that should do it.","time":\\d+}
+        {"type":"pass","name":"really gosh dang long test name. Even longer than
+ that. No, yet longer. A little more... okay, that should do it.","time":\\d+}
+        {"type":"exit","name":"exit","time":\\d+}
+        """);
+  });
+
+  test("gracefully handles multiple test failures in a row", () {
+    _expectReport(
+        """
+        // This completer ensures that the test isolate isn't killed until all
+        // errors have been thrown.
+        var completer = new Completer();
+        test('failures', () {
+          new Future.microtask(() => throw 'first error');
+          new Future.microtask(() => throw 'second error');
+          new Future.microtask(() => throw 'third error');
+          new Future.microtask(completer.complete);
+        });
+        test('wait', () => completer.future);""",
+        """
+        {"type":"enter","name":"enter","time":\\d+}
+        {"type":"start","name":"failures","time":\\d+}
+        {"type":"error","name":"failures","time":\\d+,"errorMessage":"first erro
+r","stackTrace":"test.dart 10:38  main.<fn>.<fn>\\\\n===== asynchronous gap ====
+=======================\\\\ndart:async       Future.Future.microtask\\\\ntest.da
+rt 10:15  main.<fn>\\\\n"}
+        {"type":"error","name":"failures","time":\\d+,"errorMessage":"second err
+or","stackTrace":"test.dart 11:38  main.<fn>.<fn>\\\\n===== asynchronous gap ===
+========================\\\\ndart:async       Future.Future.microtask\\\\ntest.d
+art 11:15  main.<fn>\\\\n"}
+        {"type":"error","name":"failures","time":\\d+,"errorMessage":"third erro
+r","stackTrace":"test.dart 12:38  main.<fn>.<fn>\\\\n===== asynchronous gap ====
+=======================\\\\ndart:async       Future.Future.microtask\\\\ntest.da
+rt 12:15  main.<fn>\\\\n"}
+        {"type":"start","name":"wait","time":\\d+}
+        {"type":"pass","name":"wait","time":\\d+}
+        {"type":"exit","name":"exit","time":\\d+}
+        """);
+  });
+
+  group("print:", () {
+    test("handles multiple prints", () {
+      _expectReport(
+          """
+        test('test', () {
+          print("one");
+          print("two");
+          print("three");
+          print("four");
+        });""",
+          """
+        {"type":"enter","name":"enter","time":\\d+}
+        {"type":"start","name":"test","time":\\d+}
+        {"type":"print","name":"test","time":\\d+,"message":"one"}
+        {"type":"print","name":"test","time":\\d+,"message":"two"}
+        {"type":"print","name":"test","time":\\d+,"message":"three"}
+        {"type":"print","name":"test","time":\\d+,"message":"four"}
+        {"type":"pass","name":"test","time":\\d+}
+        {"type":"exit","name":"exit","time":\\d+}
+        """);
+    });
+
+    test("handles a print after the test completes", () {
+      _expectReport(
+          """
+        // This completer ensures that the test isolate isn't killed until all
+        // prints have happened.
+        var testDone = new Completer();
+        var waitStarted = new Completer();
+        test('test', () async {
+          waitStarted.future.then((_) {
+            new Future(() => print("one"));
+            new Future(() => print("two"));
+            new Future(() => print("three"));
+            new Future(() => print("four"));
+            new Future(testDone.complete);
+          });
+        });
+
+        test('wait', () {
+          waitStarted.complete();
+          return testDone.future;
+        });""",
+          """
+        {"type":"enter","name":"enter","time":\\d+}
+        {"type":"start","name":"test","time":\\d+}
+        {"type":"pass","name":"test","time":\\d+}
+        {"type":"start","name":"wait","time":\\d+}
+        {"type":"print","name":"test","time":\\d+,"message":"one"}
+        {"type":"print","name":"test","time":\\d+,"message":"two"}
+        {"type":"print","name":"test","time":\\d+,"message":"three"}
+        {"type":"print","name":"test","time":\\d+,"message":"four"}
+        {"type":"pass","name":"wait","time":\\d+}
+        {"type":"exit","name":"exit","time":\\d+}
+        """);
+    });
+
+    test("interleaves prints and errors", () {
+      _expectReport(
+          """
+        // This completer ensures that the test isolate isn't killed until all
+        // prints have happened.
+        var completer = new Completer();
+        test('test', () {
+          scheduleMicrotask(() {
+            print("three");
+            print("four");
+            throw "second error";
+          });
+
+          scheduleMicrotask(() {
+            print("five");
+            print("six");
+            completer.complete();
+          });
+
+          print("one");
+          print("two");
+          throw "first error";
+        });
+
+        test('wait', () => completer.future);""",
+          """
+        {"type":"enter","name":"enter","time":\\d+}
+        {"type":"start","name":"test","time":\\d+}
+        {"type":"print","name":"test","time":\\d+,"message":"one"}
+        {"type":"print","name":"test","time":\\d+,"message":"two"}
+        {"type":"error","name":"test","time":\\d+,"errorMessage":"first error","
+stackTrace":"test.dart 24:11  main.<fn>\\\\n"}
+        {"type":"print","name":"test","time":\\d+,"message":"three"}
+        {"type":"print","name":"test","time":\\d+,"message":"four"}
+        {"type":"error","name":"test","time":\\d+,"errorMessage":"second error",
+"stackTrace":"test.dart 13:13  main.<fn>.<fn>\\\\n===== asynchronous gap =======
+====================\\\\ndart:async       scheduleMicrotask\\\\ntest.dart 10:11 
+ main.<fn>\\\\n"}
+        {"type":"print","name":"test","time":\\d+,"message":"five"}
+        {"type":"print","name":"test","time":\\d+,"message":"six"}
+        {"type":"start","name":"wait","time":\\d+}
+        {"type":"pass","name":"wait","time":\\d+}
+        {"type":"exit","name":"exit","time":\\d+}
+        """);
+    });
+  });
+
+  group("skip:", () {
+    test("displays skipped tests separately", () {
+      _expectReport(
+          """
+          test('skip 1', () {}, skip: true);
+          test('skip 2', () {}, skip: true);
+          test('skip 3', () {}, skip: true);""",
+          """
+          {"type":"enter","name":"enter","time":\\d+}
+          {"type":"start","name":"skip 1","time":\\d+}
+          {"type":"skip","name":"skip 1","time":\\d+}
+          {"type":"start","name":"skip 2","time":\\d+}
+          {"type":"skip","name":"skip 2","time":\\d+}
+          {"type":"start","name":"skip 3","time":\\d+}
+          {"type":"skip","name":"skip 3","time":\\d+}
+          {"type":"exit","name":"exit","time":\\d+}
+          """);
+    });
+
+    test("runs skipped tests along with successful tests", () {
+      _expectReport(
+          """
+          test('skip 1', () {}, skip: true);
+          test('success 1', () {});
+          test('skip 2', () {}, skip: true);
+          test('success 2', () {});""",
+          """
+          {"type":"enter","name":"enter","time":\\d+}
+          {"type":"start","name":"skip 1","time":\\d+}
+          {"type":"skip","name":"skip 1","time":\\d+}
+          {"type":"start","name":"success 1","time":\\d+}
+          {"type":"pass","name":"success 1","time":\\d+}
+          {"type":"start","name":"skip 2","time":\\d+}
+          {"type":"skip","name":"skip 2","time":\\d+}
+          {"type":"start","name":"success 2","time":\\d+}
+          {"type":"pass","name":"success 2","time":\\d+}
+          {"type":"exit","name":"exit","time":\\d+}
+          """);
+    });
+
+    test("runs skipped tests along with successful and failing tests", () {
+      _expectReport(
+          """
+          test('failure 1', () => throw new TestFailure('oh no'));
+          test('skip 1', () {}, skip: true);
+          test('success 1', () {});
+          test('failure 2', () => throw new TestFailure('oh no'));
+          test('skip 2', () {}, skip: true);
+          test('success 2', () {});""",
+          """
+          {"type":"enter","name":"enter","time":\\d+}
+          {"type":"start","name":"failure 1","time":\\d+}
+          {"type":"fail","name":"failure 1","time":\\d+,"failMessage":"oh no","s
+tackTrace":"test.dart 6:35  main.<fn>\\\\n"}
+          {"type":"start","name":"skip 1","time":\\d+}
+          {"type":"skip","name":"skip 1","time":1\\d+}
+          {"type":"start","name":"success 1","time":\\d+}
+          {"type":"pass","name":"success 1","time":\\d+}
+          {"type":"start","name":"failure 2","time":\\d+}
+          {"type":"fail","name":"failure 2","time":\\d+,"failMessage":"oh no","s
+tackTrace":"test.dart 9:35  main.<fn>\\\\n"}
+          {"type":"start","name":"skip 2","time":\\d+}
+          {"type":"skip","name":"skip 2","time":\\d+}
+          {"type":"start","name":"success 2","time":\\d+}
+          {"type":"pass","name":"success 2","time":\\d+}
+          {"type":"exit","name":"exit","time":\\d+}
+          """);
+    });
+
+    test("displays the skip reason if available", () {
+      _expectReport(
+          """
+          test('skip 1', () {}, skip: 'some reason');
+          test('skip 2', () {}, skip: 'or another');""",
+          """
+          {"type":"enter","name":"enter","time":\\d+}
+          {"type":"start","name":"skip 1","time":\\d+}
+          {"type":"skip","name":"skip 1","time":\\d+,"reason":"some reason"}
+          {"type":"start","name":"skip 2","time":\\d+}
+          {"type":"skip","name":"skip 2","time":\\d+,"reason":"or another"}
+          {"type":"exit","name":"exit","time":\\d+}
+          """);
+    });
+  });
+}
+
+void _expectReport(String tests, List<Map> expected) {
+  var dart = """
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+$tests
+}
+""";
+
+  d.file("test.dart", dart).create();
+
+  var test = runTest(["test.dart"], reporter: "json");
+  test.shouldExit();
+
+  schedule(() async {
+    var stdoutLines = await test.stdoutStream().toList();
+
+    expect(stdoutLines.length, equals(expected.length),
+        reason: "Expected $stdoutLines to match $expected.");
+
+    // Remove excess trailing whitespace.
+    for (var i = 0; i < stdoutLines.length; i++) {
+      var event = JSON.decode(stdoutLines[i]);
+      expect(event.remove("time"), new isInstanceOf<int>());
+      expect(event, equals(expected[i]));
+    }
+  });
+}
+
+Map _testStart(int id, String name, {skip}) {
+  var metadata;
+  if (skip == true) {
+    metadata = {"skip": true, "skipReason": null};
+  } else if (skip is String) {
+    metadata = {"skip": true, "skipReason": skip};
+  } else {
+    metadata = {"skip": false, "skipReason": null};
+  }
+
+  return {
+    "type": "testStart",
+    "test": {"id": id, "name": name, "metadata": metadata}
+  };
+}
+
+Map _testDone(int id, {String result, bool hidden: false}) {
+  result ??= "success";
+  return {"type": "testDone", "testID": id, "result": result, "hidden": hidden};
+}
+
+Map _done({bool success: true}) => {"type": "done", "success": success};