OLD | NEW |
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// A test library for testing test libraries? We must go deeper. | 5 /// A test library for testing test libraries? We must go deeper. |
6 /// | 6 /// |
7 /// Since unit testing code tends to use a lot of global state, it can be tough | 7 /// Since unit testing code tends to use a lot of global state, it can be tough |
8 /// to test. This library manages it by running each test case in a child | 8 /// to test. This library manages it by running each test case in a child |
9 /// isolate, then reporting the results back to the parent isolate. | 9 /// isolate, then reporting the results back to the parent isolate. |
10 library metatest; | 10 library metatest; |
11 | 11 |
12 import 'dart:async'; | 12 import 'dart:async'; |
13 import 'dart:isolate'; | 13 import 'dart:isolate'; |
14 | 14 |
15 import 'package:unittest/unittest.dart'; | 15 // TODO(nweiz): Stop importing from src when dart-lang/test#48 is fixed. |
| 16 import 'package:test/src/backend/declarer.dart'; |
| 17 import 'package:test/src/backend/state.dart'; |
| 18 import 'package:test/src/backend/suite.dart'; |
| 19 import 'package:test/src/runner/engine.dart'; |
| 20 import 'package:test/test.dart'; |
16 | 21 |
17 import 'src/utils.dart'; | 22 /// Declares a test with the given [description] and [body]. |
| 23 /// |
| 24 /// [body] corresponds to the `main` method of a test file. By default, this |
| 25 /// expects that all tests defined in [body] pass, but if [passing] is passed, |
| 26 /// only tests listed there are expected to pass. |
| 27 void expectTestsPass(String description, void body(), {List<String> passing}) { |
| 28 _setUpTest(description, body, (liveTests) { |
| 29 if (passing == null) { |
| 30 if (liveTests.any( |
| 31 (liveTest) => liveTest.state.result != Result.success)) { |
| 32 fail('Expected all tests to pass, but some failed:\n' |
| 33 '${_summarizeTests(liveTests)}'); |
| 34 } |
| 35 return; |
| 36 } |
18 | 37 |
19 /// Whether or not we're running in a child isolate that's supposed to run a | 38 var shouldPass = new Set.from(passing); |
20 /// test. | 39 var didPass = new Set.from(liveTests |
21 bool _inChildIsolate; | 40 .where((liveTest) => liveTest.state.result == Result.success) |
| 41 .map((liveTest) => liveTest.test.name)); |
22 | 42 |
23 /// The port with which the child isolate should communicate with the parent | 43 if (!shouldPass.containsAll(didPass) || |
24 /// isolate. | 44 !didPass.containsAll(shouldPass)) { |
25 /// | 45 stringify(tests) => '{${tests.map((t) => '"$t"').join(', ')}}'; |
26 /// `null` in the parent isolate. | |
27 SendPort _replyTo; | |
28 | 46 |
29 /// The only value of the configuration used in metatest. | 47 fail('Expected exactly ${stringify(shouldPass)} to pass, but ' |
30 final _metaConfiguration = new _MetaConfiguration(); | 48 '${stringify(didPass)} passed.\n' |
31 | 49 '${_summarizeTests(liveTests)}'); |
32 /// The function holding the tests to be run. | |
33 Function _testBody; | |
34 | |
35 /// The description of the test to run in the child isolate. | |
36 /// | |
37 /// `null` in the parent isolate. | |
38 String _testToRun; | |
39 | |
40 /// Stores the optional timeout used to override the default unittest timeout. | |
41 Duration _timeoutOverride; | |
42 | |
43 /// Runs [setUpFn] before every metatest. | |
44 /// | |
45 /// Note that [setUpFn] will be overwritten if the test itself calls [setUp]. | |
46 void metaSetUp(void setUpFn()) { | |
47 if (_inChildIsolate) setUp(setUpFn); | |
48 } | |
49 | |
50 /// Runs a set of tests defined in `body` and checks the result by comparing | |
51 /// with values in `expectedResults`. | |
52 /// | |
53 /// [expectedResults] is a list which should have a [Map] value for each test | |
54 /// that is run. Each [Map] key corresponds to values from a completed test | |
55 /// case: "description", "message", "result", and "stackTrace". | |
56 /// | |
57 /// The value of "result" can be one of: 'pass', 'fail', or 'error'. | |
58 /// | |
59 /// The value for "stackTrace" is the [String] 'null' if the property is `null` | |
60 /// on the source test case. Otherwise, it is the output of `toString`. The | |
61 /// format is not guaranteed. | |
62 /// | |
63 /// Here's an example of a `expectedResults` value for two tests, where the | |
64 /// where the first fails and the second passes. | |
65 /// | |
66 /// ```dart | |
67 /// [{ | |
68 /// 'description': 'test', | |
69 /// 'message': 'Caught error!', | |
70 /// 'result': 'fail', | |
71 /// }, { | |
72 /// 'description': 'follow up', | |
73 /// 'result': 'pass', | |
74 /// }] | |
75 /// ``` | |
76 void expectTestResults(String description, void body(), | |
77 List<Map> expectedResults) { | |
78 _setUpTest(description, body, (resultsMap) { | |
79 var list = resultsMap['results']; | |
80 expect(list, hasLength(expectedResults.length), | |
81 reason: 'The number of tests run does not match the number of expected' | |
82 ' results.'); | |
83 | |
84 for (var i = 0; i < list.length; i++) { | |
85 var expectedMap = expectedResults[i]; | |
86 var map = list[i]; | |
87 | |
88 expectedMap.forEach((key, value) { | |
89 expect(map, containsPair(key, value), reason: 'A test did not match the' | |
90 ' expected value for "$key" at index $i.'); | |
91 }); | |
92 } | 50 } |
93 }); | 51 }); |
94 } | 52 } |
95 | 53 |
96 /// Declares a test with the given [description] and [body]. [body] corresponds | 54 /// Asserts that all tests defined by [body] fail. |
97 /// to the `main` method of a test file, and will be run in an isolate. By | 55 /// |
98 /// default, this expects that all tests defined in [body] pass, but if | 56 /// [body] corresponds to the `main` method of a test file. |
99 /// [passing] is passed, only tests listed there are expected to pass. | 57 void expectTestsFail(String description, body()) { |
100 void expectTestsPass(String description, void body(), {List<String> passing}) { | 58 expectTestsPass(description, body, passing: []); |
101 _setUpTest(description, body, (results) { | |
102 if (_hasError(results)) { | |
103 fail('Expected all tests to pass, but got error(s):\n' | |
104 '${_summarizeTests(results)}'); | |
105 } else if (passing == null) { | |
106 if (results['failed'] != 0) { | |
107 fail('Expected all tests to pass, but some failed:\n' | |
108 '${_summarizeTests(results)}'); | |
109 } | |
110 } else { | |
111 var shouldPass = new Set.from(passing); | |
112 var didPass = new Set.from(results['results'] | |
113 .where((t) => t['result'] == 'pass') | |
114 .map((t) => t['description'])); | |
115 | |
116 if (!shouldPass.containsAll(didPass) || | |
117 !didPass.containsAll(shouldPass)) { | |
118 String stringify(Set<String> tests) => | |
119 '{${tests.map((t) => '"$t"').join(', ')}}'; | |
120 | |
121 fail('Expected exactly ${stringify(shouldPass)} to pass, but ' | |
122 '${stringify(didPass)} passed.\n' | |
123 '${_summarizeTests(results)}'); | |
124 } | |
125 } | |
126 }); | |
127 } | |
128 | |
129 /// Declares a test with the given [description] and [body]. [body] corresponds | |
130 /// to the `main` method of a test file, and will be run in an isolate. Expects | |
131 /// all tests defined by [body] to fail. | |
132 void expectTestsFail(String description, void body()) { | |
133 _setUpTest(description, body, (results) { | |
134 if (_hasError(results)) { | |
135 throw 'Expected all tests to fail, but got error(s):\n' | |
136 '${_summarizeTests(results)}'; | |
137 } else if (results['passed'] != 0) { | |
138 throw 'Expected all tests to fail, but some passed:\n' | |
139 '${_summarizeTests(results)}'; | |
140 } | |
141 }); | |
142 } | 59 } |
143 | 60 |
144 /// Sets up a test with the given [description] and [body]. After the test runs, | 61 /// Sets up a test with the given [description] and [body]. After the test runs, |
145 /// calls [validate] with the result map. | 62 /// calls [validate] with the result map. |
146 void _setUpTest(String description, void body(), void validate(Map map)) { | 63 void _setUpTest(String description, void body(), |
147 if (_inChildIsolate) { | 64 void validate(List<LiveTest> liveTests)) { |
148 _ensureInitialized(); | 65 test(description, () async { |
149 if (_testToRun == description) body(); | 66 var declarer = new Declarer(); |
150 } else { | 67 runZoned(body, zoneValues: {#test.declarer: declarer}); |
151 test(description, () { | 68 |
152 return _runInIsolate(description).then(validate); | 69 var engine = new Engine([new Suite(declarer.tests)]); |
153 }); | 70 for (var test in engine.liveTests) { |
154 } | 71 test.onPrint.listen(print); |
| 72 } |
| 73 await engine.run(); |
| 74 |
| 75 validate(engine.liveTests); |
| 76 }); |
155 } | 77 } |
156 | 78 |
157 /// Initialize metatest. | 79 /// Returns a string description of the test run descibed by [liveTests]. |
158 /// | 80 String _summarizeTests(List<LiveTest> liveTests) { |
159 /// [message] should be the second argument to [main]. It's used to determine | |
160 /// whether this test is in the parent isolate or a child isolate. | |
161 /// | |
162 /// [timeout], when specified, overrides the default timeout for unittest. | |
163 void initMetatest(message, {Duration timeout}) { | |
164 _timeoutOverride = timeout; | |
165 if (message == null) { | |
166 _inChildIsolate = false; | |
167 } else { | |
168 _testToRun = message['testToRun']; | |
169 _replyTo = message['replyTo']; | |
170 _inChildIsolate = true; | |
171 } | |
172 } | |
173 | |
174 // TODO(kevmoo) We need to capture the main method to allow running in an | |
175 // isolate. There is no mechanism to capture the current executing URI between | |
176 // browser and vm. Issue 1145 and/or Issue 8440 | |
177 void initTests(void testBody(message)) { | |
178 _testBody = testBody; | |
179 _testBody(null); | |
180 } | |
181 | |
182 /// Runs the test described by [description] in its own isolate. | |
183 /// | |
184 /// Returns a map describing the results of that test run. | |
185 Future<Map> _runInIsolate(String description) { | |
186 if (_testBody == null) { | |
187 throw new StateError('initTests was not called.'); | |
188 } | |
189 | |
190 var replyPort = new ReceivePort(); | |
191 return Isolate.spawn(_testBody, { | |
192 'testToRun': description, | |
193 'replyTo': replyPort.sendPort | |
194 }).then((_) => replyPort.first); | |
195 } | |
196 | |
197 /// Returns whether [results] (a test result map) describes a test run in which | |
198 /// an error occurred. | |
199 bool _hasError(Map results) { | |
200 return results['errors'] > 0 || results['uncaughtError'] != null || | |
201 (results['passed'] == 0 && results['failed'] == 0); | |
202 } | |
203 | |
204 /// Returns a string description of the test run descibed by [results]. | |
205 String _summarizeTests(Map results) { | |
206 var buffer = new StringBuffer(); | 81 var buffer = new StringBuffer(); |
207 for (var t in results["results"]) { | 82 for (var liveTest in liveTests) { |
208 buffer.writeln("${t['result'].toUpperCase()}: ${t['description']}"); | 83 buffer.writeln("${liveTest.state.result}: ${liveTest.test.name}"); |
209 if (t['message'] != '') buffer.writeln("${_indent(t['message'])}"); | 84 for (var error in liveTest.errors) { |
210 if (t['stackTrace'] != null && t['stackTrace'] != '') { | 85 buffer.writeln(error.error); |
211 buffer.writeln("${_indent(t['stackTrace'])}"); | 86 if (error.stackTrace != null) buffer.writeln(error.stackTrace); |
212 } | 87 } |
213 } | 88 } |
214 | 89 return buffer.toString(); |
215 buffer.writeln(); | |
216 | |
217 if (results['passed'] == 0 && results['failed'] == 0 && | |
218 results['errors'] == 0 && results['uncaughtError'] == null) { | |
219 buffer.write('No tests found.'); | |
220 // This is considered a failure too. | |
221 } else if (results['failed'] == 0 && results['errors'] == 0 && | |
222 results['uncaughtError'] == null) { | |
223 buffer.write('All ${results['passed']} tests passed.'); | |
224 } else { | |
225 if (results['uncaughtError'] != null) { | |
226 buffer.write('Top-level uncaught error: ${results['uncaughtError']}'); | |
227 } | |
228 buffer.write('${results['passed']} PASSED, ${results['failed']} FAILED, ' | |
229 '${results['errors']} ERRORS'); | |
230 } | |
231 return prefixLines(buffer.toString()); | |
232 } | 90 } |
233 | |
234 /// Indents each line of [str] by two spaces. | |
235 String _indent(String str) { | |
236 return str.replaceAll(new RegExp("^", multiLine: true), " "); | |
237 } | |
238 | |
239 /// Ensure that the metatest configuration is loaded. | |
240 void _ensureInitialized() { | |
241 unittestConfiguration = _metaConfiguration; | |
242 if (_timeoutOverride != null) { | |
243 unittestConfiguration.timeout = _timeoutOverride; | |
244 } | |
245 } | |
246 | |
247 /// Special test configuration for use within the child isolates. This hides all | |
248 /// output and reports data back to the parent isolate. | |
249 class _MetaConfiguration extends Configuration { | |
250 | |
251 _MetaConfiguration() : super.blank(); | |
252 | |
253 void onSummary(int passed, int failed, int errors, List<TestCase> results, | |
254 String uncaughtError) { | |
255 _replyTo.send({ | |
256 "passed": passed, | |
257 "failed": failed, | |
258 "errors": errors, | |
259 "uncaughtError": uncaughtError, | |
260 "results": results.map((testCase) => { | |
261 "description": testCase.description, | |
262 "message": testCase.message, | |
263 "result": testCase.result, | |
264 "stackTrace": testCase.stackTrace.toString() | |
265 }).toList() | |
266 }); | |
267 } | |
268 } | |
OLD | NEW |