OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// A test library for testing test libraries? We must go deeper. | 5 /// A test library for testing test libraries? We must go deeper. |
6 /// | 6 /// |
7 /// Since unit testing code tends to use a lot of global state, it can be tough | 7 /// Since unit testing code tends to use a lot of global state, it can be tough |
8 /// to test. This library manages it by running each test case in a child | 8 /// to test. This library manages it by running each test case in a child |
9 /// isolate, then reporting the results back to the parent isolate. | 9 /// isolate, then reporting the results back to the parent isolate. |
10 library metatest; | 10 library metatest; |
11 | 11 |
12 import 'dart:async'; | 12 import 'dart:async'; |
13 import 'dart:io'; | |
14 import 'dart:isolate'; | 13 import 'dart:isolate'; |
15 | 14 |
16 import 'package:path/path.dart' as path; | |
17 import 'package:unittest/unittest.dart'; | 15 import 'package:unittest/unittest.dart'; |
18 import 'package:scheduled_test/scheduled_test.dart' as scheduled_test; | |
19 | 16 |
20 import 'utils.dart'; | 17 import 'src/utils.dart'; |
| 18 |
| 19 /// Whether or not we're running in a child isolate that's supposed to run a |
| 20 /// test. |
| 21 bool _inChildIsolate; |
| 22 |
| 23 /// The port with which the child isolate should communicate with the parent |
| 24 /// isolate. |
| 25 /// |
| 26 /// `null` in the parent isolate. |
| 27 SendPort _replyTo; |
| 28 |
| 29 /// The only value of the configuration used in metatest. |
| 30 final _metaConfiguration = new _MetaConfiguration(); |
| 31 |
| 32 /// The function holding the tests to be run. |
| 33 Function _testBody; |
| 34 |
| 35 /// The description of the test to run in the child isolate. |
| 36 /// |
| 37 /// `null` in the parent isolate. |
| 38 String _testToRun; |
| 39 |
| 40 /// Stores the optional timeout used to override the default unittest timeout. |
| 41 Duration _timeoutOverride; |
| 42 |
| 43 /// Runs [setUpFn] before every metatest. |
| 44 /// |
| 45 /// Note that [setUpFn] will be overwritten if the test itself calls [setUp]. |
| 46 void metaSetUp(void setUpFn()) { |
| 47 if (_inChildIsolate) setUp(setUpFn); |
| 48 } |
| 49 |
| 50 /// Runs a set of tests defined in `body` and checks the result by comparing |
| 51 /// with values in `expectedResults`. |
| 52 /// |
| 53 /// [expectedResults] is a list which should have a [Map] value for each test |
| 54 /// that is run. Each [Map] key corresponds to values from a completed test |
| 55 /// case: "description", "message", "result", and "stackTrace". |
| 56 /// |
| 57 /// The value of "result" can be one of: 'pass', 'fail', or 'error'. |
| 58 /// |
| 59 /// The value for "stackTrace" is the [String] 'null' if the property is `null` |
| 60 /// on the source test case. Otherwise, it is the output of `toString`. The |
| 61 /// format is not guaranteed. |
| 62 /// |
| 63 /// Here's an example of a `expectedResults` value for two tests, where the |
| 64 /// where the first fails and the second passes. |
| 65 /// |
| 66 /// ```dart |
| 67 /// [{ |
| 68 /// 'description': 'test', |
| 69 /// 'message': 'Caught error!', |
| 70 /// 'result': 'fail', |
| 71 /// }, { |
| 72 /// 'description': 'follow up', |
| 73 /// 'result': 'pass', |
| 74 /// }] |
| 75 /// ``` |
| 76 void expectTestResults(String description, void body(), |
| 77 List<Map> expectedResults) { |
| 78 _setUpTest(description, body, (resultsMap) { |
| 79 var list = resultsMap['results']; |
| 80 expect(list, hasLength(expectedResults.length), |
| 81 reason: 'The number of tests run does not match the number of expected' |
| 82 ' results.'); |
| 83 |
| 84 for (var i = 0; i < list.length; i++) { |
| 85 var expectedMap = expectedResults[i]; |
| 86 var map = list[i]; |
| 87 |
| 88 expectedMap.forEach((key, value) { |
| 89 expect(map, containsPair(key, value), reason: 'A test did not match the' |
| 90 ' expected value for "$key" at index $i.'); |
| 91 }); |
| 92 } |
| 93 }); |
| 94 } |
21 | 95 |
22 /// Declares a test with the given [description] and [body]. [body] corresponds | 96 /// Declares a test with the given [description] and [body]. [body] corresponds |
23 /// to the `main` method of a test file, and will be run in an isolate. By | 97 /// to the `main` method of a test file, and will be run in an isolate. By |
24 /// default, this expects that all tests defined in [body] pass, but if | 98 /// default, this expects that all tests defined in [body] pass, but if |
25 /// [passing] is passed, only tests listed there are expected to pass. | 99 /// [passing] is passed, only tests listed there are expected to pass. |
26 void expectTestsPass(String description, void body(), {List<String> passing}) { | 100 void expectTestsPass(String description, void body(), {List<String> passing}) { |
27 _setUpTest(description, body, (results) { | 101 _setUpTest(description, body, (results) { |
28 if (_hasError(results)) { | 102 if (_hasError(results)) { |
29 throw 'Expected all tests to pass, but got error(s):\n' | 103 fail('Expected all tests to pass, but got error(s):\n' |
30 '${_summarizeTests(results)}'; | 104 '${_summarizeTests(results)}'); |
31 } else if (passing == null) { | 105 } else if (passing == null) { |
32 if (results['failed'] != 0) { | 106 if (results['failed'] != 0) { |
33 throw 'Expected all tests to pass, but some failed:\n' | 107 fail('Expected all tests to pass, but some failed:\n' |
34 '${_summarizeTests(results)}'; | 108 '${_summarizeTests(results)}'); |
35 } | 109 } |
36 } else { | 110 } else { |
37 var shouldPass = new Set.from(passing); | 111 var shouldPass = new Set.from(passing); |
38 var didPass = new Set.from(results['results'] | 112 var didPass = new Set.from(results['results'] |
39 .where((t) => t['result'] == 'pass') | 113 .where((t) => t['result'] == 'pass') |
40 .map((t) => t['description'])); | 114 .map((t) => t['description'])); |
41 | 115 |
42 if (!shouldPass.containsAll(didPass) || | 116 if (!shouldPass.containsAll(didPass) || |
43 !didPass.containsAll(shouldPass)) { | 117 !didPass.containsAll(shouldPass)) { |
44 String stringify(Set<String> tests) => | 118 String stringify(Set<String> tests) => |
45 '{${tests.map((t) => '"$t"').join(', ')}}'; | 119 '{${tests.map((t) => '"$t"').join(', ')}}'; |
46 | 120 |
47 fail('Expected exactly ${stringify(shouldPass)} to pass, but ' | 121 fail('Expected exactly ${stringify(shouldPass)} to pass, but ' |
48 '${stringify(didPass)} passed.\n' | 122 '${stringify(didPass)} passed.\n' |
49 '${_summarizeTests(results)}'); | 123 '${_summarizeTests(results)}'); |
50 } | 124 } |
51 } | 125 } |
52 }); | 126 }); |
53 } | 127 } |
54 | 128 |
55 /// Declares a test with the given [description] and [body]. | |
56 /// | |
57 /// [body] corresponds | |
58 /// to the `main` method of a test file, and will be run in an isolate. | |
59 /// | |
60 /// All tests must have an expected result in [expectedResults]. | |
61 void expectTests(String description, void body(), | |
62 Map<String, String> expectedResults) { | |
63 _setUpTest(description, body, (results) { | |
64 expectedResults = new Map.from(expectedResults); | |
65 | |
66 for (var testResult in results['results']) { | |
67 var description = testResult['description']; | |
68 | |
69 expect(expectedResults, contains(description), | |
70 reason: '"$description" did not have an expected result set.\n' | |
71 '${_summarizeTests(results)}'); | |
72 | |
73 var result = testResult['result']; | |
74 | |
75 expect(expectedResults, containsPair(description, result), | |
76 reason: 'The test "$description" not not have the expected result.\n' | |
77 '${_summarizeTests(results)}'); | |
78 | |
79 expectedResults.remove(description); | |
80 } | |
81 expect(expectedResults, isEmpty, | |
82 reason: 'Unexpected additional test results\n' | |
83 '${_summarizeTests(results)}'); | |
84 }); | |
85 } | |
86 | |
87 /// Declares a test with the given [description] and [body]. [body] corresponds | 129 /// Declares a test with the given [description] and [body]. [body] corresponds |
88 /// to the `main` method of a test file, and will be run in an isolate. Expects | 130 /// to the `main` method of a test file, and will be run in an isolate. Expects |
89 /// all tests defined by [body] to fail. | 131 /// all tests defined by [body] to fail. |
90 void expectTestsFail(String description, void body()) { | 132 void expectTestsFail(String description, void body()) { |
91 _setUpTest(description, body, (results) { | 133 _setUpTest(description, body, (results) { |
92 if (_hasError(results)) { | 134 if (_hasError(results)) { |
93 throw 'Expected all tests to fail, but got error(s):\n' | 135 throw 'Expected all tests to fail, but got error(s):\n' |
94 '${_summarizeTests(results)}'; | 136 '${_summarizeTests(results)}'; |
95 } else if (results['passed'] != 0) { | 137 } else if (results['passed'] != 0) { |
96 throw 'Expected all tests to fail, but some passed:\n' | 138 throw 'Expected all tests to fail, but some passed:\n' |
97 '${_summarizeTests(results)}'; | 139 '${_summarizeTests(results)}'; |
98 } | 140 } |
99 }); | 141 }); |
100 } | 142 } |
101 | 143 |
102 /// Runs [setUpFn] before every metatest. Note that [setUpFn] will be | |
103 /// overwritten if the test itself calls [setUp]. | |
104 void metaSetUp(void setUpFn()) { | |
105 if (_inChildIsolate) scheduled_test.setUp(setUpFn); | |
106 } | |
107 | |
108 /// Sets up a test with the given [description] and [body]. After the test runs, | 144 /// Sets up a test with the given [description] and [body]. After the test runs, |
109 /// calls [validate] with the result map. | 145 /// calls [validate] with the result map. |
110 void _setUpTest(String description, void body(), void validate(Map)) { | 146 void _setUpTest(String description, void body(), void validate(Map map)) { |
111 if (_inChildIsolate) { | 147 if (_inChildIsolate) { |
112 _ensureInitialized(); | 148 _ensureInitialized(); |
113 if (_testToRun == description) body(); | 149 if (_testToRun == description) body(); |
114 } else { | 150 } else { |
115 test(description, () { | 151 test(description, () { |
116 expect(_runInIsolate(description).then(validate), completes); | 152 return _runInIsolate(description).then(validate); |
117 }); | 153 }); |
118 } | 154 } |
119 } | 155 } |
120 | 156 |
121 /// The description of the test to run in the child isolate. | |
122 /// | |
123 /// `null` in the parent isolate. | |
124 String _testToRun; | |
125 | |
126 /// The port with which the child isolate should communicate with the parent | |
127 /// isolate. | |
128 /// | |
129 /// `null` in the parent isolate. | |
130 SendPort _replyTo; | |
131 | |
132 /// Whether or not we're running in a child isolate that's supposed to run a | |
133 /// test. | |
134 bool _inChildIsolate; | |
135 | |
136 /// Initialize metatest. | 157 /// Initialize metatest. |
137 /// | 158 /// |
138 /// [message] should be the second argument to [main]. It's used to determine | 159 /// [message] should be the second argument to [main]. It's used to determine |
139 /// whether this test is in the parent isolate or a child isolate. | 160 /// whether this test is in the parent isolate or a child isolate. |
140 void initMetatest(message) { | 161 /// |
| 162 /// [timeout], when specified, overrides the default timeout for unittest. |
| 163 void initMetatest(message, {Duration timeout}) { |
| 164 _timeoutOverride = timeout; |
141 if (message == null) { | 165 if (message == null) { |
142 _inChildIsolate = false; | 166 _inChildIsolate = false; |
143 } else { | 167 } else { |
144 _testToRun = message['testToRun']; | 168 _testToRun = message['testToRun']; |
145 _replyTo = message['replyTo']; | 169 _replyTo = message['replyTo']; |
146 _inChildIsolate = true; | 170 _inChildIsolate = true; |
147 } | 171 } |
148 } | 172 } |
149 | 173 |
150 /// Runs the test described by [description] in its own isolate. Returns a map | 174 // TODO(kevmoo) We need to capture the main method to allow running in an |
151 /// describing the results of that test run. | 175 // isolate. There is no mechanism to capture the current executing URI between |
| 176 // browser and vm. Issue 1145 and/or Issue 8440 |
| 177 void initTests(void testBody(message)) { |
| 178 _testBody = testBody; |
| 179 _testBody(null); |
| 180 } |
| 181 |
| 182 /// Runs the test described by [description] in its own isolate. |
| 183 /// |
| 184 /// Returns a map describing the results of that test run. |
152 Future<Map> _runInIsolate(String description) { | 185 Future<Map> _runInIsolate(String description) { |
| 186 if (_testBody == null) { |
| 187 throw new StateError('initTests was not called.'); |
| 188 } |
| 189 |
153 var replyPort = new ReceivePort(); | 190 var replyPort = new ReceivePort(); |
154 // TODO(nweiz): Don't use path here once issue 8440 is fixed. | 191 return Isolate.spawn(_testBody, { |
155 var uri = path.toUri(path.absolute(path.fromUri(Platform.script))); | |
156 return Isolate.spawnUri(uri, [], { | |
157 'testToRun': description, | 192 'testToRun': description, |
158 'replyTo': replyPort.sendPort | 193 'replyTo': replyPort.sendPort |
159 }).then((_) { | 194 }).then((_) => replyPort.first); |
160 // TODO(nweiz): Remove this timeout once issue 8875 is fixed and we can | |
161 // capture top-level exceptions. | |
162 return timeout(replyPort.first, 30 * 1000, () { | |
163 throw 'Timed out waiting for test to complete.'; | |
164 }); | |
165 }); | |
166 } | 195 } |
167 | 196 |
168 /// Returns whether [results] (a test result map) describes a test run in which | 197 /// Returns whether [results] (a test result map) describes a test run in which |
169 /// an error occurred. | 198 /// an error occurred. |
170 bool _hasError(Map results) { | 199 bool _hasError(Map results) { |
171 return results['errors'] > 0 || results['uncaughtError'] != null || | 200 return results['errors'] > 0 || results['uncaughtError'] != null || |
172 (results['passed'] == 0 && results['failed'] == 0); | 201 (results['passed'] == 0 && results['failed'] == 0); |
173 } | 202 } |
174 | 203 |
175 /// Returns a string description of the test run descibed by [results]. | 204 /// Returns a string description of the test run descibed by [results]. |
(...skipping 28 matching lines...) Expand all Loading... |
204 return prefixLines(buffer.toString()); | 233 return prefixLines(buffer.toString()); |
205 } | 234 } |
206 | 235 |
207 /// Indents each line of [str] by two spaces. | 236 /// Indents each line of [str] by two spaces. |
208 String _indent(String str) { | 237 String _indent(String str) { |
209 return str.replaceAll(new RegExp("^", multiLine: true), " "); | 238 return str.replaceAll(new RegExp("^", multiLine: true), " "); |
210 } | 239 } |
211 | 240 |
212 /// Ensure that the metatest configuration is loaded. | 241 /// Ensure that the metatest configuration is loaded. |
213 void _ensureInitialized() { | 242 void _ensureInitialized() { |
214 unittestConfiguration = _singleton; | 243 unittestConfiguration = _metaConfiguration; |
| 244 if (_timeoutOverride != null) { |
| 245 unittestConfiguration.timeout = _timeoutOverride; |
| 246 } |
215 } | 247 } |
216 | 248 |
217 final _singleton = new _MetaConfiguration(); | |
218 | |
219 /// Special test configuration for use within the child isolates. This hides all | 249 /// Special test configuration for use within the child isolates. This hides all |
220 /// output and reports data back to the parent isolate. | 250 /// output and reports data back to the parent isolate. |
221 class _MetaConfiguration extends Configuration { | 251 class _MetaConfiguration extends Configuration { |
222 | 252 |
223 _MetaConfiguration() : super.blank(); | 253 _MetaConfiguration() : super.blank(); |
224 | 254 |
225 void onSummary(int passed, int failed, int errors, List<TestCase> results, | 255 void onSummary(int passed, int failed, int errors, List<TestCase> results, |
226 String uncaughtError) { | 256 String uncaughtError) { |
227 _replyTo.send({ | 257 _replyTo.send({ |
228 "passed": passed, | 258 "passed": passed, |
229 "failed": failed, | 259 "failed": failed, |
230 "errors": errors, | 260 "errors": errors, |
231 "uncaughtError": uncaughtError, | 261 "uncaughtError": uncaughtError, |
232 "results": results.map((testCase) => { | 262 "results": results.map((testCase) => { |
233 "description": testCase.description, | 263 "description": testCase.description, |
234 "message": testCase.message, | 264 "message": testCase.message, |
235 "result": testCase.result, | 265 "result": testCase.result, |
236 "stackTrace": testCase.stackTrace.toString() | 266 "stackTrace": testCase.stackTrace.toString() |
237 }).toList() | 267 }).toList() |
238 }); | 268 }); |
239 } | 269 } |
240 } | 270 } |
OLD | NEW |