| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 175 * process, before the regular test is run as in the base class [TestCase]. | 175 * process, before the regular test is run as in the base class [TestCase]. |
| 176 * If the compilation command fails, then the rest of the test is not run. | 176 * If the compilation command fails, then the rest of the test is not run. |
| 177 */ | 177 */ |
| 178 class BrowserTestCase extends TestCase { | 178 class BrowserTestCase extends TestCase { |
| 179 /** | 179 /** |
| 180 * Indicates the number of potential retries remaining, to compensate for | 180 * Indicates the number of potential retries remaining, to compensate for |
| 181 * flaky browser tests. | 181 * flaky browser tests. |
| 182 */ | 182 */ |
| 183 int numRetries; | 183 int numRetries; |
| 184 | 184 |
| 185 /** | |
| 186 * True if this test is dependent on another test completing before it can | |
| 187 * star (for example, we might need to depend on some other test completing | |
| 188 * first). | |
| 189 */ | |
| 190 bool waitingForOtherTest; | |
| 191 | |
| 192 /** | |
| 193 * The set of test cases that wish to be notified when this test has | |
| 194 * completed. | |
| 195 */ | |
| 196 List<BrowserTestCase> observers; | |
| 197 | |
| 198 BrowserTestCase(displayName, commands, configuration, completedHandler, | 185 BrowserTestCase(displayName, commands, configuration, completedHandler, |
| 199 expectedOutcomes, info, isNegative, [this.waitingForOtherTest = false]) | 186 expectedOutcomes, info, isNegative) |
| 200 : super(displayName, commands, configuration, completedHandler, | 187 : super(displayName, commands, configuration, completedHandler, |
| 201 expectedOutcomes, isNegative: isNegative, info: info) { | 188 expectedOutcomes, isNegative: isNegative, info: info) { |
| 202 numRetries = 2; // Allow two retries to compensate for flaky browser tests. | 189 numRetries = 2; // Allow two retries to compensate for flaky browser tests. |
| 203 observers = []; | |
| 204 } | 190 } |
| 205 | 191 |
| 206 List<String> get _lastArguments => commands.last.arguments; | 192 List<String> get _lastArguments => commands.last.arguments; |
| 207 | 193 |
| 208 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch']; | 194 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch']; |
| 209 | 195 |
| 210 List<String> get batchTestArguments => | 196 List<String> get batchTestArguments => |
| 211 _lastArguments.getRange(1, _lastArguments.length - 1); | 197 _lastArguments.getRange(1, _lastArguments.length - 1); |
| 212 | |
| 213 /** Add a test case to listen for when this current test has completed. */ | |
| 214 void addObserver(BrowserTestCase testCase) { | |
| 215 observers.add(testCase); | |
| 216 } | |
| 217 | |
| 218 /** | |
| 219 * Notify all of the test cases that are dependent on this one that they can | |
| 220 * proceed. | |
| 221 */ | |
| 222 void notifyObservers() { | |
| 223 for (BrowserTestCase testCase in observers) { | |
| 224 testCase.waitingForOtherTest = false; | |
| 225 } | |
| 226 } | |
| 227 } | 198 } |
| 228 | 199 |
| 229 | 200 |
| 230 /** | 201 /** |
| 231 * TestOutput records the output of a completed test: the process's exit code, | 202 * TestOutput records the output of a completed test: the process's exit code, |
| 232 * the standard output and standard error, whether the process timed out, and | 203 * the standard output and standard error, whether the process timed out, and |
| 233 * the time the process took to run. It also contains a pointer to the | 204 * the time the process took to run. It also contains a pointer to the |
| 234 * [TestCase] this is the output of. | 205 * [TestCase] this is the output of. |
| 235 */ | 206 */ |
| 236 abstract class TestOutput { | 207 abstract class TestOutput { |
| (...skipping 861 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1098 if (_tests.isEmpty && _numProcesses == 0) { | 1069 if (_tests.isEmpty && _numProcesses == 0) { |
| 1099 _terminateBatchRunners().then((_) => _cleanupAndMarkDone()); | 1070 _terminateBatchRunners().then((_) => _cleanupAndMarkDone()); |
| 1100 } | 1071 } |
| 1101 } | 1072 } |
| 1102 } | 1073 } |
| 1103 | 1074 |
| 1104 /** | 1075 /** |
| 1105 * True if we are using a browser + platform combination that needs the | 1076 * True if we are using a browser + platform combination that needs the |
| 1106 * Selenium server jar. | 1077 * Selenium server jar. |
| 1107 */ | 1078 */ |
| 1108 bool get _needsSelenium => (Platform.operatingSystem == 'macos' && | 1079 bool get _needsSelenium => Platform.operatingSystem == 'macos' && |
| 1109 browserUsed == 'safari') || browserUsed == 'opera'; | 1080 browserUsed == 'safari'; |
| 1110 | 1081 |
| 1111 /** True if the Selenium Server is ready to be used. */ | 1082 /** True if the Selenium Server is ready to be used. */ |
| 1112 bool get _isSeleniumAvailable => _seleniumServer != null || | 1083 bool get _isSeleniumAvailable => _seleniumServer != null || |
| 1113 _seleniumAlreadyRunning; | 1084 _seleniumAlreadyRunning; |
| 1114 | 1085 |
| 1115 /** | 1086 /** |
| 1116 * Restart all the processes that have been waiting/stopped for the server to | 1087 * Restart all the processes that have been waiting/stopped for the server to |
| 1117 * start up. If we just call this once we end up with a single-"threaded" run. | 1088 * start up. If we just call this once we end up with a single-"threaded" run. |
| 1118 */ | 1089 */ |
| 1119 void resumeTesting() { | 1090 void resumeTesting() { |
| (...skipping 151 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1271 if (_numProcesses < _maxProcesses && !_tests.isEmpty) { | 1242 if (_numProcesses < _maxProcesses && !_tests.isEmpty) { |
| 1272 TestCase test = _tests.removeFirst(); | 1243 TestCase test = _tests.removeFirst(); |
| 1273 if (_listTests) { | 1244 if (_listTests) { |
| 1274 var fields = [test.displayName, | 1245 var fields = [test.displayName, |
| 1275 Strings.join(new List.from(test.expectedOutcomes), ','), | 1246 Strings.join(new List.from(test.expectedOutcomes), ','), |
| 1276 test.isNegative.toString()]; | 1247 test.isNegative.toString()]; |
| 1277 fields.addAll(test.commands.last.arguments); | 1248 fields.addAll(test.commands.last.arguments); |
| 1278 print(Strings.join(fields, '\t')); | 1249 print(Strings.join(fields, '\t')); |
| 1279 return; | 1250 return; |
| 1280 } | 1251 } |
| 1281 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable || (test | 1252 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) { |
| 1282 is BrowserTestCase && test.waitingForOtherTest)) { | 1253 // The server is not ready to run Selenium tests. Put the test back in |
| 1283 // The test is not yet ready to run. Put the test back in | |
| 1284 // the queue. Avoid spin-polling by using a timeout. | 1254 // the queue. Avoid spin-polling by using a timeout. |
| 1285 _tests.add(test); | 1255 _tests.add(test); |
| 1286 new Timer(100, (timer) {_tryRunTest();}); // Don't lose a process. | 1256 new Timer(1000, (timer) {_tryRunTest();}); // Don't lose a process. |
| 1287 return; | 1257 return; |
| 1288 } | 1258 } |
| 1289 if (_verbose) { | 1259 if (_verbose) { |
| 1290 int i = 1; | 1260 int i = 1; |
| 1291 for (Command command in test.commands) { | 1261 for (Command command in test.commands) { |
| 1292 print('$i. ${command.commandLine}'); | 1262 print('$i. ${command.commandLine}'); |
| 1293 i++; | 1263 i++; |
| 1294 } | 1264 } |
| 1295 } | 1265 } |
| 1296 _progress.start(test); | 1266 _progress.start(test); |
| 1297 TestCaseEvent oldCallback = test.completedHandler; | 1267 TestCaseEvent oldCallback = test.completedHandler; |
| 1298 void wrapper(TestCase test_arg) { | 1268 void wrapper(TestCase test_arg) { |
| 1299 _numProcesses--; | 1269 _numProcesses--; |
| 1300 _progress.done(test_arg); | 1270 _progress.done(test_arg); |
| 1301 if (test_arg is BrowserTestCase) test_arg.notifyObservers(); | |
| 1302 _tryRunTest(); | 1271 _tryRunTest(); |
| 1303 oldCallback(test_arg); | 1272 oldCallback(test_arg); |
| 1304 }; | 1273 }; |
| 1305 test.completedHandler = wrapper; | 1274 test.completedHandler = wrapper; |
| 1306 | 1275 if (test.configuration['compiler'] == 'dartc' && |
| 1307 if ((test.configuration['compiler'] == 'dartc' && | 1276 test.displayName != 'dartc/junit_tests') { |
| 1308 test.displayName != 'dartc/junit_tests') || | |
| 1309 (test.commands.length == 1 && test.usesWebDriver && | |
| 1310 !test.configuration['noBatch'])) { | |
| 1311 // Dartc and browser test cases that do not require a precompilation | |
| 1312 // step, start with the batch runner right away. | |
| 1313 _getBatchRunner(test).startTest(test); | 1277 _getBatchRunner(test).startTest(test); |
| 1314 } else { | 1278 } else { |
| 1315 // Once we've actually failed a test, technically, we wouldn't need to | 1279 // Once we've actually failed a test, technically, we wouldn't need to |
| 1316 // bother retrying any subsequent tests since the bot is already red. | 1280 // bother retrying any subsequent tests since the bot is already red. |
| 1317 // However, we continue to retry tests until we have actually failed | 1281 // However, we continue to retry tests until we have actually failed |
| 1318 // four tests (arbitrarily chosen) for more debugable output, so that | 1282 // four tests (arbitrarily chosen) for more debugable output, so that |
| 1319 // the developer doesn't waste his or her time trying to fix a bunch of | 1283 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1320 // tests that appear to be broken but were actually just flakes that | 1284 // tests that appear to be broken but were actually just flakes that |
| 1321 // didn't get retried because there had already been one failure. | 1285 // didn't get retried because there had already been one failure. |
| 1322 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1286 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1323 new RunningProcess(test, allowRetry, this).start(); | 1287 new RunningProcess(test, allowRetry, this).start(); |
| 1324 } | 1288 } |
| 1325 _numProcesses++; | 1289 _numProcesses++; |
| 1326 } | 1290 } |
| 1327 } | 1291 } |
| 1328 } | 1292 } |
| OLD | NEW |