| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 175 * process, before the regular test is run as in the base class [TestCase]. | 175 * process, before the regular test is run as in the base class [TestCase]. |
| 176 * If the compilation command fails, then the rest of the test is not run. | 176 * If the compilation command fails, then the rest of the test is not run. |
| 177 */ | 177 */ |
| 178 class BrowserTestCase extends TestCase { | 178 class BrowserTestCase extends TestCase { |
| 179 /** | 179 /** |
| 180 * Indicates the number of potential retries remaining, to compensate for | 180 * Indicates the number of potential retries remaining, to compensate for |
| 181 * flaky browser tests. | 181 * flaky browser tests. |
| 182 */ | 182 */ |
| 183 int numRetries; | 183 int numRetries; |
| 184 | 184 |
| 185 /** |
| 186 * True if this test is dependent on another test completing before it can |
| 187 * star (for example, we might need to depend on some other test completing |
| 188 * first). |
| 189 */ |
| 190 bool waitingForOtherTest; |
| 191 |
| 192 /** |
| 193 * The set of test cases that wish to be notified when this test has |
| 194 * completed. |
| 195 */ |
| 196 List<BrowserTestCase> observers; |
| 197 |
| 185 BrowserTestCase(displayName, commands, configuration, completedHandler, | 198 BrowserTestCase(displayName, commands, configuration, completedHandler, |
| 186 expectedOutcomes, info, isNegative) | 199 expectedOutcomes, info, isNegative, [this.waitingForOtherTest = false]) |
| 187 : super(displayName, commands, configuration, completedHandler, | 200 : super(displayName, commands, configuration, completedHandler, |
| 188 expectedOutcomes, isNegative: isNegative, info: info) { | 201 expectedOutcomes, isNegative: isNegative, info: info) { |
| 189 numRetries = 2; // Allow two retries to compensate for flaky browser tests. | 202 numRetries = 2; // Allow two retries to compensate for flaky browser tests. |
| 203 observers = []; |
| 190 } | 204 } |
| 191 | 205 |
| 192 List<String> get _lastArguments => commands.last.arguments; | 206 List<String> get _lastArguments => commands.last.arguments; |
| 193 | 207 |
| 194 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch']; | 208 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch']; |
| 195 | 209 |
| 196 List<String> get batchTestArguments => | 210 List<String> get batchTestArguments => |
| 197 _lastArguments.getRange(1, _lastArguments.length - 1); | 211 _lastArguments.getRange(1, _lastArguments.length - 1); |
| 212 |
| 213 /** Add a test case to listen for when this current test has completed. */ |
| 214 void addObserver(BrowserTestCase testCase) { |
| 215 observers.add(testCase); |
| 216 } |
| 217 |
| 218 /** |
| 219 * Notify all of the test cases that are dependent on this one that they can |
| 220 * proceed. |
| 221 */ |
| 222 void notifyObservers() { |
| 223 for (BrowserTestCase testCase in observers) { |
| 224 testCase.waitingForOtherTest = false; |
| 225 } |
| 226 } |
| 198 } | 227 } |
| 199 | 228 |
| 200 | 229 |
| 201 /** | 230 /** |
| 202 * TestOutput records the output of a completed test: the process's exit code, | 231 * TestOutput records the output of a completed test: the process's exit code, |
| 203 * the standard output and standard error, whether the process timed out, and | 232 * the standard output and standard error, whether the process timed out, and |
| 204 * the time the process took to run. It also contains a pointer to the | 233 * the time the process took to run. It also contains a pointer to the |
| 205 * [TestCase] this is the output of. | 234 * [TestCase] this is the output of. |
| 206 */ | 235 */ |
| 207 abstract class TestOutput { | 236 abstract class TestOutput { |
| (...skipping 861 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1069 if (_tests.isEmpty && _numProcesses == 0) { | 1098 if (_tests.isEmpty && _numProcesses == 0) { |
| 1070 _terminateBatchRunners().then((_) => _cleanupAndMarkDone()); | 1099 _terminateBatchRunners().then((_) => _cleanupAndMarkDone()); |
| 1071 } | 1100 } |
| 1072 } | 1101 } |
| 1073 } | 1102 } |
| 1074 | 1103 |
| 1075 /** | 1104 /** |
| 1076 * True if we are using a browser + platform combination that needs the | 1105 * True if we are using a browser + platform combination that needs the |
| 1077 * Selenium server jar. | 1106 * Selenium server jar. |
| 1078 */ | 1107 */ |
| 1079 bool get _needsSelenium => Platform.operatingSystem == 'macos' && | 1108 bool get _needsSelenium => (Platform.operatingSystem == 'macos' && |
| 1080 browserUsed == 'safari'; | 1109 browserUsed == 'safari') || browserUsed == 'opera'; |
| 1081 | 1110 |
| 1082 /** True if the Selenium Server is ready to be used. */ | 1111 /** True if the Selenium Server is ready to be used. */ |
| 1083 bool get _isSeleniumAvailable => _seleniumServer != null || | 1112 bool get _isSeleniumAvailable => _seleniumServer != null || |
| 1084 _seleniumAlreadyRunning; | 1113 _seleniumAlreadyRunning; |
| 1085 | 1114 |
| 1086 /** | 1115 /** |
| 1087 * Restart all the processes that have been waiting/stopped for the server to | 1116 * Restart all the processes that have been waiting/stopped for the server to |
| 1088 * start up. If we just call this once we end up with a single-"threaded" run. | 1117 * start up. If we just call this once we end up with a single-"threaded" run. |
| 1089 */ | 1118 */ |
| 1090 void resumeTesting() { | 1119 void resumeTesting() { |
| (...skipping 151 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1242 if (_numProcesses < _maxProcesses && !_tests.isEmpty) { | 1271 if (_numProcesses < _maxProcesses && !_tests.isEmpty) { |
| 1243 TestCase test = _tests.removeFirst(); | 1272 TestCase test = _tests.removeFirst(); |
| 1244 if (_listTests) { | 1273 if (_listTests) { |
| 1245 var fields = [test.displayName, | 1274 var fields = [test.displayName, |
| 1246 Strings.join(new List.from(test.expectedOutcomes), ','), | 1275 Strings.join(new List.from(test.expectedOutcomes), ','), |
| 1247 test.isNegative.toString()]; | 1276 test.isNegative.toString()]; |
| 1248 fields.addAll(test.commands.last.arguments); | 1277 fields.addAll(test.commands.last.arguments); |
| 1249 print(Strings.join(fields, '\t')); | 1278 print(Strings.join(fields, '\t')); |
| 1250 return; | 1279 return; |
| 1251 } | 1280 } |
| 1252 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) { | 1281 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable || (test |
| 1253 // The server is not ready to run Selenium tests. Put the test back in | 1282 is BrowserTestCase && test.waitingForOtherTest)) { |
| 1283 // The test is not yet ready to run. Put the test back in |
| 1254 // the queue. Avoid spin-polling by using a timeout. | 1284 // the queue. Avoid spin-polling by using a timeout. |
| 1255 _tests.add(test); | 1285 _tests.add(test); |
| 1256 new Timer(1000, (timer) {_tryRunTest();}); // Don't lose a process. | 1286 new Timer(100, (timer) {_tryRunTest();}); // Don't lose a process. |
| 1257 return; | 1287 return; |
| 1258 } | 1288 } |
| 1259 if (_verbose) { | 1289 if (_verbose) { |
| 1260 int i = 1; | 1290 int i = 1; |
| 1261 for (Command command in test.commands) { | 1291 for (Command command in test.commands) { |
| 1262 print('$i. ${command.commandLine}'); | 1292 print('$i. ${command.commandLine}'); |
| 1263 i++; | 1293 i++; |
| 1264 } | 1294 } |
| 1265 } | 1295 } |
| 1266 _progress.start(test); | 1296 _progress.start(test); |
| 1267 TestCaseEvent oldCallback = test.completedHandler; | 1297 TestCaseEvent oldCallback = test.completedHandler; |
| 1268 void wrapper(TestCase test_arg) { | 1298 void wrapper(TestCase test_arg) { |
| 1269 _numProcesses--; | 1299 _numProcesses--; |
| 1270 _progress.done(test_arg); | 1300 _progress.done(test_arg); |
| 1301 if (test_arg is BrowserTestCase) test_arg.notifyObservers(); |
| 1271 _tryRunTest(); | 1302 _tryRunTest(); |
| 1272 oldCallback(test_arg); | 1303 oldCallback(test_arg); |
| 1273 }; | 1304 }; |
| 1274 test.completedHandler = wrapper; | 1305 test.completedHandler = wrapper; |
| 1275 if (test.configuration['compiler'] == 'dartc' && | 1306 |
| 1276 test.displayName != 'dartc/junit_tests') { | 1307 if ((test.configuration['compiler'] == 'dartc' && |
| 1308 test.displayName != 'dartc/junit_tests') || |
| 1309 (test.commands.length == 1 && test.usesWebDriver && |
| 1310 !test.configuration['noBatch'])) { |
| 1311 // Dartc and browser test cases that do not require a precompilation |
| 1312 // step, start with the batch runner right away. |
| 1277 _getBatchRunner(test).startTest(test); | 1313 _getBatchRunner(test).startTest(test); |
| 1278 } else { | 1314 } else { |
| 1279 // Once we've actually failed a test, technically, we wouldn't need to | 1315 // Once we've actually failed a test, technically, we wouldn't need to |
| 1280 // bother retrying any subsequent tests since the bot is already red. | 1316 // bother retrying any subsequent tests since the bot is already red. |
| 1281 // However, we continue to retry tests until we have actually failed | 1317 // However, we continue to retry tests until we have actually failed |
| 1282 // four tests (arbitrarily chosen) for more debugable output, so that | 1318 // four tests (arbitrarily chosen) for more debugable output, so that |
| 1283 // the developer doesn't waste his or her time trying to fix a bunch of | 1319 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1284 // tests that appear to be broken but were actually just flakes that | 1320 // tests that appear to be broken but were actually just flakes that |
| 1285 // didn't get retried because there had already been one failure. | 1321 // didn't get retried because there had already been one failure. |
| 1286 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1322 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1287 new RunningProcess(test, allowRetry, this).start(); | 1323 new RunningProcess(test, allowRetry, this).start(); |
| 1288 } | 1324 } |
| 1289 _numProcesses++; | 1325 _numProcesses++; |
| 1290 } | 1326 } |
| 1291 } | 1327 } |
| 1292 } | 1328 } |
| OLD | NEW |