Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 70 * tests have two commands, one to compilate the source and another to execute | 70 * tests have two commands, one to compilate the source and another to execute |
| 71 * it. Some isolate tests might even have three, if they require compiling | 71 * it. Some isolate tests might even have three, if they require compiling |
| 72 * multiple sources that are run in isolation. | 72 * multiple sources that are run in isolation. |
| 73 */ | 73 */ |
| 74 List<Command> commands; | 74 List<Command> commands; |
| 75 | 75 |
| 76 Map configuration; | 76 Map configuration; |
| 77 String displayName; | 77 String displayName; |
| 78 TestOutput output; | 78 TestOutput output; |
| 79 bool isNegative; | 79 bool isNegative; |
| 80 bool usesWebDriver; | |
| 81 Set<String> expectedOutcomes; | 80 Set<String> expectedOutcomes; |
| 82 TestCaseEvent completedHandler; | 81 TestCaseEvent completedHandler; |
| 83 TestInformation info; | 82 TestInformation info; |
| 84 | 83 |
| 85 TestCase(this.displayName, | 84 TestCase(this.displayName, |
| 86 this.commands, | 85 this.commands, |
| 87 this.configuration, | 86 this.configuration, |
| 88 this.completedHandler, | 87 this.completedHandler, |
| 89 this.expectedOutcomes, | 88 this.expectedOutcomes, |
| 90 {this.isNegative: false, | 89 {this.isNegative: false, |
| 91 this.info: null, | 90 this.info: null}) { |
| 92 this.usesWebDriver: false}) { | |
| 93 if (!isNegative) { | 91 if (!isNegative) { |
| 94 this.isNegative = displayName.contains("negative_test"); | 92 this.isNegative = displayName.contains("negative_test"); |
| 95 } | 93 } |
| 96 | 94 |
| 97 // Special command handling. If a special command is specified | 95 // Special command handling. If a special command is specified |
| 98 // we have to completely rewrite the command that we are using. | 96 // we have to completely rewrite the command that we are using. |
| 99 // We generate a new command-line that is the special command where we | 97 // We generate a new command-line that is the special command where we |
| 100 // replace '@' with the original command executable, and generate | 98 // replace '@' with the original command executable, and generate |
| 101 // a command formed like the following | 99 // a command formed like the following |
| 102 // Let PREFIX be what is before the @. | 100 // Let PREFIX be what is before the @. |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 159 final runtime = configuration['runtime']; | 157 final runtime = configuration['runtime']; |
| 160 final mode = configuration['mode']; | 158 final mode = configuration['mode']; |
| 161 final arch = configuration['arch']; | 159 final arch = configuration['arch']; |
| 162 final checked = configuration['checked'] ? '-checked' : ''; | 160 final checked = configuration['checked'] ? '-checked' : ''; |
| 163 return "$compiler-$runtime$checked ${mode}_$arch"; | 161 return "$compiler-$runtime$checked ${mode}_$arch"; |
| 164 } | 162 } |
| 165 | 163 |
| 166 List<String> get batchRunnerArguments => ['-batch']; | 164 List<String> get batchRunnerArguments => ['-batch']; |
| 167 List<String> get batchTestArguments => commands.last().arguments; | 165 List<String> get batchTestArguments => commands.last().arguments; |
| 168 | 166 |
| 167 bool get usesWebDriver => TestUtils.usesWebDriver(configuration['runtime']); | |
| 168 | |
| 169 void completed() { completedHandler(this); } | 169 void completed() { completedHandler(this); } |
| 170 } | 170 } |
| 171 | 171 |
| 172 | 172 |
| 173 /** | 173 /** |
| 174 * BrowserTestCase has an extra compilation command that is run in a separate | 174 * BrowserTestCase has an extra compilation command that is run in a separate |
| 175 * process, before the regular test is run as in the base class [TestCase]. | 175 * process, before the regular test is run as in the base class [TestCase]. |
| 176 * If the compilation command fails, then the rest of the test is not run. | 176 * If the compilation command fails, then the rest of the test is not run. |
| 177 */ | 177 */ |
| 178 class BrowserTestCase extends TestCase { | 178 class BrowserTestCase extends TestCase { |
| (...skipping 408 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 587 && (testCase as BrowserTestCase).numRetries > 0) { | 587 && (testCase as BrowserTestCase).numRetries > 0) { |
| 588 // Selenium tests can be flaky. Try rerunning. | 588 // Selenium tests can be flaky. Try rerunning. |
| 589 testCase.output.requestRetry = true; | 589 testCase.output.requestRetry = true; |
| 590 } | 590 } |
| 591 if (testCase.output.requestRetry) { | 591 if (testCase.output.requestRetry) { |
| 592 testCase.output.requestRetry = false; | 592 testCase.output.requestRetry = false; |
| 593 this.timedOut = false; | 593 this.timedOut = false; |
| 594 (testCase as BrowserTestCase).numRetries--; | 594 (testCase as BrowserTestCase).numRetries--; |
| 595 print("Potential flake. Re-running ${testCase.displayName} " | 595 print("Potential flake. Re-running ${testCase.displayName} " |
| 596 "(${(testCase as BrowserTestCase).numRetries} attempt(s) remains)"); | 596 "(${(testCase as BrowserTestCase).numRetries} attempt(s) remains)"); |
| 597 // When retrying we need to reset the timeout as well. | |
| 598 // Otherwise there will be no timeout handling for the retry. | |
| 599 timeoutTimer = null; | |
|
Emily Fortuna
2012/10/24 17:14:48
This shouldn't be reached on the bots because the
Mads Ager (google)
2012/10/24 17:56:03
This is needed if step1 fails. I believe that only
| |
| 597 this.start(); | 600 this.start(); |
| 598 } else { | 601 } else { |
| 599 testCase.completed(); | 602 testCase.completed(); |
| 600 } | 603 } |
| 601 } | 604 } |
| 602 | 605 |
| 603 /** | 606 /** |
| 604 * Process exit handler called at the end of every command. It internally | 607 * Process exit handler called at the end of every command. It internally |
| 605 * treats all but the last command as compilation steps. The last command is | 608 * treats all but the last command as compilation steps. The last command is |
| 606 * the actual test and its output is analyzed in [testComplete]. | 609 * the actual test and its output is analyzed in [testComplete]. |
| 607 */ | 610 */ |
| 608 void stepExitHandler(int exitCode) { | 611 void stepExitHandler(int exitCode) { |
| 609 process.close(); | 612 process.close(); |
| 610 process = null; | 613 process = null; |
| 611 int totalSteps = testCase.commands.length; | 614 int totalSteps = testCase.commands.length; |
| 612 String suffix =' (step $currentStep of $totalSteps)'; | 615 String suffix =' (step $currentStep of $totalSteps)'; |
| 613 if (timedOut && | 616 if (timedOut) { |
| 614 !(testCase.usesWebDriver && !testCase.configuration['noBatch'])) { | |
| 615 // Non-webdriver test timed out before it could complete. Webdriver tests | 617 // Non-webdriver test timed out before it could complete. Webdriver tests |
| 616 // run their own timeouts by timing from the launch of the browser (which | 618 // run their own timeouts by timing from the launch of the browser (which |
| 617 // could be delayed). | 619 // could be delayed). |
| 618 testComplete(0, true); | 620 testComplete(0, true); |
| 619 } else if (currentStep == totalSteps) { | 621 } else if (currentStep == totalSteps) { |
| 620 // Done with all test commands. | 622 // Done with all test commands. |
| 621 testComplete(exitCode, false); | 623 testComplete(exitCode, false); |
| 622 } else if (exitCode != 0) { | 624 } else if (exitCode != 0) { |
| 623 // One of the steps failed. | 625 // One of the steps failed. |
| 624 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 626 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
| (...skipping 659 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1284 // the developer doesn't waste his or her time trying to fix a bunch of | 1286 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1285 // tests that appear to be broken but were actually just flakes that | 1287 // tests that appear to be broken but were actually just flakes that |
| 1286 // didn't get retried because there had already been one failure. | 1288 // didn't get retried because there had already been one failure. |
| 1287 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1289 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1288 new RunningProcess(test, allowRetry, this).start(); | 1290 new RunningProcess(test, allowRetry, this).start(); |
| 1289 } | 1291 } |
| 1290 _numProcesses++; | 1292 _numProcesses++; |
| 1291 } | 1293 } |
| 1292 } | 1294 } |
| 1293 } | 1295 } |
| OLD | NEW |