Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 70 * tests have two commands, one to compilate the source and another to execute | 70 * tests have two commands, one to compilate the source and another to execute |
| 71 * it. Some isolate tests might even have three, if they require compiling | 71 * it. Some isolate tests might even have three, if they require compiling |
| 72 * multiple sources that are run in isolation. | 72 * multiple sources that are run in isolation. |
| 73 */ | 73 */ |
| 74 List<Command> commands; | 74 List<Command> commands; |
| 75 | 75 |
| 76 Map configuration; | 76 Map configuration; |
| 77 String displayName; | 77 String displayName; |
| 78 TestOutput output; | 78 TestOutput output; |
| 79 bool isNegative; | 79 bool isNegative; |
| 80 bool usesWebDriver; | |
|
Mads Ager (google)
2012/10/25 08:11:55
I'm not sure what we gain by having this as a memb
Emily Fortuna
2012/10/25 18:21:20
I agree. The reason I changed this is I was findin
Mads Ager (google)
2012/10/25 18:47:44
I strongly agree with your use of TestUtils.usesWe
| |
| 80 Set<String> expectedOutcomes; | 81 Set<String> expectedOutcomes; |
| 81 TestCaseEvent completedHandler; | 82 TestCaseEvent completedHandler; |
| 82 TestInformation info; | 83 TestInformation info; |
| 83 | 84 |
| 84 TestCase(this.displayName, | 85 TestCase(this.displayName, |
| 85 this.commands, | 86 this.commands, |
| 86 this.configuration, | 87 this.configuration, |
| 87 this.completedHandler, | 88 this.completedHandler, |
| 88 this.expectedOutcomes, | 89 this.expectedOutcomes, |
| 89 {this.isNegative: false, | 90 {this.isNegative: false, |
| 90 this.info: null}) { | 91 this.info: null, |
| 92 this.usesWebDriver: false}) { | |
| 91 if (!isNegative) { | 93 if (!isNegative) { |
| 92 this.isNegative = displayName.contains("negative_test"); | 94 this.isNegative = displayName.contains("negative_test"); |
| 93 } | 95 } |
| 94 | 96 |
| 95 // Special command handling. If a special command is specified | 97 // Special command handling. If a special command is specified |
| 96 // we have to completely rewrite the command that we are using. | 98 // we have to completely rewrite the command that we are using. |
| 97 // We generate a new command-line that is the special command where we | 99 // We generate a new command-line that is the special command where we |
| 98 // replace '@' with the original command executable, and generate | 100 // replace '@' with the original command executable, and generate |
| 99 // a command formed like the following | 101 // a command formed like the following |
| 100 // Let PREFIX be what is before the @. | 102 // Let PREFIX be what is before the @. |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 157 final runtime = configuration['runtime']; | 159 final runtime = configuration['runtime']; |
| 158 final mode = configuration['mode']; | 160 final mode = configuration['mode']; |
| 159 final arch = configuration['arch']; | 161 final arch = configuration['arch']; |
| 160 final checked = configuration['checked'] ? '-checked' : ''; | 162 final checked = configuration['checked'] ? '-checked' : ''; |
| 161 return "$compiler-$runtime$checked ${mode}_$arch"; | 163 return "$compiler-$runtime$checked ${mode}_$arch"; |
| 162 } | 164 } |
| 163 | 165 |
| 164 List<String> get batchRunnerArguments => ['-batch']; | 166 List<String> get batchRunnerArguments => ['-batch']; |
| 165 List<String> get batchTestArguments => commands.last().arguments; | 167 List<String> get batchTestArguments => commands.last().arguments; |
| 166 | 168 |
| 167 bool get usesWebDriver => TestUtils.usesWebDriver(configuration['runtime']); | |
| 168 | |
| 169 void completed() { completedHandler(this); } | 169 void completed() { completedHandler(this); } |
| 170 } | 170 } |
| 171 | 171 |
| 172 | 172 |
| 173 /** | 173 /** |
| 174 * BrowserTestCase has an extra compilation command that is run in a separate | 174 * BrowserTestCase has an extra compilation command that is run in a separate |
| 175 * process, before the regular test is run as in the base class [TestCase]. | 175 * process, before the regular test is run as in the base class [TestCase]. |
| 176 * If the compilation command fails, then the rest of the test is not run. | 176 * If the compilation command fails, then the rest of the test is not run. |
| 177 */ | 177 */ |
| 178 class BrowserTestCase extends TestCase { | 178 class BrowserTestCase extends TestCase { |
| 179 /** | 179 /** |
| 180 * Indicates the number of potential retries remaining, to compensate for | 180 * Indicates the number of potential retries remaining, to compensate for |
| 181 * flaky browser tests. | 181 * flaky browser tests. |
| 182 */ | 182 */ |
| 183 int numRetries; | 183 int numRetries; |
| 184 | 184 |
| 185 BrowserTestCase(displayName, commands, configuration, completedHandler, | 185 BrowserTestCase(displayName, commands, configuration, completedHandler, |
| 186 expectedOutcomes, info, isNegative) | 186 expectedOutcomes, info, isNegative, {useWebDriver: false}) |
| 187 : super(displayName, commands, configuration, completedHandler, | 187 : super(displayName, commands, configuration, completedHandler, |
| 188 expectedOutcomes, isNegative: isNegative, info: info) { | 188 expectedOutcomes, isNegative: isNegative, info: info, |
| 189 usesWebDriver: useWebDriver) { | |
| 189 numRetries = 2; // Allow two retries to compensate for flaky browser tests. | 190 numRetries = 2; // Allow two retries to compensate for flaky browser tests. |
| 190 } | 191 } |
| 191 | 192 |
| 192 List<String> get _lastArguments => commands.last().arguments; | 193 List<String> get _lastArguments => commands.last().arguments; |
| 193 | 194 |
| 194 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch']; | 195 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch']; |
| 195 | 196 |
| 196 List<String> get batchTestArguments => | 197 List<String> get batchTestArguments => |
| 197 _lastArguments.getRange(1, _lastArguments.length - 1); | 198 _lastArguments.getRange(1, _lastArguments.length - 1); |
| 198 } | 199 } |
| (...skipping 935 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1134 print(" Error: $e"); | 1135 print(" Error: $e"); |
| 1135 // TODO(ahe): How to report this as a test failure? | 1136 // TODO(ahe): How to report this as a test failure? |
| 1136 exit(1); | 1137 exit(1); |
| 1137 return true; | 1138 return true; |
| 1138 }); | 1139 }); |
| 1139 } | 1140 } |
| 1140 } | 1141 } |
| 1141 | 1142 |
| 1142 void _runTest(TestCase test) { | 1143 void _runTest(TestCase test) { |
| 1143 if (test.usesWebDriver) { | 1144 if (test.usesWebDriver) { |
| 1144 browserUsed = test.configuration['browser']; | 1145 browserUsed = test.configuration['runtime']; |
|
Mads Ager (google)
2012/10/25 08:11:55
Thank you! I hope this will work out on the bots.
| |
| 1145 if (_needsSelenium) _ensureSeleniumServerRunning(); | 1146 if (_needsSelenium) _ensureSeleniumServerRunning(); |
| 1146 } | 1147 } |
| 1147 _progress.testAdded(); | 1148 _progress.testAdded(); |
| 1148 _tests.add(test); | 1149 _tests.add(test); |
| 1149 _tryRunTest(); | 1150 _tryRunTest(); |
| 1150 } | 1151 } |
| 1151 | 1152 |
| 1152 /** | 1153 /** |
| 1153 * Monitor the output of the Selenium server, to know when we are ready to | 1154 * Monitor the output of the Selenium server, to know when we are ready to |
| 1154 * begin running tests. | 1155 * begin running tests. |
| (...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1286 // the developer doesn't waste his or her time trying to fix a bunch of | 1287 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1287 // tests that appear to be broken but were actually just flakes that | 1288 // tests that appear to be broken but were actually just flakes that |
| 1288 // didn't get retried because there had already been one failure. | 1289 // didn't get retried because there had already been one failure. |
| 1289 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1290 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1290 new RunningProcess(test, allowRetry, this).start(); | 1291 new RunningProcess(test, allowRetry, this).start(); |
| 1291 } | 1292 } |
| 1292 _numProcesses++; | 1293 _numProcesses++; |
| 1293 } | 1294 } |
| 1294 } | 1295 } |
| 1295 } | 1296 } |
| OLD | NEW |