| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 966 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 977 | 977 |
| 978 /** Modifies the --timeout=XX parameter passed to run_selenium.py */ | 978 /** Modifies the --timeout=XX parameter passed to run_selenium.py */ |
| 979 List<String> _modifySeleniumTimeout(List<String> arguments, int timeout) { | 979 List<String> _modifySeleniumTimeout(List<String> arguments, int timeout) { |
| 980 return arguments.map((argument) { | 980 return arguments.map((argument) { |
| 981 if (argument.startsWith('--timeout=')) { | 981 if (argument.startsWith('--timeout=')) { |
| 982 return "--timeout=$timeout"; | 982 return "--timeout=$timeout"; |
| 983 } else { | 983 } else { |
| 984 return argument; | 984 return argument; |
| 985 } | 985 } |
| 986 }).toList(); | 986 }).toList(); |
| 987 return; | |
| 988 } | 987 } |
| 989 | 988 |
| 990 | 989 |
| 991 /** | 990 /** |
| 992 * A RunningProcess actually runs a test, getting the command lines from | 991 * A RunningProcess actually runs a test, getting the command lines from |
| 993 * its [TestCase], starting the test process (and first, a compilation | 992 * its [TestCase], starting the test process (and first, a compilation |
| 994 * process if the TestCase is a [BrowserTestCase]), creating a timeout | 993 * process if the TestCase is a [BrowserTestCase]), creating a timeout |
| 995 * timer, and recording the results in a new [CommandOutput] object, which it | 994 * timer, and recording the results in a new [CommandOutput] object, which it |
| 996 * attaches to the TestCase. The lifetime of the RunningProcess is limited | 995 * attaches to the TestCase. The lifetime of the RunningProcess is limited |
| 997 * to the time it takes to start the process, run the process, and record | 996 * to the time it takes to start the process, run the process, and record |
| (...skipping 202 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1200 testCase.timeout); | 1199 testCase.timeout); |
| 1201 _process.stdin.write(line); | 1200 _process.stdin.write(line); |
| 1202 _stdoutSubscription.resume(); | 1201 _stdoutSubscription.resume(); |
| 1203 _stderrSubscription.resume(); | 1202 _stderrSubscription.resume(); |
| 1204 Future.wait([_stdoutCompleter.future, | 1203 Future.wait([_stdoutCompleter.future, |
| 1205 _stderrCompleter.future]).then((_) => _reportResult()); | 1204 _stderrCompleter.future]).then((_) => _reportResult()); |
| 1206 } | 1205 } |
| 1207 | 1206 |
| 1208 String _createArgumentsLine(List<String> arguments, int timeout) { | 1207 String _createArgumentsLine(List<String> arguments, int timeout) { |
| 1209 arguments = _modifySeleniumTimeout(arguments, timeout); | 1208 arguments = _modifySeleniumTimeout(arguments, timeout); |
| 1210 return arguments.join(' ').concat('\n'); | 1209 return arguments.join(' ') + '\n'; |
| 1211 } | 1210 } |
| 1212 | 1211 |
| 1213 void _reportResult() { | 1212 void _reportResult() { |
| 1214 if (!active) return; | 1213 if (!active) return; |
| 1215 // _status == '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}' | 1214 // _status == '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}' |
| 1216 | 1215 |
| 1217 var outcome = _status.split(" ")[2]; | 1216 var outcome = _status.split(" ")[2]; |
| 1218 var exitCode = 0; | 1217 var exitCode = 0; |
| 1219 if (outcome == "CRASH") exitCode = CRASHING_BROWSER_EXITCODE; | 1218 if (outcome == "CRASH") exitCode = CRASHING_BROWSER_EXITCODE; |
| 1220 if (outcome == "FAIL" || outcome == "TIMEOUT") exitCode = 1; | 1219 if (outcome == "FAIL" || outcome == "TIMEOUT") exitCode = 1; |
| (...skipping 534 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1755 test.usesWebDriver && | 1754 test.usesWebDriver && |
| 1756 !test.configuration['noBatch']; | 1755 !test.configuration['noBatch']; |
| 1757 if (useBatchRunnerForAnalyzer || isWebdriverCommand) { | 1756 if (useBatchRunnerForAnalyzer || isWebdriverCommand) { |
| 1758 TestCaseEvent oldCallback = test.completedHandler; | 1757 TestCaseEvent oldCallback = test.completedHandler; |
| 1759 void testCompleted(TestCase test_arg) { | 1758 void testCompleted(TestCase test_arg) { |
| 1760 _numProcesses--; | 1759 _numProcesses--; |
| 1761 if (isBrowserCommand) { | 1760 if (isBrowserCommand) { |
| 1762 _numBrowserProcesses--; | 1761 _numBrowserProcesses--; |
| 1763 } | 1762 } |
| 1764 eventFinishedTestCase(test_arg); | 1763 eventFinishedTestCase(test_arg); |
| 1765 if (test_arg is BrowserTestCase) test_arg.notifyObservers(); | 1764 if (test_arg is BrowserTestCase) { |
| 1765 (test_arg as BrowserTestCase).notifyObservers(); |
| 1766 } |
| 1766 oldCallback(test_arg); | 1767 oldCallback(test_arg); |
| 1767 _tryRunTest(); | 1768 _tryRunTest(); |
| 1768 }; | 1769 }; |
| 1769 test.completedHandler = testCompleted; | 1770 test.completedHandler = testCompleted; |
| 1770 if (test.usesBrowserController) { | 1771 if (test.usesBrowserController) { |
| 1771 _startBrowserControllerTest(test); | 1772 _startBrowserControllerTest(test); |
| 1772 } else { | 1773 } else { |
| 1773 _getBatchRunner(test).startTest(test); | 1774 _getBatchRunner(test).startTest(test); |
| 1774 } | 1775 } |
| 1775 } else { | 1776 } else { |
| 1776 // Once we've actually failed a test, technically, we wouldn't need to | 1777 // Once we've actually failed a test, technically, we wouldn't need to |
| 1777 // bother retrying any subsequent tests since the bot is already red. | 1778 // bother retrying any subsequent tests since the bot is already red. |
| 1778 // However, we continue to retry tests until we have actually failed | 1779 // However, we continue to retry tests until we have actually failed |
| 1779 // four tests (arbitrarily chosen) for more debugable output, so that | 1780 // four tests (arbitrarily chosen) for more debugable output, so that |
| 1780 // the developer doesn't waste his or her time trying to fix a bunch of | 1781 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1781 // tests that appear to be broken but were actually just flakes that | 1782 // tests that appear to be broken but were actually just flakes that |
| 1782 // didn't get retried because there had already been one failure. | 1783 // didn't get retried because there had already been one failure. |
| 1783 bool allowRetry = _MAX_FAILED_NO_RETRY > _numFailedTests; | 1784 bool allowRetry = _MAX_FAILED_NO_RETRY > _numFailedTests; |
| 1784 runNextCommandWithRetries(test, allowRetry).then((TestCase testCase) { | 1785 runNextCommandWithRetries(test, allowRetry).then((TestCase testCase) { |
| 1785 _numProcesses--; | 1786 _numProcesses--; |
| 1786 if (isBrowserCommand) { | 1787 if (isBrowserCommand) { |
| 1787 _numBrowserProcesses--; | 1788 _numBrowserProcesses--; |
| 1788 } | 1789 } |
| 1789 if (isTestCaseFinished(testCase)) { | 1790 if (isTestCaseFinished(testCase)) { |
| 1790 testCase.completed(); | 1791 testCase.completed(); |
| 1791 eventFinishedTestCase(testCase); | 1792 eventFinishedTestCase(testCase); |
| 1792 if (testCase is BrowserTestCase) testCase.notifyObservers(); | 1793 if (testCase is BrowserTestCase) { |
| 1794 (testCase as BrowserTestCase).notifyObservers(); |
| 1795 } |
| 1793 } else { | 1796 } else { |
| 1794 _tests.addFirst(testCase); | 1797 _tests.addFirst(testCase); |
| 1795 } | 1798 } |
| 1796 _tryRunTest(); | 1799 _tryRunTest(); |
| 1797 }); | 1800 }); |
| 1798 } | 1801 } |
| 1799 | 1802 |
| 1800 _numProcesses++; | 1803 _numProcesses++; |
| 1801 if (isBrowserCommand) { | 1804 if (isBrowserCommand) { |
| 1802 _numBrowserProcesses++; | 1805 _numBrowserProcesses++; |
| (...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1910 } | 1913 } |
| 1911 } | 1914 } |
| 1912 | 1915 |
| 1913 void eventAllTestsDone() { | 1916 void eventAllTestsDone() { |
| 1914 for (var listener in _eventListener) { | 1917 for (var listener in _eventListener) { |
| 1915 listener.allDone(); | 1918 listener.allDone(); |
| 1916 } | 1919 } |
| 1917 } | 1920 } |
| 1918 } | 1921 } |
| 1919 | 1922 |
| OLD | NEW |