Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(284)

Side by Side Diff: tools/testing/dart/test_runner.dart

Issue 11343008: Land update to tools directory with new binaries. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 8 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « tools/testing/dart/test_progress.dart ('k') | tools/testing/dart/test_suite.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /** 5 /**
6 * Classes and methods for executing tests. 6 * Classes and methods for executing tests.
7 * 7 *
8 * This module includes: 8 * This module includes:
9 * - Managing parallel execution of tests, including timeout checks. 9 * - Managing parallel execution of tests, including timeout checks.
10 * - Evaluating the output of each test as pass/fail/crash/timeout. 10 * - Evaluating the output of each test as pass/fail/crash/timeout.
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after
97 // We generate a new command-line that is the special command where we 97 // We generate a new command-line that is the special command where we
98 // replace '@' with the original command executable, and generate 98 // replace '@' with the original command executable, and generate
99 // a command formed like the following 99 // a command formed like the following
100 // Let PREFIX be what is before the @. 100 // Let PREFIX be what is before the @.
101 // Let SUFFIX be what is after the @. 101 // Let SUFFIX be what is after the @.
102 // Let EXECUTABLE be the existing executable of the command. 102 // Let EXECUTABLE be the existing executable of the command.
103 // Let ARGUMENTS be the existing arguments to the existing executable. 103 // Let ARGUMENTS be the existing arguments to the existing executable.
104 // The new command will be: 104 // The new command will be:
105 // PREFIX EXECUTABLE SUFFIX ARGUMENTS 105 // PREFIX EXECUTABLE SUFFIX ARGUMENTS
106 var specialCommand = configuration['special-command']; 106 var specialCommand = configuration['special-command'];
107 if (!specialCommand.isEmpty()) { 107 if (!specialCommand.isEmpty) {
108 Expect.isTrue(specialCommand.contains('@'), 108 Expect.isTrue(specialCommand.contains('@'),
109 "special-command must contain a '@' char"); 109 "special-command must contain a '@' char");
110 var specialCommandSplit = specialCommand.split('@'); 110 var specialCommandSplit = specialCommand.split('@');
111 var prefix = specialCommandSplit[0].trim(); 111 var prefix = specialCommandSplit[0].trim();
112 var suffix = specialCommandSplit[1].trim(); 112 var suffix = specialCommandSplit[1].trim();
113 List<Command> newCommands = []; 113 List<Command> newCommands = [];
114 for (Command c in commands) { 114 for (Command c in commands) {
115 // If we don't have a new prefix we will use the existing executable. 115 // If we don't have a new prefix we will use the existing executable.
116 var newExecutablePath = c.executable;; 116 var newExecutablePath = c.executable;;
117 var newArguments = []; 117 var newArguments = [];
118 118
119 if (prefix.length > 0) { 119 if (prefix.length > 0) {
120 var prefixSplit = prefix.split(' '); 120 var prefixSplit = prefix.split(' ');
121 newExecutablePath = prefixSplit[0]; 121 newExecutablePath = prefixSplit[0];
122 for (int i = 1; i < prefixSplit.length; i++) { 122 for (int i = 1; i < prefixSplit.length; i++) {
123 var current = prefixSplit[i]; 123 var current = prefixSplit[i];
124 if (!current.isEmpty()) newArguments.add(current); 124 if (!current.isEmpty) newArguments.add(current);
125 } 125 }
126 newArguments.add(c.executable); 126 newArguments.add(c.executable);
127 } 127 }
128 128
129 // Add any suffixes to the arguments of the original executable. 129 // Add any suffixes to the arguments of the original executable.
130 var suffixSplit = suffix.split(' '); 130 var suffixSplit = suffix.split(' ');
131 suffixSplit.forEach((e) { 131 suffixSplit.forEach((e) {
132 if (!e.isEmpty()) newArguments.add(e); 132 if (!e.isEmpty) newArguments.add(e);
133 }); 133 });
134 134
135 newArguments.addAll(c.arguments); 135 newArguments.addAll(c.arguments);
136 final newCommand = new Command(newExecutablePath, newArguments); 136 final newCommand = new Command(newExecutablePath, newArguments);
137 newCommands.add(newCommand); 137 newCommands.add(newCommand);
138 // If there are extra spaces inside the prefix or suffix, this fails. 138 // If there are extra spaces inside the prefix or suffix, this fails.
139 String expected = 139 String expected =
140 '$prefix ${c.executable} $suffix ${Strings.join(c.arguments, ' ')}'; 140 '$prefix ${c.executable} $suffix ${Strings.join(c.arguments, ' ')}';
141 Expect.stringEquals(expected.trim(), newCommand.commandLine); 141 Expect.stringEquals(expected.trim(), newCommand.commandLine);
142 } 142 }
(...skipping 12 matching lines...) Expand all
155 String get configurationString { 155 String get configurationString {
156 final compiler = configuration['compiler']; 156 final compiler = configuration['compiler'];
157 final runtime = configuration['runtime']; 157 final runtime = configuration['runtime'];
158 final mode = configuration['mode']; 158 final mode = configuration['mode'];
159 final arch = configuration['arch']; 159 final arch = configuration['arch'];
160 final checked = configuration['checked'] ? '-checked' : ''; 160 final checked = configuration['checked'] ? '-checked' : '';
161 return "$compiler-$runtime$checked ${mode}_$arch"; 161 return "$compiler-$runtime$checked ${mode}_$arch";
162 } 162 }
163 163
164 List<String> get batchRunnerArguments => ['-batch']; 164 List<String> get batchRunnerArguments => ['-batch'];
165 List<String> get batchTestArguments => commands.last().arguments; 165 List<String> get batchTestArguments => commands.last.arguments;
166 166
167 bool get usesWebDriver => TestUtils.usesWebDriver(configuration['runtime']); 167 bool get usesWebDriver => TestUtils.usesWebDriver(configuration['runtime']);
168 168
169 void completed() { completedHandler(this); } 169 void completed() { completedHandler(this); }
170 } 170 }
171 171
172 172
173 /** 173 /**
174 * BrowserTestCase has an extra compilation command that is run in a separate 174 * BrowserTestCase has an extra compilation command that is run in a separate
175 * process, before the regular test is run as in the base class [TestCase]. 175 * process, before the regular test is run as in the base class [TestCase].
176 * If the compilation command fails, then the rest of the test is not run. 176 * If the compilation command fails, then the rest of the test is not run.
177 */ 177 */
178 class BrowserTestCase extends TestCase { 178 class BrowserTestCase extends TestCase {
179 /** 179 /**
180 * Indicates the number of potential retries remaining, to compensate for 180 * Indicates the number of potential retries remaining, to compensate for
181 * flaky browser tests. 181 * flaky browser tests.
182 */ 182 */
183 int numRetries; 183 int numRetries;
184 184
185 BrowserTestCase(displayName, commands, configuration, completedHandler, 185 BrowserTestCase(displayName, commands, configuration, completedHandler,
186 expectedOutcomes, info, isNegative) 186 expectedOutcomes, info, isNegative)
187 : super(displayName, commands, configuration, completedHandler, 187 : super(displayName, commands, configuration, completedHandler,
188 expectedOutcomes, isNegative: isNegative, info: info) { 188 expectedOutcomes, isNegative: isNegative, info: info) {
189 numRetries = 2; // Allow two retries to compensate for flaky browser tests. 189 numRetries = 2; // Allow two retries to compensate for flaky browser tests.
190 } 190 }
191 191
192 List<String> get _lastArguments => commands.last().arguments; 192 List<String> get _lastArguments => commands.last.arguments;
193 193
194 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch']; 194 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch'];
195 195
196 List<String> get batchTestArguments => 196 List<String> get batchTestArguments =>
197 _lastArguments.getRange(1, _lastArguments.length - 1); 197 _lastArguments.getRange(1, _lastArguments.length - 1);
198 } 198 }
199 199
200 200
201 /** 201 /**
202 * TestOutput records the output of a completed test: the process's exit code, 202 * TestOutput records the output of a completed test: the process's exit code,
(...skipping 226 matching lines...) Expand 10 before | Expand all | Expand 10 after
429 } 429 }
430 430
431 bool _didMultitestFail(List errors, List staticWarnings) { 431 bool _didMultitestFail(List errors, List staticWarnings) {
432 Set<String> outcome = testCase.info.multitestOutcome; 432 Set<String> outcome = testCase.info.multitestOutcome;
433 Expect.isNotNull(outcome); 433 Expect.isNotNull(outcome);
434 if (outcome.contains('compile-time error') && errors.length > 0) { 434 if (outcome.contains('compile-time error') && errors.length > 0) {
435 return true; 435 return true;
436 } else if (outcome.contains('static type warning') 436 } else if (outcome.contains('static type warning')
437 && staticWarnings.length > 0) { 437 && staticWarnings.length > 0) {
438 return true; 438 return true;
439 } else if (outcome.isEmpty() 439 } else if (outcome.isEmpty
440 && (errors.length > 0 || staticWarnings.length > 0)) { 440 && (errors.length > 0 || staticWarnings.length > 0)) {
441 return true; 441 return true;
442 } 442 }
443 return false; 443 return false;
444 } 444 }
445 445
446 bool _didStandardTestFail(List errors, List staticWarnings) { 446 bool _didStandardTestFail(List errors, List staticWarnings) {
447 bool hasFatalTypeErrors = false; 447 bool hasFatalTypeErrors = false;
448 int numStaticTypeAnnotations = 0; 448 int numStaticTypeAnnotations = 0;
449 int numCompileTimeAnnotations = 0; 449 int numCompileTimeAnnotations = 0;
(...skipping 283 matching lines...) Expand 10 before | Expand all | Expand 10 after
733 String _status; 733 String _status;
734 bool _stdoutDrained = false; 734 bool _stdoutDrained = false;
735 bool _stderrDrained = false; 735 bool _stderrDrained = false;
736 MutableValue<bool> _ignoreStreams; 736 MutableValue<bool> _ignoreStreams;
737 Date _startTime; 737 Date _startTime;
738 Timer _timer; 738 Timer _timer;
739 739
740 bool _isWebDriver; 740 bool _isWebDriver;
741 741
742 BatchRunnerProcess(TestCase testCase) { 742 BatchRunnerProcess(TestCase testCase) {
743 _executable = testCase.commands.last().executable; 743 _executable = testCase.commands.last.executable;
744 _batchArguments = testCase.batchRunnerArguments; 744 _batchArguments = testCase.batchRunnerArguments;
745 _isWebDriver = testCase.usesWebDriver; 745 _isWebDriver = testCase.usesWebDriver;
746 } 746 }
747 747
748 bool get active => _currentTest != null; 748 bool get active => _currentTest != null;
749 749
750 void startTest(TestCase testCase) { 750 void startTest(TestCase testCase) {
751 Expect.isNull(_currentTest); 751 Expect.isNull(_currentTest);
752 _currentTest = testCase; 752 _currentTest = testCase;
753 if (_process === null) { 753 if (_process === null) {
754 // Start process if not yet started. 754 // Start process if not yet started.
755 _executable = testCase.commands.last().executable; 755 _executable = testCase.commands.last.executable;
756 _startProcess(() { 756 _startProcess(() {
757 doStartTest(testCase); 757 doStartTest(testCase);
758 }); 758 });
759 } else if (testCase.commands.last().executable != _executable) { 759 } else if (testCase.commands.last.executable != _executable) {
760 // Restart this runner with the right executable for this test 760 // Restart this runner with the right executable for this test
761 // if needed. 761 // if needed.
762 _executable = testCase.commands.last().executable; 762 _executable = testCase.commands.last.executable;
763 _batchArguments = testCase.batchRunnerArguments; 763 _batchArguments = testCase.batchRunnerArguments;
764 _process.onExit = (exitCode) { 764 _process.onExit = (exitCode) {
765 _process.close(); 765 _process.close();
766 _startProcess(() { 766 _startProcess(() {
767 doStartTest(testCase); 767 doStartTest(testCase);
768 }); 768 });
769 }; 769 };
770 _process.kill(); 770 _process.kill();
771 } else { 771 } else {
772 doStartTest(testCase); 772 doStartTest(testCase);
773 } 773 }
774 } 774 }
775 775
776 Future terminate() { 776 Future terminate() {
777 if (_process == null) return new Future.immediate(true); 777 if (_process == null) return new Future.immediate(true);
778 Completer completer = new Completer(); 778 Completer completer = new Completer();
779 Timer killTimer; 779 Timer killTimer;
780 _process.onExit = (exitCode) { 780 _process.onExit = (exitCode) {
781 _process.close(); 781 _process.close();
782 if (killTimer != null) killTimer.cancel(); 782 if (killTimer != null) killTimer.cancel();
783 completer.complete(true); 783 completer.complete(true);
784 }; 784 };
785 if (_isWebDriver) { 785 if (_isWebDriver) {
786 // Use a graceful shutdown so our Selenium script can close 786 // Use a graceful shutdown so our Selenium script can close
787 // the open browser processes. On Windows, signals do not exist 787 // the open browser processes. On Windows, signals do not exist
788 // and a kill is a hard kill. 788 // and a kill is a hard kill.
789 _process.stdin.write('--terminate\n'.charCodes()); 789 _process.stdin.write('--terminate\n'.charCodes);
790 790
791 // In case the run_selenium process didn't close, kill it after 30s 791 // In case the run_selenium process didn't close, kill it after 30s
792 int shutdownMillisecs = 30000; 792 int shutdownMillisecs = 30000;
793 killTimer = new Timer(shutdownMillisecs, (e) { _process.kill(); }); 793 killTimer = new Timer(shutdownMillisecs, (e) { _process.kill(); });
794 } else { 794 } else {
795 _process.kill(); 795 _process.kill();
796 } 796 }
797 797
798 return completer.future; 798 return completer.future;
799 } 799 }
(...skipping 10 matching lines...) Expand all
810 _stderrStream.onLine = _readStderr(_stderrStream, _testStderr); 810 _stderrStream.onLine = _readStderr(_stderrStream, _testStderr);
811 _timer = new Timer(testCase.timeout * 1000, _timeoutHandler); 811 _timer = new Timer(testCase.timeout * 1000, _timeoutHandler);
812 var line = _createArgumentsLine(testCase.batchTestArguments); 812 var line = _createArgumentsLine(testCase.batchTestArguments);
813 _process.stdin.onError = (err) { 813 _process.stdin.onError = (err) {
814 print('Error on batch runner input stream stdin'); 814 print('Error on batch runner input stream stdin');
815 print(' Input line: $line'); 815 print(' Input line: $line');
816 print(' Previous test\'s status: $_status'); 816 print(' Previous test\'s status: $_status');
817 print(' Error: $err'); 817 print(' Error: $err');
818 throw err; 818 throw err;
819 }; 819 };
820 _process.stdin.write(line.charCodes()); 820 _process.stdin.write(line.charCodes);
821 } 821 }
822 822
823 String _createArgumentsLine(List<String> arguments) { 823 String _createArgumentsLine(List<String> arguments) {
824 return Strings.join(arguments, ' ').concat('\n'); 824 return Strings.join(arguments, ' ').concat('\n');
825 } 825 }
826 826
827 void _reportResult() { 827 void _reportResult() {
828 if (!active) return; 828 if (!active) return;
829 // _status == '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}' 829 // _status == '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}'
830 830
(...skipping 233 matching lines...) Expand 10 before | Expand all | Expand 10 after
1064 1064
1065 void _checkDone() { 1065 void _checkDone() {
1066 // When there are no more active test listers ask for more work 1066 // When there are no more active test listers ask for more work
1067 // from process queue users. 1067 // from process queue users.
1068 if (_activeTestListers == 0) { 1068 if (_activeTestListers == 0) {
1069 _enqueueMoreWork(this); 1069 _enqueueMoreWork(this);
1070 } 1070 }
1071 // If there is still no work, we are done. 1071 // If there is still no work, we are done.
1072 if (_activeTestListers == 0) { 1072 if (_activeTestListers == 0) {
1073 _progress.allTestsKnown(); 1073 _progress.allTestsKnown();
1074 if (_tests.isEmpty() && _numProcesses == 0) { 1074 if (_tests.isEmpty && _numProcesses == 0) {
1075 _terminateBatchRunners().then((_) => _cleanupAndMarkDone()); 1075 _terminateBatchRunners().then((_) => _cleanupAndMarkDone());
1076 } 1076 }
1077 } 1077 }
1078 } 1078 }
1079 1079
1080 /** 1080 /**
1081 * True if we are using a browser + platform combination that needs the 1081 * True if we are using a browser + platform combination that needs the
1082 * Selenium server jar. 1082 * Selenium server jar.
1083 */ 1083 */
1084 bool get _needsSelenium => Platform.operatingSystem == 'macos' && 1084 bool get _needsSelenium => Platform.operatingSystem == 'macos' &&
(...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after
1207 // TODO(ahe): How to report this as a test failure? 1207 // TODO(ahe): How to report this as a test failure?
1208 exit(1); 1208 exit(1);
1209 return true; 1209 return true;
1210 }); 1210 });
1211 } 1211 }
1212 }; 1212 };
1213 } 1213 }
1214 1214
1215 Future _terminateBatchRunners() { 1215 Future _terminateBatchRunners() {
1216 var futures = new List(); 1216 var futures = new List();
1217 for (var runners in _batchProcesses.getValues()) { 1217 for (var runners in _batchProcesses.values) {
1218 for (var runner in runners) { 1218 for (var runner in runners) {
1219 futures.add(runner.terminate()); 1219 futures.add(runner.terminate());
1220 } 1220 }
1221 } 1221 }
1222 return Futures.wait(futures); 1222 return Futures.wait(futures);
1223 } 1223 }
1224 1224
1225 BatchRunnerProcess _getBatchRunner(TestCase test) { 1225 BatchRunnerProcess _getBatchRunner(TestCase test) {
1226 // Start batch processes if needed 1226 // Start batch processes if needed
1227 var compiler = test.configuration['compiler']; 1227 var compiler = test.configuration['compiler'];
1228 var runners = _batchProcesses[compiler]; 1228 var runners = _batchProcesses[compiler];
1229 if (runners == null) { 1229 if (runners == null) {
1230 runners = new List<BatchRunnerProcess>(_maxProcesses); 1230 runners = new List<BatchRunnerProcess>(_maxProcesses);
1231 for (int i = 0; i < _maxProcesses; i++) { 1231 for (int i = 0; i < _maxProcesses; i++) {
1232 runners[i] = new BatchRunnerProcess(test); 1232 runners[i] = new BatchRunnerProcess(test);
1233 } 1233 }
1234 _batchProcesses[compiler] = runners; 1234 _batchProcesses[compiler] = runners;
1235 } 1235 }
1236 1236
1237 for (var runner in runners) { 1237 for (var runner in runners) {
1238 if (!runner.active) return runner; 1238 if (!runner.active) return runner;
1239 } 1239 }
1240 throw new Exception('Unable to find inactive batch runner.'); 1240 throw new Exception('Unable to find inactive batch runner.');
1241 } 1241 }
1242 1242
1243 void _tryRunTest() { 1243 void _tryRunTest() {
1244 _checkDone(); 1244 _checkDone();
1245 if (_numProcesses < _maxProcesses && !_tests.isEmpty()) { 1245 if (_numProcesses < _maxProcesses && !_tests.isEmpty) {
1246 TestCase test = _tests.removeFirst(); 1246 TestCase test = _tests.removeFirst();
1247 if (_listTests) { 1247 if (_listTests) {
1248 var fields = [test.displayName, 1248 var fields = [test.displayName,
1249 Strings.join(new List.from(test.expectedOutcomes), ','), 1249 Strings.join(new List.from(test.expectedOutcomes), ','),
1250 test.isNegative.toString()]; 1250 test.isNegative.toString()];
1251 fields.addAll(test.commands.last().arguments); 1251 fields.addAll(test.commands.last.arguments);
1252 print(Strings.join(fields, '\t')); 1252 print(Strings.join(fields, '\t'));
1253 return; 1253 return;
1254 } 1254 }
1255 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) { 1255 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) {
1256 // The server is not ready to run Selenium tests. Put the test back in 1256 // The server is not ready to run Selenium tests. Put the test back in
1257 // the queue. Avoid spin-polling by using a timeout. 1257 // the queue. Avoid spin-polling by using a timeout.
1258 _tests.add(test); 1258 _tests.add(test);
1259 new Timer(1000, (timer) {_tryRunTest();}); // Don't lose a process. 1259 new Timer(1000, (timer) {_tryRunTest();}); // Don't lose a process.
1260 return; 1260 return;
1261 } 1261 }
(...skipping 24 matching lines...) Expand all
1286 // the developer doesn't waste his or her time trying to fix a bunch of 1286 // the developer doesn't waste his or her time trying to fix a bunch of
1287 // tests that appear to be broken but were actually just flakes that 1287 // tests that appear to be broken but were actually just flakes that
1288 // didn't get retried because there had already been one failure. 1288 // didn't get retried because there had already been one failure.
1289 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; 1289 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
1290 new RunningProcess(test, allowRetry, this).start(); 1290 new RunningProcess(test, allowRetry, this).start();
1291 } 1291 }
1292 _numProcesses++; 1292 _numProcesses++;
1293 } 1293 }
1294 } 1294 }
1295 } 1295 }
OLDNEW
« no previous file with comments | « tools/testing/dart/test_progress.dart ('k') | tools/testing/dart/test_suite.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698