Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(17)

Side by Side Diff: tools/testing/dart/test_runner.dart

Issue 12751005: Reapply "Update the test runner to use the new dart:io API" (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 7 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « tools/testing/dart/test_progress.dart ('k') | tools/testing/dart/test_suite.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /** 5 /**
6 * Classes and methods for executing tests. 6 * Classes and methods for executing tests.
7 * 7 *
8 * This module includes: 8 * This module includes:
9 * - Managing parallel execution of tests, including timeout checks. 9 * - Managing parallel execution of tests, including timeout checks.
10 * - Evaluating the output of each test as pass/fail/crash/timeout. 10 * - Evaluating the output of each test as pass/fail/crash/timeout.
(...skipping 658 matching lines...) Expand 10 before | Expand all | Expand 10 after
669 * #EOF\n 669 * #EOF\n
670 * So we need to get the byte-range of the png data first, before 670 * So we need to get the byte-range of the png data first, before
671 * comparing it with the content of the expected output file. 671 * comparing it with the content of the expected output file.
672 * 672 *
673 * On a layout tests, the DRT output is directly compared with the 673 * On a layout tests, the DRT output is directly compared with the
674 * content of the expected output. 674 * content of the expected output.
675 */ 675 */
676 var stdout = testCase.commandOutputs[command].stdout; 676 var stdout = testCase.commandOutputs[command].stdout;
677 var file = new io.File.fromPath(command.expectedOutputFile); 677 var file = new io.File.fromPath(command.expectedOutputFile);
678 if (file.existsSync()) { 678 if (file.existsSync()) {
679 var bytesContentLength = "Content-Length:".charCodes; 679 var bytesContentLength = "Content-Length:".codeUnits;
680 var bytesNewLine = "\n".charCodes; 680 var bytesNewLine = "\n".codeUnits;
681 var bytesEOF = "#EOF\n".charCodes; 681 var bytesEOF = "#EOF\n".codeUnits;
682 682
683 var expectedContent = file.readAsBytesSync(); 683 var expectedContent = file.readAsBytesSync();
684 if (command.isPixelTest) { 684 if (command.isPixelTest) {
685 var startOfContentLength = findBytes(stdout, bytesContentLength); 685 var startOfContentLength = findBytes(stdout, bytesContentLength);
686 if (startOfContentLength >= 0) { 686 if (startOfContentLength >= 0) {
687 var newLineAfterContentLength = findBytes(stdout, 687 var newLineAfterContentLength = findBytes(stdout,
688 bytesNewLine, 688 bytesNewLine,
689 startOfContentLength); 689 startOfContentLength);
690 if (newLineAfterContentLength > 0) { 690 if (newLineAfterContentLength > 0) {
691 var startPosition = newLineAfterContentLength + 691 var startPosition = newLineAfterContentLength +
(...skipping 210 matching lines...) Expand 10 before | Expand all | Expand 10 after
902 bool escaped = false; 902 bool escaped = false;
903 for (var i = 0 ; i < line.length; i++) { 903 for (var i = 0 ; i < line.length; i++) {
904 var c = line[i]; 904 var c = line[i];
905 if (!escaped && c == '\\') { 905 if (!escaped && c == '\\') {
906 escaped = true; 906 escaped = true;
907 continue; 907 continue;
908 } 908 }
909 escaped = false; 909 escaped = false;
910 if (c == '|') { 910 if (c == '|') {
911 result.add(field.toString()); 911 result.add(field.toString());
912 field.clear(); 912 field = new StringBuffer();
913 continue; 913 continue;
914 } 914 }
915 field.add(c); 915 field.write(c);
916 } 916 }
917 result.add(field.toString()); 917 result.add(field.toString());
918 return result; 918 return result;
919 } 919 }
920 } 920 }
921 921
922 /** 922 /**
923 * A RunningProcess actually runs a test, getting the command lines from 923 * A RunningProcess actually runs a test, getting the command lines from
924 * its [TestCase], starting the test process (and first, a compilation 924 * its [TestCase], starting the test process (and first, a compilation
925 * process if the TestCase is a [BrowserTestCase]), creating a timeout 925 * process if the TestCase is a [BrowserTestCase]), creating a timeout
926 * timer, and recording the results in a new [CommandOutput] object, which it 926 * timer, and recording the results in a new [CommandOutput] object, which it
927 * attaches to the TestCase. The lifetime of the RunningProcess is limited 927 * attaches to the TestCase. The lifetime of the RunningProcess is limited
928 * to the time it takes to start the process, run the process, and record 928 * to the time it takes to start the process, run the process, and record
929 * the result; there are no pointers to it, so it should be available to 929 * the result; there are no pointers to it, so it should be available to
930 * be garbage collected as soon as it is done. 930 * be garbage collected as soon as it is done.
931 */ 931 */
932 class RunningProcess { 932 class RunningProcess {
933 TestCase testCase; 933 TestCase testCase;
934 Command command; 934 Command command;
935 bool timedOut = false; 935 bool timedOut = false;
936 Date startTime; 936 DateTime startTime;
937 Timer timeoutTimer; 937 Timer timeoutTimer;
938 List<int> stdout = <int>[]; 938 List<int> stdout = <int>[];
939 List<int> stderr = <int>[]; 939 List<int> stderr = <int>[];
940 bool compilationSkipped = false; 940 bool compilationSkipped = false;
941 Completer<CommandOutput> completer; 941 Completer<CommandOutput> completer;
942 942
943 RunningProcess(TestCase this.testCase, Command this.command); 943 RunningProcess(TestCase this.testCase, Command this.command);
944 944
945 Future<CommandOutput> start() { 945 Future<CommandOutput> start() {
946 Expect.isFalse(testCase.expectedOutcomes.contains(SKIP)); 946 Expect.isFalse(testCase.expectedOutcomes.contains(SKIP));
947 947
948 completer = new Completer<CommandOutput>(); 948 completer = new Completer<CommandOutput>();
949 startTime = new Date.now(); 949 startTime = new DateTime.now();
950 _runCommand(); 950 _runCommand();
951 return completer.future; 951 return completer.future;
952 } 952 }
953 953
954 void _runCommand() { 954 void _runCommand() {
955 command.outputIsUpToDate.then((bool isUpToDate) { 955 command.outputIsUpToDate.then((bool isUpToDate) {
956 if (isUpToDate) { 956 if (isUpToDate) {
957 compilationSkipped = true; 957 compilationSkipped = true;
958 _commandComplete(0); 958 _commandComplete(0);
959 } else { 959 } else {
960 var processOptions = _createProcessOptions(); 960 var processOptions = _createProcessOptions();
961 Future processFuture = io.Process.start(command.executable, 961 Future processFuture = io.Process.start(command.executable,
962 command.arguments, 962 command.arguments,
963 processOptions); 963 processOptions);
964 processFuture.then((io.Process process) { 964 processFuture.then((io.Process process) {
965 void timeoutHandler(_) { 965 void timeoutHandler() {
966 timedOut = true; 966 timedOut = true;
967 if (process != null) { 967 if (process != null) {
968 process.kill(); 968 process.kill();
969 } 969 }
970 } 970 }
971 process.onExit = _commandComplete; 971 process.exitCode.then(_commandComplete);
972 _drainStream(process.stdout, stdout); 972 _drainStream(process.stdout, stdout);
973 _drainStream(process.stderr, stderr); 973 _drainStream(process.stderr, stderr);
974 timeoutTimer = new Timer(1000 * testCase.timeout, timeoutHandler); 974 timeoutTimer = new Timer(new Duration(seconds: testCase.timeout),
975 timeoutHandler);
975 }).catchError((e) { 976 }).catchError((e) {
976 print("Process error:"); 977 print("Process error:");
977 print(" Command: $command"); 978 print(" Command: $command");
978 print(" Error: $e"); 979 print(" Error: $e");
979 _commandComplete(-1); 980 _commandComplete(-1);
980 return true; 981 return true;
981 }); 982 });
982 } 983 }
983 }); 984 });
984 } 985 }
985 986
986 void _commandComplete(int exitCode) { 987 void _commandComplete(int exitCode) {
987 if (timeoutTimer != null) { 988 if (timeoutTimer != null) {
988 timeoutTimer.cancel(); 989 timeoutTimer.cancel();
989 } 990 }
990 var commandOutput = _createCommandOutput(command, exitCode); 991 var commandOutput = _createCommandOutput(command, exitCode);
991 completer.complete(commandOutput); 992 completer.complete(commandOutput);
992 } 993 }
993 994
994 CommandOutput _createCommandOutput(Command command, int exitCode) { 995 CommandOutput _createCommandOutput(Command command, int exitCode) {
995 var incomplete = command != testCase.commands.last; 996 var incomplete = command != testCase.commands.last;
996 var commandOutput = new CommandOutput.fromCase( 997 var commandOutput = new CommandOutput.fromCase(
997 testCase, 998 testCase,
998 command, 999 command,
999 exitCode, 1000 exitCode,
1000 incomplete, 1001 incomplete,
1001 timedOut, 1002 timedOut,
1002 stdout, 1003 stdout,
1003 stderr, 1004 stderr,
1004 new Date.now().difference(startTime), 1005 new DateTime.now().difference(startTime),
1005 compilationSkipped); 1006 compilationSkipped);
1006 return commandOutput; 1007 return commandOutput;
1007 } 1008 }
1008 1009
1009 void _drainStream(io.InputStream source, List<int> destination) { 1010 void _drainStream(Stream<List<int>> source, List<int> destination) {
1010 void onDataHandler () { 1011 source.listen(destination.addAll);
1011 if (source.closed) {
1012 return; // TODO(whesse): Remove when bug is fixed.
1013 }
1014 var data = source.read();
1015 while (data != null) {
1016 destination.addAll(data);
1017 data = source.read();
1018 }
1019 }
1020 source.onData = onDataHandler;
1021 source.onClosed = onDataHandler;
1022 } 1012 }
1023 1013
1024 io.ProcessOptions _createProcessOptions() { 1014 io.ProcessOptions _createProcessOptions() {
1025 var baseEnvironment = command.environment != null ? 1015 var baseEnvironment = command.environment != null ?
1026 command.environment : io.Platform.environment; 1016 command.environment : io.Platform.environment;
1027 io.ProcessOptions options = new io.ProcessOptions(); 1017 io.ProcessOptions options = new io.ProcessOptions();
1028 options.environment = new Map<String, String>.from(baseEnvironment); 1018 options.environment = new Map<String, String>.from(baseEnvironment);
1029 options.environment['DART_CONFIGURATION'] = 1019 options.environment['DART_CONFIGURATION'] =
1030 TestUtils.configurationDir(testCase.configuration); 1020 TestUtils.configurationDir(testCase.configuration);
1031 return options; 1021 return options;
1032 } 1022 }
1033 } 1023 }
1034 1024
1035 /**
1036 * This class holds a value, that can be changed. It is used when
1037 * closures need a shared value, that they can all change and read.
1038 */
1039 class MutableValue<T> {
1040 MutableValue(T this.value);
1041 T value;
1042 }
1043
1044 class BatchRunnerProcess { 1025 class BatchRunnerProcess {
1045 Command _command; 1026 Command _command;
1046 String _executable; 1027 String _executable;
1047 List<String> _batchArguments; 1028 List<String> _batchArguments;
1048 1029
1049 io.Process _process; 1030 io.Process _process;
1050 io.StringInputStream _stdoutStream; 1031 Completer _stdoutCompleter;
1051 io.StringInputStream _stderrStream; 1032 Completer _stderrCompleter;
1033 StreamSubscription<String> _stdoutSubscription;
1034 StreamSubscription<String> _stderrSubscription;
1035 Function _processExitHandler;
1052 1036
1053 TestCase _currentTest; 1037 TestCase _currentTest;
1054 List<int> _testStdout; 1038 List<int> _testStdout;
1055 List<int> _testStderr; 1039 List<int> _testStderr;
1056 String _status; 1040 String _status;
1057 bool _stdoutDrained = false; 1041 DateTime _startTime;
1058 bool _stderrDrained = false;
1059 MutableValue<bool> _ignoreStreams;
1060 Date _startTime;
1061 Timer _timer; 1042 Timer _timer;
1062 1043
1063 bool _isWebDriver; 1044 bool _isWebDriver;
1064 1045
1065 BatchRunnerProcess(TestCase testCase) { 1046 BatchRunnerProcess(TestCase testCase) {
1066 _command = testCase.commands.last; 1047 _command = testCase.commands.last;
1067 _executable = testCase.commands.last.executable; 1048 _executable = testCase.commands.last.executable;
1068 _batchArguments = testCase.batchRunnerArguments; 1049 _batchArguments = testCase.batchRunnerArguments;
1069 _isWebDriver = testCase.usesWebDriver; 1050 _isWebDriver = testCase.usesWebDriver;
1070 } 1051 }
1071 1052
1072 bool get active => _currentTest != null; 1053 bool get active => _currentTest != null;
1073 1054
1074 void startTest(TestCase testCase) { 1055 void startTest(TestCase testCase) {
1075 Expect.isNull(_currentTest); 1056 Expect.isNull(_currentTest);
1076 _currentTest = testCase; 1057 _currentTest = testCase;
1077 _command = testCase.commands.last; 1058 _command = testCase.commands.last;
1078 if (_process == null) { 1059 if (_process == null) {
1079 // Start process if not yet started. 1060 // Start process if not yet started.
1080 _executable = testCase.commands.last.executable; 1061 _executable = testCase.commands.last.executable;
1081 _startProcess(() { 1062 _startProcess(() {
1082 doStartTest(testCase); 1063 doStartTest(testCase);
1083 }); 1064 });
1084 } else if (testCase.commands.last.executable != _executable) { 1065 } else if (testCase.commands.last.executable != _executable) {
1085 // Restart this runner with the right executable for this test 1066 // Restart this runner with the right executable for this test
1086 // if needed. 1067 // if needed.
1087 _executable = testCase.commands.last.executable; 1068 _executable = testCase.commands.last.executable;
1088 _batchArguments = testCase.batchRunnerArguments; 1069 _batchArguments = testCase.batchRunnerArguments;
1089 _process.onExit = (exitCode) { 1070 _processExitHandler = (_) {
1090 _startProcess(() { 1071 _startProcess(() {
1091 doStartTest(testCase); 1072 doStartTest(testCase);
1092 }); 1073 });
1093 }; 1074 };
1094 _process.kill(); 1075 _process.kill();
1095 } else { 1076 } else {
1096 doStartTest(testCase); 1077 doStartTest(testCase);
1097 } 1078 }
1098 } 1079 }
1099 1080
1100 Future terminate() { 1081 Future terminate() {
1101 if (_process == null) return new Future.immediate(true); 1082 if (_process == null) return new Future.immediate(true);
1102 Completer completer = new Completer(); 1083 Completer completer = new Completer();
1103 Timer killTimer; 1084 Timer killTimer;
1104 _process.onExit = (exitCode) { 1085 _processExitHandler = (_) {
1105 if (killTimer != null) killTimer.cancel(); 1086 if (killTimer != null) killTimer.cancel();
1106 completer.complete(true); 1087 completer.complete(true);
1107 }; 1088 };
1108 if (_isWebDriver) { 1089 if (_isWebDriver) {
1109 // Use a graceful shutdown so our Selenium script can close 1090 // Use a graceful shutdown so our Selenium script can close
1110 // the open browser processes. On Windows, signals do not exist 1091 // the open browser processes. On Windows, signals do not exist
1111 // and a kill is a hard kill. 1092 // and a kill is a hard kill.
1112 _process.stdin.write('--terminate\n'.charCodes); 1093 _process.stdin.writeln('--terminate');
1113 1094
1114 // In case the run_selenium process didn't close, kill it after 30s 1095 // In case the run_selenium process didn't close, kill it after 30s
1115 int shutdownMillisecs = 30000; 1096 killTimer = new Timer(new Duration(seconds: 30), _process.kill);
1116 killTimer = new Timer(shutdownMillisecs, (e) { _process.kill(); });
1117 } else { 1097 } else {
1118 _process.kill(); 1098 _process.kill();
1119 } 1099 }
1120 1100
1121 return completer.future; 1101 return completer.future;
1122 } 1102 }
1123 1103
1124 void doStartTest(TestCase testCase) { 1104 void doStartTest(TestCase testCase) {
1125 _startTime = new Date.now(); 1105 _startTime = new DateTime.now();
1126 _testStdout = []; 1106 _testStdout = [];
1127 _testStderr = []; 1107 _testStderr = [];
1128 _status = null; 1108 _status = null;
1129 _stdoutDrained = false; 1109 _stdoutCompleter = new Completer();
1130 _stderrDrained = false; 1110 _stderrCompleter = new Completer();
1131 _ignoreStreams = new MutableValue<bool>(false); // Captured by closures. 1111 _timer = new Timer(new Duration(seconds: testCase.timeout),
1132 _readStdout(_stdoutStream, _testStdout); 1112 _timeoutHandler);
1133 _readStderr(_stderrStream, _testStderr);
1134 _timer = new Timer(testCase.timeout * 1000, _timeoutHandler);
1135 1113
1136 if (testCase.commands.last.environment != null) { 1114 if (testCase.commands.last.environment != null) {
1137 print("Warning: command.environment != null, but we don't support custom " 1115 print("Warning: command.environment != null, but we don't support custom "
1138 "environments for batch runner tests!"); 1116 "environments for batch runner tests!");
1139 } 1117 }
1140 1118
1141 var line = _createArgumentsLine(testCase.batchTestArguments); 1119 var line = _createArgumentsLine(testCase.batchTestArguments);
1142 _process.stdin.onError = (err) { 1120 _process.stdin.write(line);
1143 print('Error on batch runner input stream stdin'); 1121 _stdoutSubscription.resume();
1144 print(' Input line: $line'); 1122 _stderrSubscription.resume();
1145 print(' Previous test\'s status: $_status'); 1123 Future.wait([_stdoutCompleter.future,
1146 print(' Error: $err'); 1124 _stderrCompleter.future]).then((_) => _reportResult());
1147 throw err;
1148 };
1149 _process.stdin.write(line.charCodes);
1150 } 1125 }
1151 1126
1152 String _createArgumentsLine(List<String> arguments) { 1127 String _createArgumentsLine(List<String> arguments) {
1153 return arguments.join(' ').concat('\n'); 1128 return arguments.join(' ').concat('\n');
1154 } 1129 }
1155 1130
1156 void _reportResult() { 1131 void _reportResult() {
1157 if (!active) return; 1132 if (!active) return;
1158 // _status == '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}' 1133 // _status == '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}'
1159 1134
1160 var outcome = _status.split(" ")[2]; 1135 var outcome = _status.split(" ")[2];
1161 var exitCode = 0; 1136 var exitCode = 0;
1162 if (outcome == "CRASH") exitCode = CRASHING_BROWSER_EXITCODE; 1137 if (outcome == "CRASH") exitCode = CRASHING_BROWSER_EXITCODE;
1163 if (outcome == "FAIL" || outcome == "TIMEOUT") exitCode = 1; 1138 if (outcome == "FAIL" || outcome == "TIMEOUT") exitCode = 1;
1164 new CommandOutput.fromCase(_currentTest, 1139 new CommandOutput.fromCase(_currentTest,
1165 _command, 1140 _command,
1166 exitCode, 1141 exitCode,
1167 false, 1142 false,
1168 (outcome == "TIMEOUT"), 1143 (outcome == "TIMEOUT"),
1169 _testStdout, 1144 _testStdout,
1170 _testStderr, 1145 _testStderr,
1171 new Date.now().difference(_startTime), 1146 new DateTime.now().difference(_startTime),
1172 false); 1147 false);
1173 var test = _currentTest; 1148 var test = _currentTest;
1174 _currentTest = null; 1149 _currentTest = null;
1175 test.completed(); 1150 test.completed();
1176 } 1151 }
1177 1152
1178 void _stderrDone() {
1179 _stderrDrained = true;
1180 // Move on when both stdout and stderr has been drained.
1181 if (_stdoutDrained) _reportResult();
1182 }
1183
1184 void _stdoutDone() {
1185 _stdoutDrained = true;
1186 // Move on when both stdout and stderr has been drained.
1187 if (_stderrDrained) _reportResult();
1188 }
1189
1190 void _readStdout(io.StringInputStream stream, List<int> buffer) {
1191 var ignoreStreams = _ignoreStreams; // Capture this mutable object.
1192 void onLineHandler() {
1193 if (ignoreStreams.value) {
1194 while (stream.readLine() != null) {
1195 // Do nothing.
1196 }
1197 return;
1198 }
1199 // Otherwise, process output and call _reportResult() when done.
1200 var line = stream.readLine();
1201 while (line != null) {
1202 if (line.startsWith('>>> TEST')) {
1203 _status = line;
1204 } else if (line.startsWith('>>> BATCH')) {
1205 // ignore
1206 } else if (line.startsWith('>>> ')) {
1207 throw new Exception(
1208 'Unexpected command from ${testCase.configuration['compiler']} '
1209 'batch runner.');
1210 } else {
1211 buffer.addAll(encodeUtf8(line));
1212 buffer.addAll("\n".charCodes);
1213 }
1214 line = stream.readLine();
1215 }
1216 if (_status != null) {
1217 _timer.cancel();
1218 _stdoutDone();
1219 }
1220 }
1221 stream.onLine = onLineHandler;
1222 }
1223
1224 void _readStderr(io.StringInputStream stream, List<int> buffer) {
1225 var ignoreStreams = _ignoreStreams; // Capture this mutable object.
1226 void onLineHandler() {
1227 if (ignoreStreams.value) {
1228 while (stream.readLine() != null) {
1229 // Do nothing.
1230 }
1231 return;
1232 }
1233 // Otherwise, process output and call _reportResult() when done.
1234 var line = stream.readLine();
1235 while (line != null) {
1236 if (line.startsWith('>>> EOF STDERR')) {
1237 _stderrDone();
1238 } else {
1239 buffer.addAll(encodeUtf8(line));
1240 buffer.addAll("\n".charCodes);
1241 }
1242 line = stream.readLine();
1243 }
1244 }
1245 stream.onLine = onLineHandler;
1246 }
1247
1248 ExitCodeEvent makeExitHandler(String status) { 1153 ExitCodeEvent makeExitHandler(String status) {
1249 void handler(int exitCode) { 1154 void handler(int exitCode) {
1250 if (active) { 1155 if (active) {
1251 if (_timer != null) _timer.cancel(); 1156 if (_timer != null) _timer.cancel();
1252 _status = status; 1157 _status = status;
1253 // Read current content of streams, ignore any later output. 1158 _stdoutSubscription.cancel();
1254 _ignoreStreams.value = true; 1159 _stderrSubscription.cancel();
1255 var line = _stdoutStream.readLine();
1256 while (line != null) {
1257 _testStdout.add(line);
1258 line = _stdoutStream.readLine();
1259 }
1260 line = _stderrStream.readLine();
1261 while (line != null) {
1262 _testStderr.add(line);
1263 line = _stderrStream.readLine();
1264 }
1265 _stderrDrained = true;
1266 _stdoutDrained = true;
1267 _startProcess(_reportResult); 1160 _startProcess(_reportResult);
1268 } else { // No active test case running. 1161 } else { // No active test case running.
1269 _process = null; 1162 _process = null;
1270 } 1163 }
1271 } 1164 }
1272 return handler; 1165 return handler;
1273 } 1166 }
1274 1167
1275 void _timeoutHandler(ignore) { 1168 void _timeoutHandler() {
1276 _process.onExit = makeExitHandler(">>> TEST TIMEOUT"); 1169 _processExitHandler = makeExitHandler(">>> TEST TIMEOUT");
1277 _process.kill(); 1170 _process.kill();
1278 } 1171 }
1279 1172
1280 _startProcess(callback) { 1173 _startProcess(callback) {
1281 Future processFuture = io.Process.start(_executable, _batchArguments); 1174 Future processFuture = io.Process.start(_executable, _batchArguments);
1282 processFuture.then((io.Process p) { 1175 processFuture.then((io.Process p) {
1283 _process = p; 1176 _process = p;
1284 _stdoutStream = new io.StringInputStream(_process.stdout); 1177
1285 _stderrStream = new io.StringInputStream(_process.stderr); 1178 var _stdoutStream =
1286 _process.onExit = makeExitHandler(">>> TEST CRASH"); 1179 _process.stdout
1180 .transform(new io.StringDecoder())
1181 .transform(new io.LineTransformer());
1182 _stdoutSubscription = _stdoutStream.listen((String line) {
1183 if (line.startsWith('>>> TEST')) {
1184 _status = line;
1185 } else if (line.startsWith('>>> BATCH')) {
1186 // ignore
1187 } else if (line.startsWith('>>> ')) {
1188 throw new Exception(
1189 'Unexpected command from ${testCase.configuration['compiler']} '
1190 'batch runner.');
1191 } else {
1192 _testStdout.addAll(encodeUtf8(line));
1193 _testStdout.addAll("\n".codeUnits);
1194 }
1195 if (_status != null) {
1196 _stdoutSubscription.pause();
1197 _timer.cancel();
1198 _stdoutCompleter.complete(null);
1199 }
1200 });
1201 _stdoutSubscription.pause();
1202
1203 var _stderrStream =
1204 _process.stderr
1205 .transform(new io.StringDecoder())
1206 .transform(new io.LineTransformer());
1207 _stderrSubscription = _stderrStream.listen((String line) {
1208 if (line.startsWith('>>> EOF STDERR')) {
1209 _stderrSubscription.pause();
1210 _stderrCompleter.complete(null);
1211 } else {
1212 _testStderr.addAll(encodeUtf8(line));
1213 _testStderr.addAll("\n".codeUnits);
1214 }
1215 });
1216 _stderrSubscription.pause();
1217
1218 _processExitHandler = makeExitHandler(">>> TEST CRASH");
1219 _process.exitCode.then((exitCode) {
1220 _processExitHandler(exitCode);
1221 });
1222
1223 _process.stdin.done.catchError((err) {
1224 print('Error on batch runner input stream stdin');
1225 print(' Previous test\'s status: $_status');
1226 print(' Error: $err');
1227 throw err;
1228 });
1287 callback(); 1229 callback();
1288 }).catchError((e) { 1230 }).catchError((e) {
1289 print("Process error:"); 1231 print("Process error:");
1290 print(" Command: $_executable ${_batchArguments.join(' ')}"); 1232 print(" Command: $_executable ${_batchArguments.join(' ')}");
1291 print(" Error: $e"); 1233 print(" Error: $e");
1292 // If there is an error starting a batch process, chances are that 1234 // If there is an error starting a batch process, chances are that
1293 // it will always fail. So rather than re-trying a 1000+ times, we 1235 // it will always fail. So rather than re-trying a 1000+ times, we
1294 // exit. 1236 // exit.
1295 io.exit(1); 1237 io.exit(1);
1296 return true; 1238 return true;
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
1349 io.Process _seleniumServer = null; 1291 io.Process _seleniumServer = null;
1350 1292
1351 /** True if we are in the process of starting the server. */ 1293 /** True if we are in the process of starting the server. */
1352 bool _startingServer = false; 1294 bool _startingServer = false;
1353 1295
1354 /** True if we find that there is already a selenium jar running. */ 1296 /** True if we find that there is already a selenium jar running. */
1355 bool _seleniumAlreadyRunning = false; 1297 bool _seleniumAlreadyRunning = false;
1356 1298
1357 ProcessQueue(this._maxProcesses, 1299 ProcessQueue(this._maxProcesses,
1358 this._maxBrowserProcesses, 1300 this._maxBrowserProcesses,
1359 Date startTime, 1301 DateTime startTime,
1360 testSuites, 1302 testSuites,
1361 this._eventListener, 1303 this._eventListener,
1362 this._allDone, 1304 this._allDone,
1363 [bool verbose = false, 1305 [bool verbose = false,
1364 bool listTests = false]) 1306 bool listTests = false])
1365 : _verbose = verbose, 1307 : _verbose = verbose,
1366 _listTests = listTests, 1308 _listTests = listTests,
1367 _tests = new Queue<TestCase>(), 1309 _tests = new Queue<TestCase>(),
1368 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), 1310 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(),
1369 _testCache = new Map<String, List<TestInformation>>() { 1311 _testCache = new Map<String, List<TestInformation>>() {
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after
1435 String cmd = 'ps'; 1377 String cmd = 'ps';
1436 var arg = ['aux']; 1378 var arg = ['aux'];
1437 if (io.Platform.operatingSystem == 'windows') { 1379 if (io.Platform.operatingSystem == 'windows') {
1438 cmd = 'tasklist'; 1380 cmd = 'tasklist';
1439 arg.add('/v'); 1381 arg.add('/v');
1440 } 1382 }
1441 1383
1442 Future processFuture = io.Process.start(cmd, arg); 1384 Future processFuture = io.Process.start(cmd, arg);
1443 processFuture.then((io.Process p) { 1385 processFuture.then((io.Process p) {
1444 // Drain stderr to not leak resources. 1386 // Drain stderr to not leak resources.
1445 p.stderr.onData = p.stderr.read; 1387 p.stderr.listen((_) {});
1446 final io.StringInputStream stdoutStringStream = 1388 final Stream<String> stdoutStringStream =
1447 new io.StringInputStream(p.stdout); 1389 p.stdout.transform(new io.StringDecoder())
1448 stdoutStringStream.onLine = () { 1390 .transform(new io.LineTransformer());
1449 var line = stdoutStringStream.readLine(); 1391 stdoutStringStream.listen((String line) {
1450 while (null != line) { 1392 var regexp = new RegExp(r".*selenium-server-standalone.*");
1451 var regexp = new RegExp(r".*selenium-server-standalone.*"); 1393 if (regexp.hasMatch(line)) {
1452 if (regexp.hasMatch(line)) { 1394 _seleniumAlreadyRunning = true;
1453 _seleniumAlreadyRunning = true; 1395 resumeTesting();
1454 resumeTesting();
1455 }
1456 line = stdoutStringStream.readLine();
1457 } 1396 }
1458 if (!_isSeleniumAvailable) { 1397 if (!_isSeleniumAvailable) {
1459 _startSeleniumServer(); 1398 _startSeleniumServer();
1460 } 1399 }
1461 }; 1400 });
1462 }).catchError((e) { 1401 }).catchError((e) {
1463 print("Error starting process:"); 1402 print("Error starting process:");
1464 print(" Command: $cmd ${arg.join(' ')}"); 1403 print(" Command: $cmd ${arg.join(' ')}");
1465 print(" Error: $e"); 1404 print(" Error: $e");
1466 // TODO(ahe): How to report this as a test failure? 1405 // TODO(ahe): How to report this as a test failure?
1467 io.exit(1); 1406 io.exit(1);
1468 return true; 1407 return true;
1469 }); 1408 });
1470 } 1409 }
1471 } 1410 }
1472 1411
1473 void _runTest(TestCase test) { 1412 void _runTest(TestCase test) {
1474 if (test.usesWebDriver) { 1413 if (test.usesWebDriver) {
1475 browserUsed = test.configuration['browser']; 1414 browserUsed = test.configuration['browser'];
1476 if (_needsSelenium) _ensureSeleniumServerRunning(); 1415 if (_needsSelenium) _ensureSeleniumServerRunning();
1477 } 1416 }
1478 eventTestAdded(test); 1417 eventTestAdded(test);
1479 _tests.add(test); 1418 _tests.add(test);
1480 _tryRunTest(); 1419 _tryRunTest();
1481 } 1420 }
1482 1421
1483 /** 1422 /**
1484 * Monitor the output of the Selenium server, to know when we are ready to 1423 * Monitor the output of the Selenium server, to know when we are ready to
1485 * begin running tests. 1424 * begin running tests.
1486 * source: Output(Stream) from the Java server. 1425 * source: Output(Stream) from the Java server.
1487 */ 1426 */
1488 VoidFunction makeSeleniumServerHandler(io.StringInputStream source) { 1427 void seleniumServerHandler(String line) {
1489 void handler() { 1428 if (new RegExp(r".*Started.*Server.*").hasMatch(line) ||
1490 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. 1429 new RegExp(r"Exception.*Selenium is already running.*").hasMatch(
1491 var line = source.readLine(); 1430 line)) {
1492 while (null != line) { 1431 resumeTesting();
1493 if (new RegExp(r".*Started.*Server.*").hasMatch(line) ||
1494 new RegExp(r"Exception.*Selenium is already running.*").hasMatch(
1495 line)) {
1496 resumeTesting();
1497 }
1498 line = source.readLine();
1499 }
1500 } 1432 }
1501 return handler;
1502 } 1433 }
1503 1434
1504 /** 1435 /**
1505 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 1436 * For browser tests using Safari or Opera, we need to use the Selenium 1.0
1506 * Java server. 1437 * Java server.
1507 */ 1438 */
1508 void _startSeleniumServer() { 1439 void _startSeleniumServer() {
1509 // Get the absolute path to the Selenium jar. 1440 // Get the absolute path to the Selenium jar.
1510 String filePath = TestUtils.testScriptPath; 1441 String filePath = TestUtils.testScriptPath;
1511 String pathSep = io.Platform.pathSeparator; 1442 String pathSep = io.Platform.pathSeparator;
1512 int index = filePath.lastIndexOf(pathSep); 1443 int index = filePath.lastIndexOf(pathSep);
1513 filePath = '${filePath.substring(0, index)}${pathSep}testing${pathSep}'; 1444 filePath = '${filePath.substring(0, index)}${pathSep}testing${pathSep}';
1514 var lister = new io.Directory(filePath).list(); 1445 new io.Directory(filePath).list().listen((io.FileSystemEntity fse) {
1515 lister.onFile = (String file) { 1446 if (fse is io.File) {
1516 if (new RegExp(r"selenium-server-standalone-.*\.jar").hasMatch(file) 1447 String file = fse.path;
1517 && _seleniumServer == null) { 1448 if (new RegExp(r"selenium-server-standalone-.*\.jar").hasMatch(file)
1518 Future processFuture = io.Process.start('java', ['-jar', file]); 1449 && _seleniumServer == null) {
1519 processFuture.then((io.Process server) { 1450 Future processFuture = io.Process.start('java', ['-jar', file]);
1520 _seleniumServer = server; 1451 processFuture.then((io.Process server) {
1521 // Heads up: there seems to an obscure data race of some form in 1452 _seleniumServer = server;
1522 // the VM between launching the server process and launching the test 1453 // Heads up: there seems to an obscure data race of some form in
1523 // tasks that disappears when you read IO (which is convenient, since 1454 // the VM between launching the server process and launching the
1524 // that is our condition for knowing that the server is ready). 1455 // test tasks that disappears when you read IO (which is
1525 io.StringInputStream stdoutStringStream = 1456 // convenient, since that is our condition for knowing that the
1526 new io.StringInputStream(_seleniumServer.stdout); 1457 // server is ready).
1527 io.StringInputStream stderrStringStream = 1458 Stream<String> stdoutStringStream =
1528 new io.StringInputStream(_seleniumServer.stderr); 1459 _seleniumServer.stdout.transform(new io.StringDecoder())
1529 stdoutStringStream.onLine = 1460 .transform(new io.LineTransformer());
1530 makeSeleniumServerHandler(stdoutStringStream); 1461 Stream<String> stderrStringStream =
1531 stderrStringStream.onLine = 1462 _seleniumServer.stderr.transform(new io.StringDecoder())
1532 makeSeleniumServerHandler(stderrStringStream); 1463 .transform(new io.LineTransformer());
1533 }).catchError((e) { 1464 stdoutStringStream.listen(seleniumServerHandler);
1534 print("Process error:"); 1465 stderrStringStream.listen(seleniumServerHandler);
1535 print(" Command: java -jar $file"); 1466 }).catchError((e) {
1536 print(" Error: $e"); 1467 print("Process error:");
1537 // TODO(ahe): How to report this as a test failure? 1468 print(" Command: java -jar $file");
1538 io.exit(1); 1469 print(" Error: $e");
1539 return true; 1470 // TODO(ahe): How to report this as a test failure?
1540 }); 1471 io.exit(1);
1472 return true;
1473 });
1474 }
1541 } 1475 }
1542 }; 1476 });
1543 } 1477 }
1544 1478
1545 Future _terminateBatchRunners() { 1479 Future _terminateBatchRunners() {
1546 var futures = new List(); 1480 var futures = new List();
1547 for (var runners in _batchProcesses.values) { 1481 for (var runners in _batchProcesses.values) {
1548 for (var runner in runners) { 1482 for (var runner in runners) {
1549 futures.add(runner.terminate()); 1483 futures.add(runner.terminate());
1550 } 1484 }
1551 } 1485 }
1552 // Change to Future.wait when updating binaries. 1486 // Change to Future.wait when updating binaries.
(...skipping 28 matching lines...) Expand all
1581 test.isNegative.toString()]; 1515 test.isNegative.toString()];
1582 fields.addAll(test.commands.last.arguments); 1516 fields.addAll(test.commands.last.arguments);
1583 print(fields.join('\t')); 1517 print(fields.join('\t'));
1584 return; 1518 return;
1585 } 1519 }
1586 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable || (test 1520 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable || (test
1587 is BrowserTestCase && test.waitingForOtherTest)) { 1521 is BrowserTestCase && test.waitingForOtherTest)) {
1588 // The test is not yet ready to run. Put the test back in 1522 // The test is not yet ready to run. Put the test back in
1589 // the queue. Avoid spin-polling by using a timeout. 1523 // the queue. Avoid spin-polling by using a timeout.
1590 _tests.add(test); 1524 _tests.add(test);
1591 new Timer(100, (_) => _tryRunTest()); // Don't lose a process. 1525 new Timer(new Duration(milliseconds: 100),
1526 _tryRunTest); // Don't lose a process.
1592 return; 1527 return;
1593 } 1528 }
1594 // Before running any commands, we print out all commands if '--verbose' 1529 // Before running any commands, we print out all commands if '--verbose'
1595 // was specified. 1530 // was specified.
1596 if (_verbose && test.commandOutputs.length == 0) { 1531 if (_verbose && test.commandOutputs.length == 0) {
1597 int i = 1; 1532 int i = 1;
1598 if (test is BrowserTestCase) { 1533 if (test is BrowserTestCase) {
1599 // Additional command for rerunning the steps locally after the fact. 1534 // Additional command for rerunning the steps locally after the fact.
1600 var command = 1535 var command =
1601 test.configuration["_servers_"].httpServerCommandline(); 1536 test.configuration["_servers_"].httpServerCommandline();
1602 print('$i. $command'); 1537 print('$i. $command');
1603 i++; 1538 i++;
1604 } 1539 }
1605 for (Command command in test.commands) { 1540 for (Command command in test.commands) {
1606 print('$i. $command'); 1541 print('$i. $command');
1607 i++; 1542 i++;
1608 } 1543 }
1609 } 1544 }
1610 1545
1611 var isLastCommand = 1546 var isLastCommand =
1612 ((test.commands.length-1) == test.commandOutputs.length); 1547 ((test.commands.length-1) == test.commandOutputs.length);
1613 var isBrowserCommand = isLastCommand && (test is BrowserTestCase); 1548 var isBrowserCommand = isLastCommand && (test is BrowserTestCase);
1614 if (isBrowserCommand && _numBrowserProcesses == _maxBrowserProcesses) { 1549 if (isBrowserCommand && _numBrowserProcesses == _maxBrowserProcesses) {
1615 // If there is no free browser runner, put it back into the queue. 1550 // If there is no free browser runner, put it back into the queue.
1616 _tests.add(test); 1551 _tests.add(test);
1617 new Timer(100, (_) => _tryRunTest()); // Don't lose a process. 1552 new Timer(new Duration(milliseconds: 100),
1553 _tryRunTest); // Don't lose a process.
1618 return; 1554 return;
1619 } 1555 }
1620 1556
1621 eventStartTestCase(test); 1557 eventStartTestCase(test);
1622 1558
1623 // Analyzer and browser test commands can be run by a [BatchRunnerProcess] 1559 // Analyzer and browser test commands can be run by a [BatchRunnerProcess]
1624 var nextCommandIndex = test.commandOutputs.keys.length; 1560 var nextCommandIndex = test.commandOutputs.keys.length;
1625 var numberOfCommands = test.commands.length; 1561 var numberOfCommands = test.commands.length;
1626 1562
1627 var useBatchRunnerForAnalyzer = 1563 var useBatchRunnerForAnalyzer =
1628 test.configuration['analyzer'] && 1564 test.configuration['analyzer'] &&
1629 test.displayName != 'dartc/junit_tests'; 1565 test.displayName != 'dartc/junit_tests';
1630 var isWebdriverCommand = nextCommandIndex == (numberOfCommands - 1) && 1566 var isWebdriverCommand = nextCommandIndex == (numberOfCommands - 1) &&
1631 test.usesWebDriver && 1567 test.usesWebDriver &&
1632 !test.configuration['noBatch']; 1568 !test.configuration['noBatch'];
1633 if (useBatchRunnerForAnalyzer || isWebdriverCommand) { 1569 if (useBatchRunnerForAnalyzer || isWebdriverCommand) {
1634 TestCaseEvent oldCallback = test.completedHandler; 1570 TestCaseEvent oldCallback = test.completedHandler;
1635 void testCompleted(TestCase test_arg) { 1571 void testCompleted(TestCase test_arg) {
1636 _numProcesses--; 1572 _numProcesses--;
1637 if (isBrowserCommand) { 1573 if (isBrowserCommand) {
(...skipping 142 matching lines...) Expand 10 before | Expand all | Expand 10 after
1780 } 1716 }
1781 } 1717 }
1782 1718
1783 void eventAllTestsDone() { 1719 void eventAllTestsDone() {
1784 for (var listener in _eventListener) { 1720 for (var listener in _eventListener) {
1785 listener.allDone(); 1721 listener.allDone();
1786 } 1722 }
1787 } 1723 }
1788 } 1724 }
1789 1725
OLDNEW
« no previous file with comments | « tools/testing/dart/test_progress.dart ('k') | tools/testing/dart/test_suite.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698