| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| 11 */ | 11 */ |
| 12 library test_runner; | 12 library test_runner; |
| 13 | 13 |
| 14 import "dart:async"; | 14 import "dart:async"; |
| 15 import "dart:collection" show Queue; | 15 import "dart:collection" show Queue; |
| 16 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow | 16 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow |
| 17 // CommandOutput.exitCode in subclasses of CommandOutput. | 17 // CommandOutput.exitCode in subclasses of CommandOutput. |
| 18 import "dart:io" as io; | 18 import "dart:io" as io; |
| 19 import "dart:isolate"; | 19 import "dart:isolate"; |
| 20 import "dart:uri"; | |
| 21 import "browser_controller.dart"; | 20 import "browser_controller.dart"; |
| 22 import "http_server.dart" as http_server; | 21 import "http_server.dart" as http_server; |
| 23 import "status_file_parser.dart"; | 22 import "status_file_parser.dart"; |
| 24 import "test_progress.dart"; | 23 import "test_progress.dart"; |
| 25 import "test_suite.dart"; | 24 import "test_suite.dart"; |
| 26 import "utils.dart"; | 25 import "utils.dart"; |
| 27 import 'record_and_replay.dart'; | 26 import 'record_and_replay.dart'; |
| 28 | 27 |
| 29 const int NO_TIMEOUT = 0; | 28 const int NO_TIMEOUT = 0; |
| 30 const int SLOW_TIMEOUT_MULTIPLIER = 4; | 29 const int SLOW_TIMEOUT_MULTIPLIER = 4; |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 115 // TODO(efortuna): Remove this when fixed (Issue 1306). | 114 // TODO(efortuna): Remove this when fixed (Issue 1306). |
| 116 executable = executable.replaceAll('/', '\\'); | 115 executable = executable.replaceAll('/', '\\'); |
| 117 } | 116 } |
| 118 var quotedArguments = []; | 117 var quotedArguments = []; |
| 119 arguments.forEach((argument) => quotedArguments.add('"$argument"')); | 118 arguments.forEach((argument) => quotedArguments.add('"$argument"')); |
| 120 commandLine = "\"$executable\" ${quotedArguments.join(' ')}"; | 119 commandLine = "\"$executable\" ${quotedArguments.join(' ')}"; |
| 121 } | 120 } |
| 122 | 121 |
| 123 String toString() => commandLine; | 122 String toString() => commandLine; |
| 124 | 123 |
| 125 Future<bool> get outputIsUpToDate => new Future.immediate(false); | 124 Future<bool> get outputIsUpToDate => new Future.value(false); |
| 126 io.Path get expectedOutputFile => null; | 125 io.Path get expectedOutputFile => null; |
| 127 bool get isPixelTest => false; | 126 bool get isPixelTest => false; |
| 128 } | 127 } |
| 129 | 128 |
| 130 class CompilationCommand extends Command { | 129 class CompilationCommand extends Command { |
| 131 String _outputFile; | 130 String _outputFile; |
| 132 bool _neverSkipCompilation; | 131 bool _neverSkipCompilation; |
| 133 List<Uri> _bootstrapDependencies; | 132 List<Uri> _bootstrapDependencies; |
| 134 | 133 |
| 135 CompilationCommand(this._outputFile, | 134 CompilationCommand(this._outputFile, |
| 136 this._neverSkipCompilation, | 135 this._neverSkipCompilation, |
| 137 this._bootstrapDependencies, | 136 this._bootstrapDependencies, |
| 138 String executable, | 137 String executable, |
| 139 List<String> arguments) | 138 List<String> arguments) |
| 140 : super(executable, arguments); | 139 : super(executable, arguments); |
| 141 | 140 |
| 142 Future<bool> get outputIsUpToDate { | 141 Future<bool> get outputIsUpToDate { |
| 143 if (_neverSkipCompilation) return new Future.immediate(false); | 142 if (_neverSkipCompilation) return new Future.value(false); |
| 144 | 143 |
| 145 Future<List<Uri>> readDepsFile(String path) { | 144 Future<List<Uri>> readDepsFile(String path) { |
| 146 var file = new io.File(new io.Path(path).toNativePath()); | 145 var file = new io.File(new io.Path(path).toNativePath()); |
| 147 if (!file.existsSync()) { | 146 if (!file.existsSync()) { |
| 148 return new Future.immediate(null); | 147 return new Future.value(null); |
| 149 } | 148 } |
| 150 return file.readAsLines().then((List<String> lines) { | 149 return file.readAsLines().then((List<String> lines) { |
| 151 var dependencies = new List<Uri>(); | 150 var dependencies = new List<Uri>(); |
| 152 for (var line in lines) { | 151 for (var line in lines) { |
| 153 line = line.trim(); | 152 line = line.trim(); |
| 154 if (line.length > 0) { | 153 if (line.length > 0) { |
| 155 dependencies.add(new Uri(line)); | 154 dependencies.add(Uri.parse(line)); |
| 156 } | 155 } |
| 157 } | 156 } |
| 158 return dependencies; | 157 return dependencies; |
| 159 }); | 158 }); |
| 160 } | 159 } |
| 161 | 160 |
| 162 return readDepsFile("$_outputFile.deps").then((dependencies) { | 161 return readDepsFile("$_outputFile.deps").then((dependencies) { |
| 163 if (dependencies != null) { | 162 if (dependencies != null) { |
| 164 dependencies.addAll(_bootstrapDependencies); | 163 dependencies.addAll(_bootstrapDependencies); |
| 165 var jsOutputLastModified = TestUtils.lastModifiedCache.getLastModified( | 164 var jsOutputLastModified = TestUtils.lastModifiedCache.getLastModified( |
| 166 new Uri.fromComponents(scheme: 'file', path: _outputFile)); | 165 new Uri(scheme: 'file', path: _outputFile)); |
| 167 if (jsOutputLastModified != null) { | 166 if (jsOutputLastModified != null) { |
| 168 for (var dependency in dependencies) { | 167 for (var dependency in dependencies) { |
| 169 var dependencyLastModified = | 168 var dependencyLastModified = |
| 170 TestUtils.lastModifiedCache.getLastModified(dependency); | 169 TestUtils.lastModifiedCache.getLastModified(dependency); |
| 171 if (dependencyLastModified == null || | 170 if (dependencyLastModified == null || |
| 172 dependencyLastModified.isAfter(jsOutputLastModified)) { | 171 dependencyLastModified.isAfter(jsOutputLastModified)) { |
| 173 return false; | 172 return false; |
| 174 } | 173 } |
| 175 } | 174 } |
| 176 return true; | 175 return true; |
| (...skipping 844 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1021 _runCommand(); | 1020 _runCommand(); |
| 1022 return completer.future; | 1021 return completer.future; |
| 1023 } | 1022 } |
| 1024 | 1023 |
| 1025 void _runCommand() { | 1024 void _runCommand() { |
| 1026 command.outputIsUpToDate.then((bool isUpToDate) { | 1025 command.outputIsUpToDate.then((bool isUpToDate) { |
| 1027 if (isUpToDate) { | 1026 if (isUpToDate) { |
| 1028 compilationSkipped = true; | 1027 compilationSkipped = true; |
| 1029 _commandComplete(0); | 1028 _commandComplete(0); |
| 1030 } else { | 1029 } else { |
| 1031 var processOptions = _createProcessOptions(); | 1030 var processEnvironment = _createProcessEnvironment(); |
| 1032 var commandArguments = _modifySeleniumTimeout(command.arguments, | 1031 var commandArguments = _modifySeleniumTimeout(command.arguments, |
| 1033 testCase.timeout); | 1032 testCase.timeout); |
| 1034 Future processFuture = io.Process.start(command.executable, | 1033 Future processFuture = |
| 1035 commandArguments, | 1034 io.Process.start(command.executable, |
| 1036 processOptions); | 1035 commandArguments, |
| 1036 environment: processEnvironment); |
| 1037 processFuture.then((io.Process process) { | 1037 processFuture.then((io.Process process) { |
| 1038 // Close stdin so that tests that try to block on input will fail. | 1038 // Close stdin so that tests that try to block on input will fail. |
| 1039 process.stdin.close(); | 1039 process.stdin.close(); |
| 1040 void timeoutHandler() { | 1040 void timeoutHandler() { |
| 1041 timedOut = true; | 1041 timedOut = true; |
| 1042 if (process != null) { | 1042 if (process != null) { |
| 1043 process.kill(); | 1043 process.kill(); |
| 1044 } | 1044 } |
| 1045 } | 1045 } |
| 1046 process.exitCode.then(_commandComplete); | 1046 process.exitCode.then(_commandComplete); |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1080 stderr, | 1080 stderr, |
| 1081 new DateTime.now().difference(startTime), | 1081 new DateTime.now().difference(startTime), |
| 1082 compilationSkipped); | 1082 compilationSkipped); |
| 1083 return commandOutput; | 1083 return commandOutput; |
| 1084 } | 1084 } |
| 1085 | 1085 |
| 1086 void _drainStream(Stream<List<int>> source, List<int> destination) { | 1086 void _drainStream(Stream<List<int>> source, List<int> destination) { |
| 1087 source.listen(destination.addAll); | 1087 source.listen(destination.addAll); |
| 1088 } | 1088 } |
| 1089 | 1089 |
| 1090 io.ProcessOptions _createProcessOptions() { | 1090 Map<String, String> _createProcessEnvironment() { |
| 1091 var baseEnvironment = command.environment != null ? | 1091 var baseEnvironment = command.environment != null ? |
| 1092 command.environment : io.Platform.environment; | 1092 command.environment : io.Platform.environment; |
| 1093 io.ProcessOptions options = new io.ProcessOptions(); | 1093 var environment = new Map<String, String>.from(baseEnvironment); |
| 1094 options.environment = new Map<String, String>.from(baseEnvironment); | 1094 environment['DART_CONFIGURATION'] = |
| 1095 options.environment['DART_CONFIGURATION'] = | |
| 1096 TestUtils.configurationDir(testCase.configuration); | 1095 TestUtils.configurationDir(testCase.configuration); |
| 1097 | 1096 |
| 1098 for (var excludedEnvironmentVariable in EXCLUDED_ENVIRONMENT_VARIABLES) { | 1097 for (var excludedEnvironmentVariable in EXCLUDED_ENVIRONMENT_VARIABLES) { |
| 1099 options.environment.remove(excludedEnvironmentVariable); | 1098 environment.remove(excludedEnvironmentVariable); |
| 1100 } | 1099 } |
| 1101 | 1100 |
| 1102 return options; | 1101 return environment; |
| 1103 } | 1102 } |
| 1104 } | 1103 } |
| 1105 | 1104 |
| 1106 class BatchRunnerProcess { | 1105 class BatchRunnerProcess { |
| 1107 Command _command; | 1106 Command _command; |
| 1108 String _executable; | 1107 String _executable; |
| 1109 List<String> _batchArguments; | 1108 List<String> _batchArguments; |
| 1110 | 1109 |
| 1111 io.Process _process; | 1110 io.Process _process; |
| 1112 Completer _stdoutCompleter; | 1111 Completer _stdoutCompleter; |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1152 doStartTest(testCase); | 1151 doStartTest(testCase); |
| 1153 }); | 1152 }); |
| 1154 }; | 1153 }; |
| 1155 _process.kill(); | 1154 _process.kill(); |
| 1156 } else { | 1155 } else { |
| 1157 doStartTest(testCase); | 1156 doStartTest(testCase); |
| 1158 } | 1157 } |
| 1159 } | 1158 } |
| 1160 | 1159 |
| 1161 Future terminate() { | 1160 Future terminate() { |
| 1162 if (_process == null) return new Future.immediate(true); | 1161 if (_process == null) return new Future.value(true); |
| 1163 Completer completer = new Completer(); | 1162 Completer completer = new Completer(); |
| 1164 Timer killTimer; | 1163 Timer killTimer; |
| 1165 _processExitHandler = (_) { | 1164 _processExitHandler = (_) { |
| 1166 if (killTimer != null) killTimer.cancel(); | 1165 if (killTimer != null) killTimer.cancel(); |
| 1167 completer.complete(true); | 1166 completer.complete(true); |
| 1168 }; | 1167 }; |
| 1169 if (_isWebDriver) { | 1168 if (_isWebDriver) { |
| 1170 // Use a graceful shutdown so our Selenium script can close | 1169 // Use a graceful shutdown so our Selenium script can close |
| 1171 // the open browser processes. On Windows, signals do not exist | 1170 // the open browser processes. On Windows, signals do not exist |
| 1172 // and a kill is a hard kill. | 1171 // and a kill is a hard kill. |
| (...skipping 483 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1656 testRunner.logger = DebugLogger.info; | 1655 testRunner.logger = DebugLogger.info; |
| 1657 _browserTestRunners[runtime] = testRunner; | 1656 _browserTestRunners[runtime] = testRunner; |
| 1658 return testRunner.start().then((started) { | 1657 return testRunner.start().then((started) { |
| 1659 if (started) { | 1658 if (started) { |
| 1660 return testRunner; | 1659 return testRunner; |
| 1661 } | 1660 } |
| 1662 print("Issue starting browser test runner"); | 1661 print("Issue starting browser test runner"); |
| 1663 io.exit(1); | 1662 io.exit(1); |
| 1664 }); | 1663 }); |
| 1665 } | 1664 } |
| 1666 return new Future.immediate(_browserTestRunners[runtime]); | 1665 return new Future.value(_browserTestRunners[runtime]); |
| 1667 } | 1666 } |
| 1668 | 1667 |
| 1669 void _startBrowserControllerTest(var test) { | 1668 void _startBrowserControllerTest(var test) { |
| 1670 var callback = (var output, var duration) { | 1669 var callback = (var output, var duration) { |
| 1671 var nextCommandIndex = test.commandOutputs.keys.length; | 1670 var nextCommandIndex = test.commandOutputs.keys.length; |
| 1672 new CommandOutput.fromCase(test, | 1671 new CommandOutput.fromCase(test, |
| 1673 test.commands[nextCommandIndex], | 1672 test.commands[nextCommandIndex], |
| 1674 0, | 1673 0, |
| 1675 false, | 1674 false, |
| 1676 output == "TIMEOUT", | 1675 output == "TIMEOUT", |
| (...skipping 233 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1910 } | 1909 } |
| 1911 } | 1910 } |
| 1912 | 1911 |
| 1913 void eventAllTestsDone() { | 1912 void eventAllTestsDone() { |
| 1914 for (var listener in _eventListener) { | 1913 for (var listener in _eventListener) { |
| 1915 listener.allDone(); | 1914 listener.allDone(); |
| 1916 } | 1915 } |
| 1917 } | 1916 } |
| 1918 } | 1917 } |
| 1919 | 1918 |
| OLD | NEW |